Merge branch 'main' into 6866/model-versions2

.changes/unreleased/Features-20230325-192830.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Added prettier printing to ContractError class
+time: 2023-03-25T19:28:30.171461-07:00
+custom:
+  Author: kentkr
+  Issue: "7209"

.changes/unreleased/Features-20230329-102736.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add support for model-level constraints
+time: 2023-03-29T10:27:36.94177-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "6754"

.changes/unreleased/Features-20230410-115538.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add --no-populate-cache to optionally skip relation cache population
+time: 2023-04-10T11:55:38.360997-05:00
+custom:
+  Author: stu-k
+  Issue: "1751"

.changes/unreleased/Features-20230410-174824.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Add relation info (database, schema, alias) to node_info dictionary in structured
+  logging
+time: 2023-04-10T17:48:24.870944+02:00
+custom:
+  Author: jtcohen6
+  Issue: "6724"

.changes/unreleased/Features-20230411-084645.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: select resources by patch path
+time: 2023-04-11T08:46:45.641682-05:00
+custom:
+  Author: dave-connors-3
+  Issue: "7315"

.changes/unreleased/Fixes-20230329-113203.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Handle internal exceptions
+time: 2023-03-29T11:32:03.259072-05:00
+custom:
+  Author: stu-k
+  Issue: "7118"

.changes/unreleased/Fixes-20230405-124315.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: avoid dbtRunner default callbacks being shared across instances
+time: 2023-04-05T12:43:15.418016-04:00
+custom:
+  Author: chamini2
+  Issue: "7278"

.changes/unreleased/Fixes-20230410-105007.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Duplicated flags now throw errors instead of being overidden by parent-level
+  flag
+time: 2023-04-10T10:50:07.178386-05:00
+custom:
+  Author: iknox-fa
+  Issue: "6913"

.changes/unreleased/Under the Hood-20230331-132119.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Generalize constraint compatibility warnings
+time: 2023-03-31T13:21:19.507995-05:00
+custom:
+  Author: emmyoop
+  Issue: "7067"

.changes/unreleased/Under the Hood-20230410-140228.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add unique_id to ShowNode and CompiledNode logging events
+time: 2023-04-10T14:02:28.86602-04:00
+custom:
+  Author: gshank
+  Issue: "7305"

.changes/unreleased/Under the Hood-20230410-164543.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: '`Parse` now returns manifest when invoked via dbtRunner'
+time: 2023-04-10T16:45:43.17915-05:00
+custom:
+  Author: iknox-fa
+  Issue: "6547"

.changes/unreleased/Under the Hood-20230410-214724.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Prettify message for ListRelations event
+time: 2023-04-10T21:47:24.052122+02:00
+custom:
+  Author: jtcohen6
+  Issue: "7310"

core/dbt/adapters/base/__init__.py

@@ -1,14 +1,19 @@
 # these are all just exports, #noqa them so flake8 will be happy
 
 # TODO: Should we still include this in the `adapters` namespace?
-from dbt.contracts.connection import Credentials  # noqa
-from dbt.adapters.base.meta import available  # noqa
-from dbt.adapters.base.connections import BaseConnectionManager  # noqa
-from dbt.adapters.base.relation import (  # noqa
+from dbt.contracts.connection import Credentials  # noqa: F401
+from dbt.adapters.base.meta import available  # noqa: F401
+from dbt.adapters.base.connections import BaseConnectionManager  # noqa: F401
+from dbt.adapters.base.relation import (  # noqa: F401
     BaseRelation,
     RelationType,
     SchemaSearchMap,
 )
-from dbt.adapters.base.column import Column  # noqa
-from dbt.adapters.base.impl import AdapterConfig, BaseAdapter, PythonJobHelper  # noqa
-from dbt.adapters.base.plugin import AdapterPlugin  # noqa
+from dbt.adapters.base.column import Column  # noqa: F401
+from dbt.adapters.base.impl import (  # noqa: F401
+    AdapterConfig,
+    BaseAdapter,
+    PythonJobHelper,
+    ConstraintSupport,
+)
+from dbt.adapters.base.plugin import AdapterPlugin  # noqa: F401

core/dbt/adapters/base/impl.py

@@ -2,6 +2,7 @@
 from concurrent.futures import as_completed, Future
 from contextlib import contextmanager
 from datetime import datetime
+from enum import Enum
 import time
 from itertools import chain
 from typing import (
@@ -16,9 +17,10 @@
     Set,
     Tuple,
     Type,
+    Union,
 )
 
-from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType
+from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType, ModelLevelConstraint
 
 import agate
 import pytz
@@ -53,6 +55,8 @@
     CodeExecution,
     CodeExecutionStatus,
     CatalogGenerationError,
+    ConstraintNotSupported,
+    ConstraintNotEnforced,
 )
 from dbt.utils import filter_null_values, executor, cast_to_str, AttrDict
 
@@ -73,6 +77,12 @@
 FRESHNESS_MACRO_NAME = "collect_freshness"
 
 
+class ConstraintSupport(str, Enum):
+    ENFORCED = "enforced"
+    NOT_ENFORCED = "not_enforced"
+    NOT_SUPPORTED = "not_supported"
+
+
 def _expect_row_value(key: str, row: agate.Row):
     if key not in row.keys():
         raise DbtInternalError(
@@ -204,6 +214,14 @@ class BaseAdapter(metaclass=AdapterMeta):
     # for use in materializations
     AdapterSpecificConfigs: Type[AdapterConfig] = AdapterConfig
 
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.foreign_key: ConstraintSupport.ENFORCED,
+    }
+
     def __init__(self, config):
         self.config = config
         self.cache = RelationsCache()
@@ -718,6 +736,18 @@ def list_relations(self, database: Optional[str], schema: str) -> List[BaseRelat
         # we can't build the relations cache because we don't have a
         # manifest so we can't run any operations.
         relations = self.list_relations_without_caching(schema_relation)
+
+        # if the cache is already populated, add this schema in
+        # otherwise, skip updating the cache and just ignore
+        if self.cache:
+            for relation in relations:
+                self.cache.add(relation)
+            if not relations:
+                # it's possible that there were no relations in some schemas. We want
+                # to insert the schemas we query into the cache's `.schemas` attribute
+                # so we can check it later
+                self.cache.update_schemas([(database, schema)])
+
         fire_event(
             ListRelations(
                 database=cast_to_str(database),
@@ -1273,14 +1303,8 @@ def _parse_column_constraint(cls, raw_constraint: Dict[str, Any]) -> ColumnLevel
         except Exception:
             raise DbtValidationError(f"Could not parse constraint: {raw_constraint}")
 
-    @available
-    @classmethod
-    def render_raw_column_constraint(cls, raw_constraint: Dict[str, Any]) -> str:
-        constraint = cls._parse_column_constraint(raw_constraint)
-        return cls.render_column_constraint(constraint)
-
     @classmethod
-    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> str:
+    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]:
         """Render the given constraint as DDL text. Should be overriden by adapters which need custom constraint
         rendering."""
         if constraint.type == ConstraintType.check and constraint.expression:
@@ -1296,7 +1320,84 @@ def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> str:
         elif constraint.type == ConstraintType.custom and constraint.expression:
             return constraint.expression
         else:
-            return ""
+            return None
+
+    @available
+    @classmethod
+    def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List:
+        rendered_column_constraints = []
+
+        for v in raw_columns.values():
+            rendered_column_constraint = [f"{v['name']} {v['data_type']}"]
+            for con in v.get("constraints", None):
+                constraint = cls._parse_column_constraint(con)
+                c = cls.process_parsed_constraint(constraint, cls.render_column_constraint)
+                if c is not None:
+                    rendered_column_constraint.append(c)
+            rendered_column_constraints.append(" ".join(rendered_column_constraint))
+
+        return rendered_column_constraints
+
+    @classmethod
+    def process_parsed_constraint(
+        cls, parsed_constraint: Union[ColumnLevelConstraint, ModelLevelConstraint], render_func
+    ) -> Optional[str]:
+        if (
+            parsed_constraint.warn_unsupported
+            and cls.CONSTRAINT_SUPPORT[parsed_constraint.type] == ConstraintSupport.NOT_SUPPORTED
+        ):
+            warn_or_error(
+                ConstraintNotSupported(constraint=parsed_constraint.type.value, adapter=cls.type())
+            )
+        if (
+            parsed_constraint.warn_unenforced
+            and cls.CONSTRAINT_SUPPORT[parsed_constraint.type] == ConstraintSupport.NOT_ENFORCED
+        ):
+            warn_or_error(
+                ConstraintNotEnforced(constraint=parsed_constraint.type.value, adapter=cls.type())
+            )
+        if cls.CONSTRAINT_SUPPORT[parsed_constraint.type] != ConstraintSupport.NOT_SUPPORTED:
+            return render_func(parsed_constraint)
+
+        return None
+
+    @classmethod
+    def _parse_model_constraint(cls, raw_constraint: Dict[str, Any]) -> ModelLevelConstraint:
+        try:
+            ModelLevelConstraint.validate(raw_constraint)
+            c = ModelLevelConstraint.from_dict(raw_constraint)
+            return c
+        except Exception:
+            raise DbtValidationError(f"Could not parse constraint: {raw_constraint}")
+
+    @available
+    @classmethod
+    def render_raw_model_constraints(cls, raw_constraints: List[Dict[str, Any]]) -> List[str]:
+        return [c for c in map(cls.render_raw_model_constraint, raw_constraints) if c is not None]
+
+    @classmethod
+    def render_raw_model_constraint(cls, raw_constraint: Dict[str, Any]) -> Optional[str]:
+        constraint = cls._parse_model_constraint(raw_constraint)
+        return cls.process_parsed_constraint(constraint, cls.render_model_constraint)
+
+    @classmethod
+    def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]:
+        """Render the given constraint as DDL text. Should be overriden by adapters which need custom constraint
+        rendering."""
+        constraint_prefix = f"constraint {constraint.name} " if constraint.name else ""
+        column_list = ", ".join(constraint.columns)
+        if constraint.type == ConstraintType.check and constraint.expression:
+            return f"{constraint_prefix}check {constraint.expression}"
+        elif constraint.type == ConstraintType.unique:
+            return f"{constraint_prefix}unique ({column_list})"
+        elif constraint.type == ConstraintType.primary_key:
+            return f"{constraint_prefix}primary key ({column_list})"
+        elif constraint.type == ConstraintType.foreign_key:
+            return f"{constraint_prefix}foreign key ({column_list})"
+        elif constraint.type == ConstraintType.custom and constraint.expression:
+            return f"{constraint_prefix}{constraint.expression}"
+        else:
+            return None
 
 
 COLUMNS_EQUAL_SQL = """

core/dbt/cli/README.md

@@ -1 +1,49 @@
-TODO
+# Exception Handling
+
+## `requires.py`
+
+### `postflight`
+In the postflight decorator, the click command is invoked (i.e. `func(*args, **kwargs)`) and wrapped in a `try/except` block to handle any exceptions thrown.
+Any exceptions thrown from `postflight` are wrapped by custom exceptions from the `dbt.cli.exceptions` module (i.e. `ResultExit`, `ExceptionExit`) to instruct click to complete execution with a particular exit code.
+
+Some `dbt-core` handled exceptions have an attribute named `results` which contains results from running nodes (e.g. `FailFastError`). These are wrapped in the `ResultExit` exception to represent runs that have failed in a way that `dbt-core` expects.
+If the invocation of the command does not throw any exceptions but does not succeed, `postflight` will still raise the `ResultExit` exception to make use of the exit code.
+These exceptions produce an exit code of `1`.
+
+Exceptions wrapped with `ExceptionExit` may be thrown by `dbt-core` intentionally (i.e. an exception that inherits from `dbt.exceptions.Exception`) or unintentionally (i.e. exceptions thrown by the python runtime). In either case these are considered errors that `dbt-core` did not expect and are treated as genuine exceptions.
+These exceptions produce an exit code of `2`.
+
+If no exceptions are thrown from invoking the command and the command succeeds, `postflight` will not raise any exceptions.
+When no exceptions are raised an exit code of `0` is produced.
+
+## `main.py`
+
+### `dbtRunner`
+`dbtRunner` provides a programmatic interface for our click CLI and wraps the invocation of the click commands to handle any exceptions thrown.
+
+`dbtRunner.invoke` should ideally only ever return an instantiated `dbtRunnerResult` which contains the following fields:
+- `success`: A boolean representing whether the command invocation was successful
+- `result`: The optional result of the command invoked. This attribute can have many types, please see the definition of `dbtRunnerResult` for more information
+- `exception`: If an exception was thrown during command invocation it will be saved here, otherwise it will be `None`. Please note that the exceptions held in this attribute are not the exceptions thrown by `preflight` but instead the exceptions that `ResultExit` and `ExceptionExit` wrap
+
+Programmatic exception handling might look like the following:
+```python
+res = dbtRunner().invoke(["run"])
+if not res.success:
+    ...
+if type(res.exception) == SomeExceptionType:
+    ...
+```
+
+## `dbt/tests/util.py`
+
+### `run_dbt`
+In many of our functional and integration tests, we want to be sure that an invocation of `dbt` raises a certain exception.
+A common pattern for these assertions:
+```python
+class TestSomething:
+    def test_something(self, project):
+        with pytest.raises(SomeException):
+            run_dbt(["run"])
+```
+To allow these tests to assert that exceptions have been thrown, the `run_dbt` function will raise any exceptions it recieves from the invocation of a `dbt` command.