Merge branch 'main' into unit_testing_feature_branch

.changes/unreleased/Features-20231116-234049.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support --empty flag for schema-only dry runs
+time: 2023-11-16T23:40:49.96651-05:00
+custom:
+  Author: michelleark
+  Issue: "8971"

.changes/unreleased/Fixes-20231127-154310.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: 'deps: Lock git packages to commit SHA during resolution'
+time: 2023-11-27T15:43:10.122069+01:00
+custom:
+  Author: jtcohen6
+  Issue: "9050"

.changes/unreleased/Fixes-20231127-154347.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: 'deps: Use PackageRenderer to read package-lock.json'
+time: 2023-11-27T15:43:47.842423+01:00
+custom:
+  Author: jtcohen6
+  Issue: "9127"

.changes/unreleased/Fixes-20231128-155225.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: 'Get sources working again in dbt docs generate'
+time: 2023-11-28T15:52:25.738256Z
+custom:
+  Author: aranke
+  Issue: "9119"

.changes/unreleased/Under the Hood-20231111-175350.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Fix test_current_timestamp_matches_utc test; allow for MacOS runner system clock
+  variance
+time: 2023-11-11T17:53:50.098843-05:00
+custom:
+  Author: mikealfare
+  Issue: "9057"

.changes/unreleased/Under the Hood-20231120-134735.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Clean up unused adaptor folders
+time: 2023-11-20T13:47:35.923794-08:00
+custom:
+  Author: ChenyuLInx
+  Issue: "9123"

core/dbt/adapters/base/relation.py

@@ -42,6 +42,7 @@ class BaseRelation(FakeAPIObject, Hashable):
     include_policy: Policy = field(default_factory=lambda: Policy())
     quote_policy: Policy = field(default_factory=lambda: Policy())
     dbt_created: bool = False
+    limit: Optional[int] = None
 
     # register relation types that can be renamed for the purpose of replacing relations using stages and backups
     # adding a relation type here also requires defining the associated rename macro
@@ -200,6 +201,15 @@ def render(self) -> str:
         # if there is nothing set, this will return the empty string.
         return ".".join(part for _, part in self._render_iterator() if part is not None)
 
+    def render_limited(self) -> str:
+        rendered = self.render()
+        if self.limit is None:
+            return rendered
+        elif self.limit == 0:
+            return f"(select * from {rendered} where false limit 0) _dbt_limit_subq"
+        else:
+            return f"(select * from {rendered} limit {self.limit}) _dbt_limit_subq"
+
     def quoted(self, identifier):
         return "{quote_char}{identifier}{quote_char}".format(
             quote_char=self.quote_character,
@@ -235,13 +245,11 @@ def create_ephemeral_from_node(
         cls: Type[Self],
         config: HasQuoting,
         node: ManifestNode,
+        limit: Optional[int],
     ) -> Self:
         # Note that ephemeral models are based on the name.
         identifier = cls.add_ephemeral_prefix(node.name)
-        return cls.create(
-            type=cls.CTE,
-            identifier=identifier,
-        ).quote(identifier=False)
+        return cls.create(type=cls.CTE, identifier=identifier, limit=limit).quote(identifier=False)
 
     @classmethod
     def create_from_node(
@@ -323,7 +331,7 @@ def __hash__(self) -> int:
         return hash(self.render())
 
     def __str__(self) -> str:
-        return self.render()
+        return self.render() if self.limit is None else self.render_limited()
 
     @property
     def database(self) -> Optional[str]:

core/dbt/cli/main.py

@@ -342,6 +342,7 @@ def docs_serve(ctx, **kwargs):
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.empty
 @p.select
 @p.selector
 @p.inline
@@ -599,6 +600,7 @@ def parse(ctx, **kwargs):
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.empty
 @p.select
 @p.selector
 @p.state

core/dbt/cli/params.py

@@ -90,6 +90,12 @@
     is_flag=True,
 )
 
+empty = click.option(
+    "--empty/--no-empty",
+    envvar="DBT_EMPTY",
+    help="If specified, limit input refs and sources to zero rows.",
+    is_flag=True,
+)
 
 enable_legacy_logger = click.option(
     "--enable-legacy-logger/--no-enable-legacy-logger",

core/dbt/clients/git.py

@@ -111,7 +111,7 @@ def checkout(cwd, repo, revision=None):
 def get_current_sha(cwd):
     out, err = run_cmd(cwd, ["git", "rev-parse", "HEAD"], env={"LC_ALL": "C"})
 
-    return out.decode("utf-8")
+    return out.decode("utf-8").strip()
 
 
 def remove_remote(cwd):

core/dbt/config/project.py

@@ -128,10 +128,19 @@ def package_and_project_data_from_root(project_root):
     return packages_dict, packages_specified_path
 
 
-def package_config_from_data(packages_data: Dict[str, Any]) -> PackageConfig:
+def package_config_from_data(
+    packages_data: Dict[str, Any],
+    unrendered_packages_data: Optional[Dict[str, Any]] = None,
+) -> PackageConfig:
     if not packages_data:
         packages_data = {"packages": []}
 
+    # this depends on the two lists being in the same order
+    if unrendered_packages_data:
+        unrendered_packages_data = deepcopy(unrendered_packages_data)
+        for i in range(0, len(packages_data.get("packages", []))):
+            packages_data["packages"][i]["unrendered"] = unrendered_packages_data["packages"][i]
+
     if PACKAGE_LOCK_HASH_KEY in packages_data:
         packages_data.pop(PACKAGE_LOCK_HASH_KEY)
     try:
@@ -326,7 +335,7 @@ def render(self, renderer: DbtProjectYamlRenderer) -> "Project":
 
     def render_package_metadata(self, renderer: PackageRenderer) -> ProjectPackageMetadata:
         packages_data = renderer.render_data(self.packages_dict)
-        packages_config = package_config_from_data(packages_data)
+        packages_config = package_config_from_data(packages_data, self.packages_dict)
         if not self.project_name:
             raise DbtProjectError("Package dbt_project.yml must have a name!")
         return ProjectPackageMetadata(self.project_name, packages_config.packages)
@@ -463,8 +472,9 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         on_run_end: List[str] = value_or(cfg.on_run_end, [])
 
         query_comment = _query_comment_from_cfg(cfg.query_comment)
-
-        packages: PackageConfig = package_config_from_data(rendered.packages_dict)
+        packages: PackageConfig = package_config_from_data(
+            rendered.packages_dict, unrendered.packages_dict
+        )
         selectors = selector_config_from_data(rendered.selectors_dict)
         manifest_selectors: Dict[str, Any] = {}
         if rendered.selectors_dict and rendered.selectors_dict["selectors"]:

core/dbt/context/providers.py

@@ -218,6 +218,10 @@ def current_project(self):
     def Relation(self):
         return self.db_wrapper.Relation
 
+    @property
+    def resolve_limit(self) -> Optional[int]:
+        return 0 if getattr(self.config.args, "EMPTY", False) else None
+
     @abc.abstractmethod
     def __call__(self, *args: str) -> Union[str, RelationProxy, MetricReference]:
         pass
@@ -533,9 +537,11 @@ def resolve(
     def create_relation(self, target_model: ManifestNode) -> RelationProxy:
         if target_model.is_ephemeral_model:
             self.model.set_cte(target_model.unique_id, None)
-            return self.Relation.create_ephemeral_from_node(self.config, target_model)
+            return self.Relation.create_ephemeral_from_node(
+                self.config, target_model, limit=self.resolve_limit
+            )
         else:
-            return self.Relation.create_from(self.config, target_model)
+            return self.Relation.create_from(self.config, target_model, limit=self.resolve_limit)
 
     def validate(
         self,
@@ -603,7 +609,7 @@ def resolve(self, source_name: str, table_name: str):
                 target_kind="source",
                 disabled=(isinstance(target_source, Disabled)),
             )
-        return self.Relation.create_from_source(target_source)
+        return self.Relation.create_from_source(target_source, limit=self.resolve_limit)
 
 
 class RuntimeUnitTestSourceResolver(BaseSourceResolver):

core/dbt/contracts/project.py

@@ -46,6 +46,7 @@ class Package(dbtClassMixin, Replaceable):
 @dataclass
 class LocalPackage(Package):
     local: str
+    unrendered: Dict[str, Any] = field(default_factory=dict)
 
 
 # `float` also allows `int`, according to PEP484 (and jsonschema!)
@@ -56,6 +57,7 @@ class LocalPackage(Package):
 class TarballPackage(Package):
     tarball: str
     name: str
+    unrendered: Dict[str, Any] = field(default_factory=dict)
 
 
 @dataclass
@@ -64,6 +66,7 @@ class GitPackage(Package):
     revision: Optional[RawVersion] = None
     warn_unpinned: Optional[bool] = field(default=None, metadata={"alias": "warn-unpinned"})
     subdirectory: Optional[str] = None
+    unrendered: Dict[str, Any] = field(default_factory=dict)
 
     def get_revisions(self) -> List[str]:
         if self.revision is None:
@@ -77,6 +80,7 @@ class RegistryPackage(Package):
     package: str
     version: Union[RawVersion, List[RawVersion]]
     install_prerelease: Optional[bool] = False
+    unrendered: Dict[str, Any] = field(default_factory=dict)
 
     def get_versions(self) -> List[str]:
         if isinstance(self.version, list):

core/dbt/deps/git.py

@@ -9,9 +9,9 @@
     GitPackage,
 )
 from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
-from dbt.exceptions import ExecutableError, MultipleVersionGitDepsError
+from dbt.exceptions import ExecutableError, MultipleVersionGitDepsError, scrub_secrets, env_secrets
 from dbt.events.functions import fire_event, warn_or_error
-from dbt.events.types import EnsureGitInstalled, DepsUnpinned
+from dbt.events.types import EnsureGitInstalled, DepsUnpinned, DepsScrubbedPackageName
 from dbt.utils import md5
 
 
@@ -23,10 +23,12 @@ class GitPackageMixin:
     def __init__(
         self,
         git: str,
+        git_unrendered: str,
         subdirectory: Optional[str] = None,
     ) -> None:
         super().__init__()
         self.git = git
+        self.git_unrendered = git_unrendered
         self.subdirectory = subdirectory
 
     @property
@@ -41,19 +43,23 @@ class GitPinnedPackage(GitPackageMixin, PinnedPackage):
     def __init__(
         self,
         git: str,
+        git_unrendered: str,
         revision: str,
         warn_unpinned: bool = True,
         subdirectory: Optional[str] = None,
     ) -> None:
-        super().__init__(git, subdirectory)
+        super().__init__(git, git_unrendered, subdirectory)
         self.revision = revision
         self.warn_unpinned = warn_unpinned
         self.subdirectory = subdirectory
         self._checkout_name = md5sum(self.name)
 
     def to_dict(self) -> Dict[str, str]:
+        git_scrubbed = scrub_secrets(self.git_unrendered, env_secrets())
+        if self.git_unrendered != git_scrubbed:
+            warn_or_error(DepsScrubbedPackageName(package_name=git_scrubbed))
         ret = {
-            "git": self.git,
+            "git": git_scrubbed,
             "revision": self.revision,
         }
         if self.subdirectory:
@@ -96,8 +102,13 @@ def _fetch_metadata(
     ) -> ProjectPackageMetadata:
         path = self._checkout()
 
+        # raise warning (or error) if this package is not pinned
         if (self.revision == "HEAD" or self.revision in ("main", "master")) and self.warn_unpinned:
-            warn_or_error(DepsUnpinned(git=self.git))
+            warn_or_error(DepsUnpinned(revision=self.revision, git=self.git))
+
+        # now overwrite 'revision' with actual commit SHA
+        self.revision = git.get_current_sha(path)
+
         partial = PartialProject.from_project_root(path)
         return partial.render_package_metadata(renderer)
 
@@ -116,11 +127,12 @@ class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
     def __init__(
         self,
         git: str,
+        git_unrendered: str,
         revisions: List[str],
         warn_unpinned: bool = True,
         subdirectory: Optional[str] = None,
     ) -> None:
-        super().__init__(git, subdirectory)
+        super().__init__(git, git_unrendered, subdirectory)
         self.revisions = revisions
         self.warn_unpinned = warn_unpinned
         self.subdirectory = subdirectory
@@ -133,6 +145,7 @@ def from_contract(cls, contract: GitPackage) -> "GitUnpinnedPackage":
         warn_unpinned = contract.warn_unpinned is not False
         return cls(
             git=contract.git,
+            git_unrendered=(contract.unrendered.get("git") or contract.git),
             revisions=revisions,
             warn_unpinned=warn_unpinned,
             subdirectory=contract.subdirectory,
@@ -157,6 +170,7 @@ def incorporate(self, other: "GitUnpinnedPackage") -> "GitUnpinnedPackage":
 
         return GitUnpinnedPackage(
             git=self.git,
+            git_unrendered=self.git_unrendered,
             revisions=self.revisions + other.revisions,
             warn_unpinned=warn_unpinned,
             subdirectory=self.subdirectory,
@@ -168,9 +182,9 @@ def resolved(self) -> GitPinnedPackage:
             requested = {"HEAD"}
         elif len(requested) > 1:
             raise MultipleVersionGitDepsError(self.name, requested)
-
         return GitPinnedPackage(
             git=self.git,
+            git_unrendered=self.git_unrendered,
             revision=requested.pop(),
             warn_unpinned=self.warn_unpinned,
             subdirectory=self.subdirectory,