Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move UserConfig to dbt_project.yml and rename to ProjectFlags #9289

Merged
merged 24 commits into from
Jan 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
3022f79
Rename UserConfig to ProjectFlags
gshank Dec 14, 2023
3daa178
Reorganize a bunch of ProjectFlags. failing test_cli_flags.py
gshank Dec 15, 2023
5375282
initialize profile_project_flags
gshank Dec 19, 2023
a5819cf
changie
gshank Dec 19, 2023
a583db6
Switch project fixture to flags in dbt_project
gshank Dec 19, 2023
85614e2
Do not stringify profiles_dir and project_dir
gshank Dec 19, 2023
88f246b
Add deprecation events and classes
gshank Dec 19, 2023
520432a
Refine errors
gshank Dec 19, 2023
aa427ba
Call deprecation.show for using 'config' in profiles.yml
gshank Dec 19, 2023
56cd74d
Fix handling multiple configs
gshank Dec 19, 2023
31c3b21
Fix test_config.py
gshank Dec 19, 2023
163ec8d
Fix a few functional tests
gshank Dec 19, 2023
daddfda
Don't throw an error when instantiating flags for lack of
gshank Dec 19, 2023
68dcf97
Fix a couple of more tests
gshank Dec 19, 2023
0d52794
Fix a few more tests
gshank Dec 20, 2023
4152ac7
Various cleanup
gshank Dec 20, 2023
d52bcdc
Add extra line for black
gshank Dec 20, 2023
45ed0da
Add deprecation test
gshank Dec 20, 2023
8430569
Test for flags in multiple places
gshank Dec 20, 2023
51564d0
Add test that all ProjectFlags except send_anonymous_usage_stats has
gshank Dec 20, 2023
88695a1
Merge branch 'main' into 9183-user_config_in_dbt_project
gshank Jan 2, 2024
fb41e5a
Use DBT_PROJECT_FILE_NAME
gshank Jan 3, 2024
bf5e5a2
Remove merge error in types.proto
gshank Jan 3, 2024
7350dba
Merge branch 'main' into 9183-user_config_in_dbt_project
gshank Jan 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20231218-195854.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Move flags from UserConfig in profiles.yml to flags in dbt_project.yml
time: 2023-12-18T19:58:54.075811-05:00
custom:
Author: gshank
Issue: "9183"
8 changes: 0 additions & 8 deletions core/dbt/adapters/contracts/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,17 +184,9 @@ def __post_serialize__(self, dct):
return dct


class UserConfigContract(Protocol):
send_anonymous_usage_stats: bool
use_colors: Optional[bool] = None
partial_parse: Optional[bool] = None
printer_width: Optional[int] = None


class HasCredentials(Protocol):
credentials: Credentials
profile_name: str
user_config: UserConfigContract
target_name: str
threads: int

Expand Down
33 changes: 21 additions & 12 deletions core/dbt/cli/flags.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import sys
from dataclasses import dataclass
from importlib import import_module
from pathlib import Path
from pprint import pformat as pf
from typing import Any, Callable, Dict, List, Optional, Set, Union

Expand All @@ -10,12 +11,12 @@
from dbt.cli.exceptions import DbtUsageException
from dbt.cli.resolvers import default_log_path, default_project_dir
from dbt.cli.types import Command as CliCommand
from dbt.config.project import read_project_flags
from dbt.contracts.project import ProjectFlags
from dbt.common import ui
from dbt.common.events import functions
from dbt.common.exceptions import DbtInternalError
from dbt.common.clients import jinja
from dbt.config.profile import read_user_config
from dbt.contracts.project import UserConfig
from dbt.deprecations import renamed_env_var
from dbt.common.helper_types import WarnErrorOptions

Expand All @@ -26,7 +27,7 @@
FLAGS_DEFAULTS = {
"INDIRECT_SELECTION": "eager",
"TARGET_PATH": None,
# Cli args without user_config or env var option.
# Cli args without project_flags or env var option.
"FULL_REFRESH": False,
"STRICT_MODE": False,
"STORE_FAILURES": False,
Expand Down Expand Up @@ -78,7 +79,7 @@ class Flags:
"""Primary configuration artifact for running dbt"""

def __init__(
self, ctx: Optional[Context] = None, user_config: Optional[UserConfig] = None
self, ctx: Optional[Context] = None, project_flags: Optional[ProjectFlags] = None
) -> None:

# Set the default flags.
Expand Down Expand Up @@ -203,23 +204,29 @@ def _assign_params(
invoked_subcommand_ctx, params_assigned_from_default, deprecated_env_vars
)

if not user_config:
if not project_flags:
project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir()))
profiles_dir = getattr(self, "PROFILES_DIR", None)
user_config = read_user_config(profiles_dir) if profiles_dir else None
if profiles_dir and project_dir:
project_flags = read_project_flags(project_dir, profiles_dir)
else:
project_flags = None

# Add entire invocation command to flags
object.__setattr__(self, "INVOCATION_COMMAND", "dbt " + " ".join(sys.argv[1:]))

# Overwrite default assignments with user config if available.
if user_config:
if project_flags:
param_assigned_from_default_copy = params_assigned_from_default.copy()
for param_assigned_from_default in params_assigned_from_default:
user_config_param_value = getattr(user_config, param_assigned_from_default, None)
if user_config_param_value is not None:
project_flags_param_value = getattr(
project_flags, param_assigned_from_default, None
)
if project_flags_param_value is not None:
object.__setattr__(
self,
param_assigned_from_default.upper(),
convert_config(param_assigned_from_default, user_config_param_value),
convert_config(param_assigned_from_default, project_flags_param_value),
)
param_assigned_from_default_copy.remove(param_assigned_from_default)
params_assigned_from_default = param_assigned_from_default_copy
Expand All @@ -236,9 +243,11 @@ def _assign_params(
# Starting in v1.5, if `log-path` is set in `dbt_project.yml`, it will raise a deprecation warning,
# with the possibility of removing it in a future release.
if getattr(self, "LOG_PATH", None) is None:
project_dir = getattr(self, "PROJECT_DIR", default_project_dir())
project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir()))
version_check = getattr(self, "VERSION_CHECK", True)
object.__setattr__(self, "LOG_PATH", default_log_path(project_dir, version_check))
object.__setattr__(
self, "LOG_PATH", default_log_path(Path(project_dir), version_check)
)

# Support console DO NOT TRACK initiative.
if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"):
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/config/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# all these are just exports, they need "noqa" so flake8 will not complain.
from .profile import Profile, read_user_config # noqa
from .profile import Profile # noqa
from .project import Project, IsFQNResource, PartialProject # noqa
from .runtime import RuntimeConfig # noqa
39 changes: 1 addition & 38 deletions core/dbt/config/profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
from dbt.flags import get_flags
from dbt.common.clients.system import load_file_contents
from dbt.clients.yaml_helper import load_yaml_text
from dbt.contracts.project import ProfileConfig
from dbt.adapters.contracts.connection import Credentials, HasCredentials
from dbt.contracts.project import ProfileConfig, UserConfig
from dbt.events.types import MissingProfileTarget
from dbt.exceptions import (
CompilationError,
Expand All @@ -19,7 +19,6 @@
)
from dbt.common.exceptions import DbtValidationError
from dbt.common.events.functions import fire_event
from dbt.utils import coerce_dict_str

from .renderer import ProfileRenderer

Expand Down Expand Up @@ -51,27 +50,13 @@ def read_profile(profiles_dir: str) -> Dict[str, Any]:
return {}


def read_user_config(directory: str) -> UserConfig:
try:
profile = read_profile(directory)
if profile:
user_config = coerce_dict_str(profile.get("config", {}))
if user_config is not None:
UserConfig.validate(user_config)
return UserConfig.from_dict(user_config)
except (DbtRuntimeError, ValidationError):
pass
return UserConfig()


# The Profile class is included in RuntimeConfig, so any attribute
# additions must also be set where the RuntimeConfig class is created
# `init=False` is a workaround for https://bugs.python.org/issue45081
@dataclass(init=False)
class Profile(HasCredentials):
profile_name: str
target_name: str
user_config: UserConfig
threads: int
credentials: Credentials
profile_env_vars: Dict[str, Any]
Expand All @@ -81,7 +66,6 @@ def __init__(
self,
profile_name: str,
target_name: str,
user_config: UserConfig,
threads: int,
credentials: Credentials,
) -> None:
Expand All @@ -90,7 +74,6 @@ def __init__(
"""
self.profile_name = profile_name
self.target_name = target_name
self.user_config = user_config
self.threads = threads
self.credentials = credentials
self.profile_env_vars = {} # never available on init
Expand All @@ -110,12 +93,10 @@ def to_profile_info(self, serialize_credentials: bool = False) -> Dict[str, Any]
result = {
"profile_name": self.profile_name,
"target_name": self.target_name,
"user_config": self.user_config,
"threads": self.threads,
"credentials": self.credentials,
}
if serialize_credentials:
result["user_config"] = self.user_config.to_dict(omit_none=True)
result["credentials"] = self.credentials.to_dict(omit_none=True)
return result

Expand All @@ -128,7 +109,6 @@ def to_target_dict(self) -> Dict[str, Any]:
"name": self.target_name,
"target_name": self.target_name,
"profile_name": self.profile_name,
"config": self.user_config.to_dict(omit_none=True),
}
)
return target
Expand Down Expand Up @@ -250,7 +230,6 @@ def from_credentials(
threads: int,
profile_name: str,
target_name: str,
user_config: Optional[Dict[str, Any]] = None,
) -> "Profile":
"""Create a profile from an existing set of Credentials and the
remaining information.
Expand All @@ -259,20 +238,13 @@ def from_credentials(
:param threads: The number of threads to use for connections.
:param profile_name: The profile name used for this profile.
:param target_name: The target name used for this profile.
:param user_config: The user-level config block from the
raw profiles, if specified.
:raises DbtProfileError: If the profile is invalid.
:returns: The new Profile object.
"""
if user_config is None:
user_config = {}
UserConfig.validate(user_config)
user_config_obj: UserConfig = UserConfig.from_dict(user_config)

profile = cls(
profile_name=profile_name,
target_name=target_name,
user_config=user_config_obj,
threads=threads,
credentials=credentials,
)
Expand Down Expand Up @@ -320,7 +292,6 @@ def from_raw_profile_info(
raw_profile: Dict[str, Any],
profile_name: str,
renderer: ProfileRenderer,
user_config: Optional[Dict[str, Any]] = None,
target_override: Optional[str] = None,
threads_override: Optional[int] = None,
) -> "Profile":
Expand All @@ -332,8 +303,6 @@ def from_raw_profile_info(
disk as yaml and its values rendered with jinja.
:param profile_name: The profile name used.
:param renderer: The config renderer.
:param user_config: The global config for the user, if it
was present.
:param target_override: The target to use, if provided on
the command line.
:param threads_override: The thread count to use, if
Expand All @@ -342,9 +311,6 @@ def from_raw_profile_info(
target could not be found
:returns: The new Profile object.
"""
# user_config is not rendered.
if user_config is None:
user_config = raw_profile.get("config")
# TODO: should it be, and the values coerced to bool?
target_name, profile_data = cls.render_profile(
raw_profile, profile_name, target_override, renderer
Expand All @@ -365,7 +331,6 @@ def from_raw_profile_info(
profile_name=profile_name,
target_name=target_name,
threads=threads,
user_config=user_config,
)

@classmethod
Expand Down Expand Up @@ -400,13 +365,11 @@ def from_raw_profiles(
if not raw_profile:
msg = f"Profile {profile_name} in profiles.yml is empty"
raise DbtProfileError(INVALID_PROFILE_MESSAGE.format(error_string=msg))
user_config = raw_profiles.get("config")

return cls.from_raw_profile_info(
raw_profile=raw_profile,
profile_name=profile_name,
renderer=renderer,
user_config=user_config,
target_override=target_override,
threads_override=threads_override,
)
Expand Down
Loading
Loading