Metaflow Configs

metaflow/__init__.py

@@ -105,6 +105,8 @@ class and related decorators.
 
 from .parameters import Parameter, JSONTypeClass
 
+from .config_parameters import Config, config_expr, eval_config
+
 JSONType = JSONTypeClass()
 
 # data layer

metaflow/cli.py

@@ -1,4 +1,5 @@
 import inspect
+import json
 import os
 import sys
 import traceback
@@ -46,6 +47,7 @@
     resolve_identity,
     write_latest_run_id,
 )
+from .config_parameters import LocalFileInput, config_options
 
 ERASE_TO_EOL = "\033[K"
 HIGHLIGHT = "red"
@@ -429,6 +431,7 @@ def step(
 
     if decospecs:
         decorators._attach_decorators_to_step(func, decospecs)
+        decorators._init(ctx.obj.flow, only_non_static=True)
 
     step_kwargs = ctx.params
     # Remove argument `step_name` from `step_kwargs`.
@@ -522,7 +525,7 @@ def init(obj, run_id=None, task_id=None, tags=None, **kwargs):
         obj.monitor,
         run_id=run_id,
     )
-    obj.flow._set_constants(obj.graph, kwargs)
+    obj.flow._set_constants(obj.graph, kwargs, obj.config_options)
     runtime.persist_constants(task_id=task_id)
 
 
@@ -771,7 +774,7 @@ def run(
     write_latest_run_id(obj, runtime.run_id)
     write_file(run_id_file, runtime.run_id)
 
-    obj.flow._set_constants(obj.graph, kwargs)
+    obj.flow._set_constants(obj.graph, kwargs, obj.config_options)
     runtime.print_workflow_info()
     runtime.persist_constants()
     write_file(
@@ -783,6 +786,7 @@ def run(
             "/".join((obj.flow.name, runtime.run_id)),
         ),
     )
+
     runtime.execute()
 
 
@@ -816,6 +820,7 @@ def before_run(obj, tags, decospecs):
     )
     if all_decospecs:
         decorators._attach_decorators(obj.flow, all_decospecs)
+        decorators._init(obj.flow, only_non_static=True)
         obj.graph = FlowGraph(obj.flow.__class__)
 
     obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint)
@@ -842,17 +847,24 @@ def version(obj):
 
 
 @tracing.cli_entrypoint("cli/start")
+# NOTE: add_decorator_options should be TL because it checks to make sure
+# that no option conflict with the ones below
 @decorators.add_decorator_options
+@config_options
 @click.command(
     cls=click.CommandCollection,
     sources=[cli] + plugins.get_plugin_cli(),
     invoke_without_command=True,
 )
+# Quiet is eager to make sure it is available when processing --config options since
+# we need it to construct a context to pass to any DeployTimeField for the default
+# value.
 @click.option(
     "--quiet/--not-quiet",
     show_default=True,
     default=False,
     help="Suppress unnecessary messages",
+    is_eager=True,
 )
 @click.option(
     "--metadata",
@@ -868,12 +880,14 @@ def version(obj):
     type=click.Choice(["local"] + [m.TYPE for m in ENVIRONMENTS]),
     help="Execution environment type",
 )
+# See comment for --quiet
 @click.option(
     "--datastore",
     default=DEFAULT_DATASTORE,
     show_default=True,
     type=click.Choice([d.TYPE for d in DATASTORES]),
     help="Data backend type",
+    is_eager=True,
 )
 @click.option("--datastore-root", help="Root path for datastore")
 @click.option(
@@ -910,6 +924,15 @@ def version(obj):
     type=click.Choice(MONITOR_SIDECARS),
     help="Monitoring backend type",
 )
+@click.option(
+    "--local-config-file",
+    type=LocalFileInput(exists=True, readable=True, dir_okay=False, resolve_path=True),
+    required=False,
+    default=None,
+    help="A filename containing the dumped configuration values. Internal use only.",
+    hidden=True,
+    is_eager=True,
+)
 @click.pass_context
 def start(
     ctx,
@@ -923,6 +946,8 @@ def start(
     pylint=None,
     event_logger=None,
     monitor=None,
+    local_config_file=None,
+    config_options=None,
     **deco_options
 ):
     global echo
@@ -940,11 +965,17 @@ def start(
     echo(" executing *%s*" % ctx.obj.flow.name, fg="magenta", nl=False)
     echo(" for *%s*" % resolve_identity(), fg="magenta")
 
+    # At this point, we are able to resolve the user-configuration options so we can
+    # process all those decorators that the user added that will modify the flow based
+    # on those configurations. It is important to do this as early as possible since it
+    # actually modifies the flow itself
+    ctx.obj.flow = ctx.obj.flow._process_config_funcs(config_options)
+
     cli_args._set_top_kwargs(ctx.params)
     ctx.obj.echo = echo
     ctx.obj.echo_always = echo_always
     ctx.obj.is_quiet = quiet
-    ctx.obj.graph = FlowGraph(ctx.obj.flow.__class__)
+    ctx.obj.graph = ctx.obj.flow._graph
     ctx.obj.logger = logger
     ctx.obj.check = _check
     ctx.obj.pylint = pylint
@@ -996,6 +1027,10 @@ def start(
         ctx.obj.monitor,
     )
 
+    ctx.obj.config_options = config_options
+
+    decorators._init(ctx.obj.flow)
+
     # It is important to initialize flow decorators early as some of the
     # things they provide may be used by some of the objects initialized after.
     decorators._init_flow_decorators(
@@ -1018,7 +1053,10 @@ def start(
     # initialize current and parameter context for deploy-time parameters
     current._set_env(flow=ctx.obj.flow, is_running=False)
     parameters.set_parameter_context(
-        ctx.obj.flow.name, ctx.obj.echo, ctx.obj.flow_datastore
+        ctx.obj.flow.name,
+        ctx.obj.echo,
+        ctx.obj.flow_datastore,
+        dict(ctx.obj.flow.configs),
     )
 
     if ctx.invoked_subcommand not in ("run", "resume"):
@@ -1029,6 +1067,7 @@ def start(
         )
         if all_decospecs:
             decorators._attach_decorators(ctx.obj.flow, all_decospecs)
+            decorators._init(ctx.obj.flow, only_non_static=True)
             # Regenerate graph if we attached more decorators
             ctx.obj.graph = FlowGraph(ctx.obj.flow.__class__)
 

metaflow/cli_args.py

@@ -12,7 +12,14 @@
 # well as the converting of options in runtime.py. We should make it so that we
 # can properly shlex things and un-shlex when using. Ideally this should all be
 # done in one place.
+#
+# NOTE: There is an important between these two as well:
+#  - this one will include local_config_file whereas the other one WILL NOT.
+#    This is because this is used when constructing the parallel UBF command which
+#    executes locally and therefore needs the local_config_file but the other (remote)
+#    commands do not.
 
+from .config_parameters import ConfigInput
 from .util import to_unicode
 
 
@@ -65,6 +72,14 @@ def _options(mapping):
             # keyword in Python, so we call it 'decospecs' in click args
             if k == "decospecs":
                 k = "with"
+            if k == "config_options":
+                # Special handling here since we gather them all in one option but actually
+                # need to send them one at a time using --config <name> kv.<name>
+                for config_name in v.keys():
+                    yield "--config"
+                    yield to_unicode(config_name)
+                    yield to_unicode(ConfigInput.make_key_name(config_name))
+                continue
             k = k.replace("_", "-")
             v = v if isinstance(v, (list, tuple, set)) else [v]
             for value in v: