- Allow providing a config with init_args but no class_path #113.

- When a class_path is overridden, now only the config values that the new subclass doesn't accept are discarded.
- Single dash '-' incorrectly parsed as [None].

Author: mauvilsa

CHANGELOG.rst

@@ -24,6 +24,8 @@ Added
   <https://github.com/PyTorchLightning/pytorch-lightning/issues/11653>`__.
 - Support init args for unresolved parameters in subclasses `#114
   <https://github.com/omni-us/jsonargparse/issues/114>`__.
+- Allow providing a config with ``init_args`` but no ``class_path`` `#113
+  <https://github.com/omni-us/jsonargparse/issues/113>`__.
 
 Fixed
 ^^^^^
@@ -35,6 +37,7 @@ Fixed
 - In some cases ``print_config`` could output invalid values. Now a lenient
   check is done while dumping.
 - Resolved some issues related to the logger property and reconplogger.
+- Single dash ``'-'`` incorrectly parsed as ``[None]``.
 
 -Changed
 ^^^^^^^
@@ -45,6 +48,8 @@ Fixed
 - ``JSONARGPARSE_DEBUG`` now also sets the reconplogger level to ``DEBUG``.
 - Renamed the test files to follow the more standard ``test_*.py`` pattern.
 - Now ``bool(Namespace())`` evaluates to ``False``.
+- When a ``class_path`` is overridden, now only the config values that the new
+  subclass doesn't accept are discarded.
 
 Deprecated
 ^^^^^^^^^^

jsonargparse/core.py

@@ -17,7 +17,7 @@
 from .loaders_dumpers import check_valid_dump_format, dump_using_format, get_loader_exceptions, loaders, load_value, load_value_context, yaml_load
 from .namespace import is_meta_key, Namespace, split_key, split_key_leaf, strip_meta
 from .signatures import is_pure_dataclass, SignatureArguments
-from .typehints import ActionTypeHint, is_class_object
+from .typehints import ActionTypeHint, is_subclass_spec
 from .typing import is_final_class
 from .actions import (
     ActionParser,
@@ -570,7 +570,7 @@ def parse_string(
                 with_meta=with_meta,
                 skip_check=_skip_check,
                 fail_no_subcommand=_fail_no_subcommand,
-                log_message=(f'Parsed {self.parser_mode} string.'),
+                log_message=f'Parsed {self.parser_mode} string.',
             )
 
         except (TypeError, KeyError) as ex:
@@ -740,7 +740,7 @@ def _dump_delete_default_entries(self, subcfg, subdefaults):
                 val = subcfg[key]
                 default = subdefaults[key]
                 class_object_val = None
-                if is_class_object(val):
+                if is_subclass_spec(val):
                     if val['class_path'] != default.get('class_path'):
                         parser = ActionTypeHint.get_class_parser(val['class_path'])
                         default = {'init_args': parser.get_defaults().as_dict()}
@@ -1218,8 +1218,7 @@ def _apply_actions(self, cfg: Union[Namespace, Dict[str, Any]], parent_key: str
         return cfg[parent_key] if parent_key else cfg
 
 
-    @staticmethod
-    def merge_config(cfg_from: Namespace, cfg_to: Namespace) -> Namespace:
+    def merge_config(self, cfg_from: Namespace, cfg_to: Namespace) -> Namespace:
         """Merges the first configuration into the second configuration.
 
         Args:
@@ -1229,14 +1228,8 @@ def merge_config(cfg_from: Namespace, cfg_to: Namespace) -> Namespace:
         Returns:
             A new object with the merged configuration.
         """
-        del_keys = []
-        for key_class_path in [k for k in cfg_from.keys() if k.endswith('.class_path')]:
-            key_init_args = key_class_path[:-len('class_path')] + 'init_args'
-            if key_class_path in cfg_to and key_init_args in cfg_to and cfg_from[key_class_path] != cfg_to[key_class_path]:
-                del_keys.append(key_init_args)
         cfg = cfg_to.clone()
-        for key in reversed(del_keys):
-            del cfg[key]
+        ActionTypeHint.discard_init_args_on_class_path_change(self, cfg, cfg_from)
         cfg.update(cfg_from)
         return cfg
 

jsonargparse/loaders_dumpers.py

@@ -64,7 +64,10 @@ def remove_implicit_resolver(cls, tag_to_remove):
 
 
 def yaml_load(stream):
-    value = yaml.load(stream, Loader=DefaultLoader)
+    if stream.strip() == '-':
+        value = stream
+    else:
+        value = yaml.load(stream, Loader=DefaultLoader)
     if isinstance(value, dict) and all(v is None for v in value.values()):
         keys = {k for k in regex_curly_comma.split(stream) if k}
         if len(keys) > 0 and keys == set(value.keys()):

jsonargparse/parameter_resolvers.py

@@ -23,7 +23,7 @@ class ParamData:
 
 
 ParamList = List[ParamData]
-parameter_attributes = [s[1:] for s in inspect.Parameter.__slots__]
+parameter_attributes = [s[1:] for s in inspect.Parameter.__slots__]  # type: ignore
 kinds = inspect._ParameterKind
 
 

jsonargparse/typehints.py

@@ -49,6 +49,7 @@
     is_subclass,
     iter_to_set_str,
     lenient_check,
+    lenient_check_context,
     NoneType,
     object_path_serializer,
     ParserError,
@@ -240,6 +241,15 @@ def parse_subclass_arg(arg_string):
             subclass_arg_parser.set(subparser)
 
 
+    @staticmethod
+    def discard_init_args_on_class_path_change(parser, cfg_to, cfg_from):
+        for action in [a for a in parser._actions if ActionTypeHint.is_subclass_typehint(a)]:
+            val_to = cfg_to.get(action.dest)
+            val_from = cfg_from.get(action.dest)
+            if is_subclass_spec(val_to) and is_subclass_spec(val_from):
+                discard_init_args_on_class_path_change(action, val_to, val_from)
+
+
     @staticmethod
     @contextmanager
     def subclass_arg_context(parser):
@@ -585,7 +595,7 @@ def adapt_typehints(val, typehint, serialize=False, instantiate_classes=False, p
     # Callable
     elif typehint_origin in callable_origin_types or typehint in callable_origin_types:
         if serialize:
-            if is_class_object(val):
+            if is_subclass_spec(val):
                 val = adapt_class_type(val, True, False, sub_add_kwargs)
             else:
                 val = object_path_serializer(val)
@@ -600,12 +610,11 @@ def adapt_typehints(val, typehint, serialize=False, instantiate_classes=False, p
                     else:
                         raise ImportError(f'Unexpected import object {val_obj}')
                 if isinstance(val, (dict, Namespace)):
-                    if not is_class_object(val):
+                    if not is_subclass_spec(val):
                         raise ImportError(f'Dict must include a class_path and optionally init_args, but got {val}')
-                    val = Namespace(val)
-                    val_class = import_object(val.class_path)
+                    val_class = import_object(val['class_path'])
                     if not (inspect.isclass(val_class) and callable_instances(val_class)):
-                        raise ImportError(f'"{val.class_path}" is not a callable class.')
+                        raise ImportError(f'{val["class_path"]!r} is not a callable class.')
                     val['class_path'] = get_import_path(val_class)
                     val = adapt_class_type(val, False, instantiate_classes, sub_add_kwargs)
             except (ImportError, AttributeError, ParserError) as ex:
@@ -619,25 +628,25 @@ def adapt_typehints(val, typehint, serialize=False, instantiate_classes=False, p
             return val
         if serialize and isinstance(val, str):
             return val
-        if not (isinstance(val, str) or is_class_object(val)):
-            raise ValueError(f'Type {typehint} expects an str or a Dict/Namespace with a class_path entry but got "{val}"')
+
+        val_input = val
+        val = subclass_spec_as_namespace(val)
+        if not is_subclass_spec(val) and prev_val and 'class_path' in prev_val:
+            if 'init_args' in val:
+                val['class_path'] = prev_val['class_path']
+            else:
+                val = Namespace(class_path=prev_val['class_path'], init_args=val)
+        if not is_subclass_spec(val):
+            raise ValueError(
+                f'Type {typehint} expects: a class path (str); or a dict with a class_path entry; '
+                f'or a dict with init_args (if class path given previously). Got "{val_input}".'
+            )
+
         try:
-            if isinstance(val, str):
-                val = Namespace(class_path=val)
-            elif isinstance(val, dict):
-                val = Namespace(val)
             val_class = import_object(resolve_class_path_by_name(typehint, val['class_path']))
             if not is_subclass(val_class, typehint):
                 raise ValueError(f'"{val["class_path"]}" is not a subclass of {typehint}')
-            val['class_path'] = class_path = get_import_path(val_class)
-            if isinstance(prev_val, Namespace) and 'class_path' in prev_val and 'init_args' not in val:
-                prev_class_path = prev_val['class_path']
-                prev_init_args = prev_val.get('init_args')
-                if prev_class_path != class_path and prev_init_args:
-                    warnings.warn(
-                        f'Changing class_path to {class_path} implies discarding init_args {prev_init_args.as_dict()} '
-                        f'defined for class_path {prev_class_path}.'
-                    )
+            val['class_path'] = get_import_path(val_class)
             val = adapt_class_type(val, serialize, instantiate_classes, sub_add_kwargs, prev_val=prev_val)
         except (ImportError, AttributeError, AssertionError, ParserError) as ex:
             class_path = val if isinstance(val, str) else val['class_path']
@@ -647,14 +656,24 @@ def adapt_typehints(val, typehint, serialize=False, instantiate_classes=False, p
     return val
 
 
-def is_class_object(val):
+def is_subclass_spec(val):
     is_class = isinstance(val, (dict, Namespace)) and 'class_path' in val
     if is_class:
         keys = getattr(val, '__dict__', val).keys()
         is_class = len(set(keys)-{'class_path', 'init_args', 'dict_kwargs', '__path__'}) == 0
     return is_class
 
 
+def subclass_spec_as_namespace(val):
+    if isinstance(val, str):
+        return Namespace(class_path=val)
+    if isinstance(val, dict):
+        val = Namespace(val)
+    if 'init_args' in val and isinstance(val['init_args'], dict):
+        val['init_args'] = Namespace(val['init_args'])
+    return val
+
+
 class NestedArg:
     def __init__(self, key, val):
         self.key = key
@@ -716,8 +735,34 @@ def dump_kwargs_context(kwargs):
     yield
 
 
+def discard_init_args_on_class_path_change(parser_or_action, prev_val, value):
+    if prev_val and 'init_args' in prev_val and prev_val['class_path'] != value.class_path:
+        parser = parser_or_action
+        if isinstance(parser_or_action, ActionTypeHint):
+            sub_add_kwargs = getattr(parser_or_action, 'sub_add_kwargs', {})
+            parser = ActionTypeHint.get_class_parser(value.class_path, sub_add_kwargs)
+        prev_val = subclass_spec_as_namespace(prev_val)
+        del_args = {}
+        for key, val in list(prev_val.init_args.__dict__.items()):
+            action = _find_action(parser, key)
+            if action:
+                with lenient_check_context(lenient=False):
+                    try:
+                        parser._check_value_key(action, val, key, Namespace())
+                    except Exception:
+                        action = None
+            if not action:
+                del_args[key] = prev_val.init_args.pop(key)
+        if del_args:
+            warnings.warn(
+                f'Due to class_path change from {prev_val.class_path!r} to {value.class_path!r}, '
+                f'discarding init_args: {del_args}.'
+            )
+
+
 def adapt_class_type(value, serialize, instantiate_classes, sub_add_kwargs, prev_val=None):
-    val_class = import_object(value['class_path'])
+    value = subclass_spec_as_namespace(value)
+    val_class = import_object(value.class_path)
     parser = ActionTypeHint.get_class_parser(val_class, sub_add_kwargs)
 
     # No need to re-create the linked arg but just "inform" the corresponding parser actions that it exists upstream.
@@ -736,18 +781,18 @@ def adapt_class_type(value, serialize, instantiate_classes, sub_add_kwargs, prev
 
             break
 
-    unresolved = value.pop('dict_kwargs', {})
+    discard_init_args_on_class_path_change(parser, prev_val, value)
+
+    dict_kwargs = value.pop('dict_kwargs', {})
     init_args = value.get('init_args', Namespace())
-    if isinstance(init_args, dict):
-        value['init_args'] = init_args = Namespace(init_args)
 
     if instantiate_classes:
         init_args = parser.instantiate_classes(init_args)
         if not sub_add_kwargs.get('instantiate', True):
             if init_args:
                 value['init_args'] = init_args
             return value
-        return val_class(**{**init_args, **unresolved})
+        return val_class(**{**init_args, **dict_kwargs})
 
     if isinstance(init_args, NestedArg):
         value['init_args'] = parser.parse_args(
@@ -761,29 +806,29 @@ def adapt_class_type(value, serialize, instantiate_classes, sub_add_kwargs, prev
         if init_args:
             value['init_args'] = load_value(parser.dump(init_args, **dump_kwargs.get()))
     else:
-        if isinstance(unresolved, str):
-            unresolved = load_value(unresolved)
-        if isinstance(unresolved, (dict, Namespace)):
-            for key in list(unresolved.keys()):
+        if isinstance(dict_kwargs, str):
+            dict_kwargs = load_value(dict_kwargs)
+        if isinstance(dict_kwargs, (dict, Namespace)):
+            for key in list(dict_kwargs.keys()):
                 if _find_action(parser, key):
-                    init_args[key] = unresolved.pop(key)
-            if isinstance(unresolved, Namespace):
-                unresolved = dict(unresolved)
-        elif unresolved:
-            init_args['dict_kwargs'] = unresolved
-            unresolved = None
+                    init_args[key] = dict_kwargs.pop(key)
+            if isinstance(dict_kwargs, Namespace):
+                dict_kwargs = dict(dict_kwargs)
+        elif dict_kwargs:
+            init_args['dict_kwargs'] = dict_kwargs
+            dict_kwargs = None
         init_args = parser.parse_object(init_args, defaults=sub_defaults.get())
         if init_args:
             value['init_args'] = init_args
-    if unresolved:
-        value['dict_kwargs'] = unresolved
+    if dict_kwargs:
+        value['dict_kwargs'] = dict_kwargs
     return value
 
 
 def not_append_diff(val1, val2):
     if isinstance(val1, list) and isinstance(val2, list):
-        val1 = [x.get('class_path') if is_class_object(x) else x for x in val1]
-        val2 = [x.get('class_path') if is_class_object(x) else x for x in val2]
+        val1 = [x.get('class_path') if is_subclass_spec(x) else x for x in val1]
+        val2 = [x.get('class_path') if is_subclass_spec(x) else x for x in val2]
     return val1 != val2
 
 

jsonargparse/util.py

@@ -252,8 +252,8 @@ def object_path_serializer(value):
 
 
 @contextmanager
-def lenient_check_context(caller=None):
-    t = lenient_check.set(False if caller == 'argcomplete' else True)
+def lenient_check_context(caller=None, lenient=True):
+    t = lenient_check.set(False if caller == 'argcomplete' else lenient)
     try:
         yield
     finally:

jsonargparse_tests/test_core.py

@@ -13,6 +13,7 @@
 from contextlib import redirect_stderr, redirect_stdout
 from collections import OrderedDict
 from random import randint, shuffle
+from typing import Optional
 from jsonargparse import (
     ActionConfigFile,
     ActionJsonnet,
@@ -447,7 +448,6 @@ def test_subcommands(self):
         self.assertRaises(ParserError, lambda: parser.parse_string('{"a": {"ap1": "ap1_cfg", "unk": "unk_cfg"}}'))
 
         with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter("always")
             cfg = parser.parse_string('{"a": {"ap1": "ap1_cfg"}, "b": {"nums": {"val1": 2}}}')
             self.assertEqual(cfg.subcommand, 'a')
             self.assertFalse(hasattr(cfg, 'b'))
@@ -1057,9 +1057,12 @@ def test_strip_unknown(self):
 
 
     def test_merge_config(self):
+        parser = ArgumentParser()
+        for key in [1, 2, 3]:
+            parser.add_argument(f'--op{key}', type=Optional[int])
         cfg_from = Namespace(op1=1, op2=None)
         cfg_to = Namespace(op1=None, op2=2, op3=3)
-        cfg = ArgumentParser.merge_config(cfg_from, cfg_to)
+        cfg = parser.merge_config(cfg_from, cfg_to)
         self.assertEqual(cfg, Namespace(op1=1, op2=None, op3=3))
 
 

jsonargparse_tests/test_loaders_dumpers.py

@@ -5,7 +5,7 @@
 from importlib.util import find_spec
 from typing import List
 from jsonargparse import ActionConfigFile, ArgumentParser, set_dumper, set_loader, ParserError
-from jsonargparse.loaders_dumpers import yaml_dump
+from jsonargparse.loaders_dumpers import load_value, load_value_context, yaml_dump
 from jsonargparse.optionals import dump_preserve_order_support
 
 
@@ -120,5 +120,11 @@ def test_dump_header_invalid(self):
             parser.dump_header = True
 
 
+    def test_load_value_dash(self):
+        with load_value_context('yaml'):
+            self.assertEqual('-', load_value('-'))
+            self.assertEqual(' -  ', load_value(' -  '))
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)