Allow overriding configuration values not set in the YAML (NVIDIA#85)

* This removes the need to repeat default config values in a YAML for the sole purpose allowing users to override the values with the `--override` flag.
* Remove unnecessary `assert` in `src/aiq/data_models/common.py` as this is actually already handled by the code, allowing users to prototype config classes directly in the interpreter.
* Add unittests for the `LayeredConfig` class.

Closes NVIDIA#83

## By Submitting this PR I confirm:
- I am familiar with the [Contributing Guidelines](https://github.com/NVIDIA/AgentIQ/blob/develop/docs/source/advanced/contributing.md).
- We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license.
  - Any contribution which contains commits that are not Signed-Off will not be accepted.
- When the PR is ready for review, new or existing tests cover these changes.
- When the PR is ready for review, the documentation is up to date with these changes.

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Anuradha Karuppiah (https://github.com/AnuradhaKaruppiah)

URL: NVIDIA#85
Signed-off-by: Yuchen Zhang <[email protected]>

Author: dagardner-nv

docs/source/concepts/evaluate.md

@@ -317,9 +317,9 @@ The output of the evaluators are stored in distinct files in the same `output_di
 ## Customizing the output
 You can customize the output of the pipeline by providing custom scripts. One or more Python scripts can be provided in the `eval.general.output_scripts` section of the `config.yml` file.
 
-The custom scripts are executed after the evaluation is complete. They are executed as Python scripts with the kwargs provided in the `eval.general.output.custom_scripts.<script_name>.kwargs` section.
+The custom scripts are executed after the evaluation is complete. They are executed as Python scripts with the `kwargs` provided in the `eval.general.output.custom_scripts.<script_name>.kwargs` section.
 
-The kwargs typically include the file or directory to operate on. To avoid overwriting contents it is recommended to provide a unique output file or directory name for the customization. It is also recommended that changes be limited to the contents of the output directory to avoid unintended side effects.
+The `kwargs` typically include the file or directory to operate on. To avoid overwriting contents it is recommended to provide a unique output file or directory name for the customization. It is also recommended that changes be limited to the contents of the output directory to avoid unintended side effects.
 
 **Example:**
 ```yaml

src/aiq/cli/cli_utils/config_override.py

@@ -27,6 +27,11 @@
 logger = logging.getLogger(__name__)
 
 
+class _Placeholder:
+    """Placeholder class to represent a value that is not set yet."""
+    pass
+
+
 class LayeredConfig:
 
     def __init__(self, base_config: dict[str, Any]):
@@ -46,7 +51,11 @@ def validate_path(self, path: str) -> None:
                 current_path = '.'.join(parts[:i])
                 raise click.BadParameter(f"Cannot navigate through non-dictionary value at '{current_path}'")
             if part not in current:
-                raise click.BadParameter(f"Path '{path}' not found in config. '{part}' is invalid.")
+                if i == len(parts) - 1:
+                    current[part] = _Placeholder()
+                else:
+                    current[part] = {}
+
             current = current[part]
 
     def set_override(self, path: str, value: str) -> None:
@@ -70,9 +79,10 @@ def set_override(self, path: str, value: str) -> None:
             # Convert string value to appropriate type
             try:
                 if isinstance(original_value, bool):
-                    if value.lower() not in ['true', 'false']:
+                    lower_value = value.lower().strip()
+                    if lower_value not in ['true', 'false']:
                         raise ValueError(f"Boolean value must be 'true' or 'false', got '{value}'")
-                    value = value.lower() == 'true'
+                    value = lower_value == 'true'
                 elif isinstance(original_value, (int, float)):
                     value = type(original_value)(value)
                 elif isinstance(original_value, list):
@@ -86,7 +96,12 @@ def set_override(self, path: str, value: str) -> None:
             # Store converted value
             self.overrides[path] = value
             self._effective_config = None
-            logger.info("Successfully set override for %s with value: %s", path, value)
+
+            log_msg = f"Successfully set override for {path} with value: {value}"
+            if not isinstance(original_value, _Placeholder):
+                log_msg += f" with type {type(value)})"
+
+            logger.info(log_msg)
 
         except Exception as e:
             logger.error("Failed to set override for %s: %s", path, str(e))

src/aiq/data_models/common.py

@@ -109,9 +109,7 @@ def __init_subclass__(cls, name: str | None = None):
             module = inspect.getmodule(cls)
 
             assert module is not None, f"Module not found for class {cls} when registering {name}"
-            assert module.__package__ is not None, f"Package not found for class {cls} when registering {name}"
-
-            package_name: str = module.__package__
+            package_name: str | None = module.__package__
 
             # If the package name is not set, then we use the module name. Must have some namespace which will be unique
             if (not package_name):

tests/aiq/cli/cli_utils/test_config_override.py

@@ -0,0 +1,96 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import pytest
+
+from aiq.cli.cli_utils import config_override
+from aiq.data_models.function import FunctionBaseConfig
+
+
+@pytest.fixture(name="base_config")
+def fixture_base_config() -> dict:
+    return {"a": {"b": 1, "c": 2}, "d": 3, "bool_val": True}
+
+
+def test_layered_config_set_override(base_config: dict):
+    layered_config = config_override.LayeredConfig(base_config)
+
+    # Override a value that already exists
+    layered_config.set_override("a.b", '10')
+
+    # Override a value that doesn't exist
+    layered_config.set_override("a.e", '20')
+
+    # override a nested value
+    layered_config.set_override("f.g", '30')
+
+    layered_config.set_override("bool_val", '\tfALse ')
+
+    assert layered_config.get_effective_config() == {
+        "a": {
+            "b": 10, "c": 2, "e": '20'
+        }, "d": 3, "f": {
+            "g": '30'
+        }, "bool_val": False
+    }
+
+
+def test_layered_config_set_override_error(base_config: dict):
+    layered_config = config_override.LayeredConfig(base_config)
+
+    # Attempt to set an override with an invalid path
+    with pytest.raises(click.BadParameter, match="Cannot navigate through non-dictionary value at 'a.b'"):
+        layered_config.set_override("a.b.c", '10')
+
+    # Attempt to set an override a boolean value with an invalid string
+    with pytest.raises(click.BadParameter, match="Boolean value must be 'true' or 'false', got 'not_a_bool'"):
+        layered_config.set_override("bool_val", 'not_a_bool')
+
+    # Attempt to set a value with a type that doesn't match the original
+    with pytest.raises(click.BadParameter, match=r"Type mismatch for 'a\.b'"):
+        layered_config.set_override("a.b", 'not_a_number')
+
+
+def test_layered_config_constructor_error(base_config: dict):
+    # Attempt to set an override with an invalid base config
+    with pytest.raises(ValueError, match="Base config must be a dictionary"):
+        config_override.LayeredConfig("invalid_base_config")
+
+
+def test_config_casting():
+    """
+    Test to verify that pydantic's casting works as expected in situations where LayeredConfig
+    is unable to determine the type of the value being set.
+    """
+
+    class TestConfig(FunctionBaseConfig, name="TestConfig"):
+        a: bool
+        b: int
+        c: float
+
+    layered_config = config_override.LayeredConfig({})
+    for (field, value) in (
+        ("a", "false"),
+        ("b", "45"),
+        ("c", "5.6"),
+    ):
+        layered_config.set_override(field, value)
+
+    effective_config = layered_config.get_effective_config()
+    config = TestConfig(**effective_config)
+    assert config.a is False
+    assert config.b == 45
+    assert config.c == 5.6