Engine configuration utility (#1828)

* Create the configuration utility * Gen header. * Fix types test. * Add test. * Fix test types. * Update config to take in file locations. * Ensure the ini config file is included. * Add missing docstring and formatting * Fix ambiguity with smarts and smarts.core.smarts * Update documentation * Update docs/sim/configuration.rst Co-authored-by: adai <[email protected]> * Move sim configuration to next steps. * Assign pybullet frequency to make clear. * Update docs/sim/configuration.rst Co-authored-by: Saul Field <[email protected]> --------- Co-authored-by: adai <[email protected]> Co-authored-by: Saul Field <[email protected]>
huawei-noah · Feb 10, 2023 · 41ec4b7 · 41ec4b7
1 parent 4dae72c
commit 41ec4b7
Show file tree

Hide file tree

Showing 16 changed files with 346 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ Copy and pasting the git commit messages is __NOT__ enough.
 
 ## [Unreleased]
 ### Added
+- Engine configuration utility that uses the following locations to allow configuration of the SMARTS engine. The engine consumes the configuration files from the following locations in the following priority: `./engine.ini`, `~/.smarts/engine.ini`, `$GLOBAL_USER/smarts/engine.ini`, and `${PYTHON_ENV}/lib/${PYTHON_VERSION}/site-packages/smarts/engine.ini`.
 - Added map source uri as `map_source` inside of `hiway-v1` reset info to indicate what the current map is on reset.
 - Added NGSIM documentation.
 ### Changed

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -2,6 +2,7 @@ include smarts/core/glsl/*.vert smarts/core/glsl/*.frag
 include smarts/core/models/*.glb smarts/core/models/*.urdf
 include smarts/core/models/controller_parameters.yaml
 include envision/web/dist/*
+include smarts/*.ini
 recursive-include smarts/benchmark *.yaml *.yml
 recursive-include smarts/ros/src *.launch *.msg *.srv package.xml CMakeLists.txt *.py
 recursive-include smarts/scenarios *.xml *.py

diff --git a/docs/conf.py b/docs/conf.py
@@ -38,6 +38,7 @@
     "sphinx.ext.coverage",  # to generate documentation coverage reports
     "sphinx.ext.extlinks",  # shorten external links
     "sphinx.ext.napoleon",  # support Numpy and Google doc style
+    "sphinx.ext.todo", # support for todo items
     "sphinx.ext.viewcode",  # link to sourcecode from docs
     "sphinx_rtd_theme",  # Read The Docs theme
     "sphinx_click",  # extract documentation from a `click` application
@@ -91,6 +92,8 @@
     "waymo_open_dataset",
 ]
 
+todo_include_todos = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

diff --git a/docs/ecosystem/sumo.rst b/docs/ecosystem/sumo.rst
@@ -5,7 +5,7 @@ SUMO
 
 Learn SUMO through its user `documentation <https://sumo.dlr.de/docs/index.html>`_ . 
 
-SMARTS currently directly installs SUMO version >=1.12.0 via `pip`. 
+SMARTS currently directly installs SUMO version >=1.15.0 via `pip`. 
 
 .. code-block:: bash
 

diff --git a/docs/index.rst b/docs/index.rst
@@ -46,6 +46,7 @@ If you use SMARTS in your research, please cite the `paper <https://arxiv.org/ab
    sim/simulator.rst
    sim/scenario_studio.rst
    sim/bubbles.rst
+   sim/configuration.rst
 
 .. toctree::
    :hidden:
@@ -89,4 +90,5 @@ If you use SMARTS in your research, please cite the `paper <https://arxiv.org/ab
    resources/containers.rst
    resources/diagnostic.rst
    resources/faq.rst
-   resources/contributing.rst
+   resources/contributing.rst
+   resources/todo.rst
diff --git a/docs/resources/contributing.rst b/docs/resources/contributing.rst
@@ -100,6 +100,10 @@ Execute the following to build the docs locally.
     $ python3.8 -m http.server 8000 --bind 127.0.0.1 -d docs/_build/html
     # Open http://127.0.0.1:8000 in your browser
 
+If documentation is incomplete please mark the area with a ``.. todo::`` as described in `sphinx.ext.todo <https://www.sphinx-doc.org/en/master/usage/extensions/todo.html>`_.
+
+Feel free to further contribute to the documentation and look at :ref:`todo` for sections that may yet need to be filled.
+
 Communication
 -------------
 
@@ -141,4 +145,4 @@ Things inevitably become slow, when this happens, Flame Graph is a great tool to
     $ mkdir -p flamegraph_dir
     $ curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl > ./utils/third_party/tools/flamegraph.pl
     $ chmod 777 {$flamegraph_dir}/flamegraph.pl
-    $ make flamegraph scenario=./scenarios/sumo/loop script=./examples/control/chase_via_points.py
+    $ make flamegraph scenario=./scenarios/sumo/loop script=./examples/control/chase_via_points.py
diff --git a/docs/resources/todo.rst b/docs/resources/todo.rst
@@ -0,0 +1,8 @@
+.. _todo:
+
+TODO List
+=========
+
+A list of current documentation TODO.
+
+.. todolist_::
diff --git a/docs/sim/configuration.rst b/docs/sim/configuration.rst
@@ -0,0 +1,27 @@
+.. _configuration:
+
+Configuration
+=============
+
+You can change the behavior of the underlying SMARTS engine.
+
+Configuration of the engine can come from several sources. These locations take precedence as noted:
+
+1. Individual ``SMARTS_`` prefixed environment variables (e.g. ``SMARTS_SENSOR_WORKER_COUNT``)
+2. Local directory engine configuration (./smarts_engine.ini )
+3. Local user engine configuration, ``~/.smarts/engine.ini``, if local directory configuration is not found.
+4. Global engine configuration, ``/etc/smarts/engine.ini``, if local configuration is not found.
+5. Package default configuration, ``$PYTHON_PATH/smarts/engine.ini``, if global configuration is not found.
+
+Note that configuration files resolve all settings at the first found configuration file (they do not layer.)
+
+Options
+-------
+
+All settings demonstrated as environment variables are formatted to ``UPPERCASE`` and prefixed with ``SMARTS_`` (e.g. ``[core] logging`` can be configured with ``SMARTS_CORE_LOGGING``)
+
+These settings are as follows:
+
+.. todo::
+
+    List engine settings
diff --git a/smarts/core/__init__.py b/smarts/core/__init__.py
@@ -25,9 +25,13 @@
 
 import random
 import uuid
+from functools import lru_cache, partial
+from pathlib import Path
 
 import numpy as np
 
+from smarts.core.configuration import Config
+
 _current_seed = None
 
 
@@ -48,3 +52,56 @@ def gen_id():
     """Generates a unique but deterministic id if `smarts.core.seed` has set the core seed."""
     id_ = uuid.UUID(int=random.getrandbits(128))
     return str(id_)[:8]
+
+
+@lru_cache(maxsize=1)
+def config(default: str = "./smarts_engine.ini") -> Config:
+    """Get the SMARTS environment config for the smarts engine.
+
+    .. note::
+
+        This searches the following locations and loads the first one it finds:
+        ./smarts_engine.ini
+        ~/.smarts/engine.ini
+        /etc/smarts/engine.ini
+        $PYTHON_PATH/smarts/engine.ini
+
+    Args:
+        default (str, optional): The default configurable location. Defaults to "./smarts_engine.ini".
+
+    Returns:
+        Config: A configuration utility that allows resolving environment and `engine.ini` configuration.
+    """
+    from smarts.core.utils.file import smarts_global_user_dir, smarts_local_user_dir
+
+    def get_file(config_file: Path):
+        try:
+            if not config_file.is_file():
+                return ""
+        except PermissionError:
+            return ""
+
+        return str(config_file)
+
+    conf = partial(Config, environment_prefix="SMARTS")
+
+    file = get_file(Path(default).absolute())
+    if file:
+        return conf(file)
+
+    try:
+        local_dir = smarts_local_user_dir()
+    except PermissionError:
+        file = ""
+    else:
+        file = get_file(Path(local_dir) / "engine.ini")
+    if file:
+        return conf(file)
+
+    global_dir = smarts_global_user_dir()
+    file = get_file(Path(global_dir) / "engine.ini")
+    if file:
+        return conf(file)
+
+    default_path = Path(__file__).parents[1].resolve() / "engine.ini"
+    return conf(get_file(default_path))
diff --git a/smarts/core/configuration.py b/smarts/core/configuration.py
@@ -0,0 +1,113 @@
+# MIT License
+#
+# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+import configparser
+import functools
+import os
+from pathlib import Path
+from typing import Any, Callable, Optional, Union
+
+_UNSET = object()
+
+
+class Config:
+    """A configuration utility that handles configuration from file and environment variable.
+
+    Args:
+        config_file (Union[str, Path]): The path to the configuration file.
+        environment_prefix (str, optional): The prefix given to the environment variables. Defaults to "".
+
+    Raises:
+        FileNotFoundError: If the configuration file cannot be found at the given file location.
+    """
+
+    def __init__(
+        self, config_file: Union[str, Path], environment_prefix: str = ""
+    ) -> None:
+        self._config = configparser.ConfigParser(
+            interpolation=configparser.ExtendedInterpolation()
+        )
+        self._environment_prefix = environment_prefix.upper()
+        self._environment_variable_format_string = self._environment_prefix + "_{}_{}"
+
+        if isinstance(config_file, str):
+            config_file = Path(config_file)
+        config_file = config_file.resolve()
+        if not config_file.is_file():
+            raise FileNotFoundError(f"Configuration file not found at {config_file}")
+
+        self._config.read(str(config_file.absolute()))
+
+    @property
+    def environment_prefix(self):
+        """The prefix that environment variables configuration is provided with."""
+        return self._environment_prefix
+
+    @functools.lru_cache(maxsize=100)
+    def get_setting(
+        self,
+        section: str,
+        option: str,
+        default: Any = _UNSET,
+        cast: Callable[[str], Any] = str,
+    ) -> Optional[Any]:
+        """Finds the given configuration checking the following in order: environment variable,
+        configuration file, and default.
+
+        Args:
+            section (str): The grouping that the configuration option is under.
+            option (str): The specific configuration option.
+            default (Any, optional): The default if the requested configuration option is not found. Defaults to _UNSET.
+            cast (Callable, optional): A function that takes a string and returns the desired type. Defaults to str.
+
+
+        Returns:
+            Optional[str]: The value of the configuration.
+
+        Raises:
+            KeyError: If the configuration option is not found in the configuration file and no default is provided.
+            configparser.NoSectionError: If the section in the configuration file is not found and no default is provided.
+        """
+        env_variable = self._environment_variable_format_string.format(
+            section.upper(), option.upper()
+        )
+        setting = os.getenv(env_variable)
+        if setting is not None:
+            return cast(setting)
+        try:
+            value = self._config[section][option]
+        except (configparser.NoSectionError, KeyError):
+            if default is _UNSET:
+                raise
+            return default
+        return cast(value)
+
+    def __call__(
+        self,
+        section: str,
+        option: str,
+        default: Any = _UNSET,
+        cast: Callable[[str], Any] = str,
+    ) -> Optional[Any]:
+        return self.get_setting(section, option, default, cast)
+
+    def __repr__(self) -> str:
+        return f"Config(config_file={ {k: dict(v.items()) for k, v in self._config.items(raw=True)} }, environment_prefix={self._environment_prefix})"
diff --git a/smarts/core/smarts.py b/smarts/core/smarts.py
@@ -32,7 +32,7 @@
 from smarts.core.plan import Plan
 from smarts.core.utils.logging import timeit
 
-from . import models
+from . import config, models
 from .actor import ActorRole, ActorState
 from .agent_interface import AgentInterface
 from .agent_manager import AgentManager
@@ -728,7 +728,9 @@ def _setup_bullet_client(self, client: bc.BulletClient):
         client.configureDebugVisualizer(
             pybullet.COV_ENABLE_GUI, 0  # pylint: disable=no-member
         )
-
+        max_pybullet_freq = config()(
+            "physics", "max_pybullet_freq", default=MAX_PYBULLET_FREQ, cast=int
+        )
         # PyBullet defaults the timestep to 240Hz. Several parameters are tuned with
         # this value in mind. For example the number of solver iterations and the error
         # reduction parameters (erp) for contact, friction and non-contact joints.
@@ -741,11 +743,11 @@ def _setup_bullet_client(self, client: bc.BulletClient):
         self._pybullet_period = (
             self._fixed_timestep_sec
             if self._fixed_timestep_sec
-            else 1 / MAX_PYBULLET_FREQ
+            else 1 / max_pybullet_freq
         )
         client.setPhysicsEngineParameter(
             fixedTimeStep=self._pybullet_period,
-            numSubSteps=int(self._pybullet_period * MAX_PYBULLET_FREQ),
+            numSubSteps=int(self._pybullet_period * max_pybullet_freq),
             numSolverIterations=10,
             solverResidualThreshold=0.001,
             # warmStartingFactor=0.99
@@ -857,7 +859,7 @@ def __del__(self):
                     " go away.",
                     e,
                 )
-            except (AttributeError, KeyboardInterrupt):
+            except (TypeError, KeyboardInterrupt):
                 return
             raise exception
 
@@ -1280,8 +1282,10 @@ def fixed_timestep_sec(self) -> float:
     @fixed_timestep_sec.setter
     def fixed_timestep_sec(self, fixed_timestep_sec: float):
         if not fixed_timestep_sec:
-            # This is the fastest we could possibly run given constraints from pybullet
-            self._rounder = rounder_for_dt(round(1 / MAX_PYBULLET_FREQ, 6))
+            max_pybullet_freq = config()(
+                "physics", "max_pybullet_freq", default=MAX_PYBULLET_FREQ, cast=int
+            )
+            self._rounder = rounder_for_dt(round(1 / max_pybullet_freq, 6))
         else:
             self._rounder = rounder_for_dt(fixed_timestep_sec)
         self._fixed_timestep_sec = fixed_timestep_sec