From 732c5cd9f405586c1c227b9b155b00800ec2d67f Mon Sep 17 00:00:00 2001 From: Tucker Date: Thu, 2 Feb 2023 12:41:48 -0500 Subject: [PATCH 01/14] Create the configuration utility --- smarts/core/configuration.py | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 smarts/core/configuration.py diff --git a/smarts/core/configuration.py b/smarts/core/configuration.py new file mode 100644 index 0000000000..99b1c16431 --- /dev/null +++ b/smarts/core/configuration.py @@ -0,0 +1,75 @@ +import configparser +import functools +import os +from pathlib import Path +from typing import Any, Callable, Optional, Union + +_UNSET = object() + + +class Config: + """A configuration utility that handles configuration from file and environment variable. + + Args: + config_file (Union[str, Path]): The path to the configuration file. + environment_prefix (str, optional): The prefix given to the environment variables. Defaults to "". + + Raises: + + """ + + def __init__( + self, config_file: Union[str, Path], environment_prefix: str = "" + ) -> None: + config_file = str(config_file) + if not Path(config_file).is_file(): + raise FileNotFoundError(f"Configuration file not found at {config_file}") + + self._config = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation + ) + self._config.read(config_file) + self._environment_prefix = environment_prefix.upper() + self._format_string = self._environment_prefix + "_{}_{}" + + @property + def environment_prefix(self): + """The prefix that environment variables configuration is provided with.""" + return self._environment_prefix + + @functools.lru_cache(maxsize=100) + def get_setting( + self, + section: str, + option: str, + default: Any = _UNSET, + cast: Callable[[str], Any] = str, + ) -> Optional[Any]: + """Finds the given configuration checking the following in order: environment variable, + configuration file, and default. + + Args: + section (str): The grouping that the configuration option is under. + option (str): The specific configuration option. + default (Any, optional): The default if the requested configuration option is not found. Defaults to _UNSET. + cast (Callable, optional): A function that takes a string and returns the desired type. Defaults to str. + + + Returns: + Optional[str]: The value of the configuration. + + Raises: + KeyError: If the configuration option is not found in the configuration file and no default is provided. + configparser.NoSectionError: If the section in the configuration file is not found and no default is provided. + """ + env_variable = self._format_string.format(section.upper(), option.upper()) + setting = os.getenv(env_variable) + if setting is not None: + return cast(setting) + try: + value = self._config[section][option] + except (configparser.NoSectionError, KeyError): + if default is _UNSET: + raise + return default + return cast(value) From 24cd9403c730c32fecc4ae57b71f88014f43aff0 Mon Sep 17 00:00:00 2001 From: Tucker Date: Thu, 2 Feb 2023 12:46:15 -0500 Subject: [PATCH 02/14] Gen header. --- smarts/core/configuration.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/smarts/core/configuration.py b/smarts/core/configuration.py index 99b1c16431..e47dfe8721 100644 --- a/smarts/core/configuration.py +++ b/smarts/core/configuration.py @@ -1,3 +1,24 @@ +# MIT License +# +# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. import configparser import functools import os From c8e9e290a50047e33ef3ceee96c0dd41d964ee6a Mon Sep 17 00:00:00 2001 From: Tucker Date: Thu, 2 Feb 2023 15:24:48 -0500 Subject: [PATCH 03/14] Fix types test. --- smarts/core/configuration.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/smarts/core/configuration.py b/smarts/core/configuration.py index e47dfe8721..bab3973704 100644 --- a/smarts/core/configuration.py +++ b/smarts/core/configuration.py @@ -1,17 +1,17 @@ # MIT License -# +# # Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. -# +# # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: -# +# # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. -# +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE @@ -42,14 +42,15 @@ class Config: def __init__( self, config_file: Union[str, Path], environment_prefix: str = "" ) -> None: - config_file = str(config_file) - if not Path(config_file).is_file(): + if isinstance(config_file, str): + config_file = Path(config_file) + if not config_file.is_file(): raise FileNotFoundError(f"Configuration file not found at {config_file}") self._config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation + interpolation=configparser.ExtendedInterpolation() ) - self._config.read(config_file) + self._config.read(str(config_file.absolute())) self._environment_prefix = environment_prefix.upper() self._format_string = self._environment_prefix + "_{}_{}" From 0589ab322bab5393ee1739fc722eb11d225d7021 Mon Sep 17 00:00:00 2001 From: Tucker Date: Thu, 2 Feb 2023 16:15:19 -0500 Subject: [PATCH 04/14] Add test. --- smarts/core/configuration.py | 2 +- smarts/core/tests/test_configuration.py | 69 +++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 smarts/core/tests/test_configuration.py diff --git a/smarts/core/configuration.py b/smarts/core/configuration.py index bab3973704..e905d93dd8 100644 --- a/smarts/core/configuration.py +++ b/smarts/core/configuration.py @@ -36,7 +36,7 @@ class Config: environment_prefix (str, optional): The prefix given to the environment variables. Defaults to "". Raises: - + FileNotFoundError: If the configuration file cannot be found at the given file location. """ def __init__( diff --git a/smarts/core/tests/test_configuration.py b/smarts/core/tests/test_configuration.py new file mode 100644 index 0000000000..64360c3a84 --- /dev/null +++ b/smarts/core/tests/test_configuration.py @@ -0,0 +1,69 @@ +# MIT License +# +# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +import configparser +import functools +import os +import tempfile +from pathlib import Path + +import pytest + +from smarts.core.configuration import Config + + +@pytest.fixture +def config_path() -> str: + config = configparser.ConfigParser() + config["section_1"] = {"string_option": "value", "option_2": "value_2"} + config["section_2"] = {"bool_option": "True", "float_option": "3.14"} + with tempfile.TemporaryDirectory() as tmpdir: + file = Path(tmpdir) / "config.ini" + with file.open("w") as file_pointer: + config.write(fp=file_pointer) + yield str(file) + + +def test_get_setting_with_file(config_path): + config = Config(config_path) + assert config.get_setting("section_1", "string_option") == "value" + partition_string = functools.partial( + lambda source, sep: str.partition(source, sep), sep="_" + ) + assert config.get_setting("section_1", "option_2", cast=partition_string) == ( + "value", + "_", + "2", + ) + assert config.get_setting("section_2", "bool_option", cast=bool) is True + assert config.get_setting("section_2", "float_option", cast=float) == 3.14 + with pytest.raises(KeyError): + config.get_setting("nonexistent", "option") + + +def test_get_setting_with_environment_variables(config_path): + config = Config(config_path, "smarts") + assert config.get_setting("nonexistent", "option", default=None) is None + + os.environ["SMARTS_NONEXISTENT_OPTION"] = "now_exists" + config = Config(config_path, "smarts") + assert config.get_setting("nonexistent", "option", default=None) == "now_exists" + del os.environ["SMARTS_NONEXISTENT_OPTION"] From 1394667210293f48f36e29d85b7328dd57dbefbe Mon Sep 17 00:00:00 2001 From: Tucker Date: Thu, 2 Feb 2023 16:26:04 -0500 Subject: [PATCH 05/14] Fix test types. --- smarts/core/tests/test_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smarts/core/tests/test_configuration.py b/smarts/core/tests/test_configuration.py index 64360c3a84..678cd8fc43 100644 --- a/smarts/core/tests/test_configuration.py +++ b/smarts/core/tests/test_configuration.py @@ -31,7 +31,7 @@ @pytest.fixture -def config_path() -> str: +def config_path(): config = configparser.ConfigParser() config["section_1"] = {"string_option": "value", "option_2": "value_2"} config["section_2"] = {"bool_option": "True", "float_option": "3.14"} From 45715ba9bdd6b56c1eea2827120ff252c7b5ef9e Mon Sep 17 00:00:00 2001 From: Tucker Date: Wed, 8 Feb 2023 17:52:43 -0500 Subject: [PATCH 06/14] Update config to take in file locations. --- CHANGELOG.md | 1 + .../entrypoints/benchmark_runner_v0.py | 1 - smarts/core/__init__.py | 41 +++++++++++++++++++ smarts/core/configuration.py | 12 ++++++ smarts/core/smarts.py | 14 +++---- smarts/core/tests/test_configuration.py | 14 +++++++ smarts/core/utils/file.py | 20 +++++++-- smarts/engine.ini | 8 ++++ smarts/env/utils/record.py | 4 +- 9 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 smarts/engine.ini diff --git a/CHANGELOG.md b/CHANGELOG.md index 6307331305..baaf01f979 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Copy and pasting the git commit messages is __NOT__ enough. ## [Unreleased] ### Added +- Engine configuration utility that uses the following locations to allow configuration of the SMARTS engine. The engine consumes the configuration files from the following locations in the following priority: `./engine.ini`, `~/.smarts/engine.ini`, `$GLOBAL_USER/smarts/engine.ini`, and `${PYTHON_ENV}/lib/${PYTHON_VERSION}/site-packages/smarts/engine.ini`. - Added map source uri as `map_source` inside of `hiway-v1` reset info to indicate what the current map is on reset. ### Changed - Made changes in the docs to reflect `master` branch as the main development branch. diff --git a/smarts/benchmark/entrypoints/benchmark_runner_v0.py b/smarts/benchmark/entrypoints/benchmark_runner_v0.py index 4aaa8a7b0f..e9b5ad10cf 100644 --- a/smarts/benchmark/entrypoints/benchmark_runner_v0.py +++ b/smarts/benchmark/entrypoints/benchmark_runner_v0.py @@ -28,7 +28,6 @@ import ray from smarts.benchmark.driving_smarts import load_config -from smarts.benchmark.driving_smarts.v0 import DEFAULT_CONFIG from smarts.core.utils.logging import suppress_output from smarts.env.gymnasium.wrappers.metrics import Metrics, Score from smarts.zoo import registry as agent_registry diff --git a/smarts/core/__init__.py b/smarts/core/__init__.py index c9f6a7a615..4ae62c92cd 100644 --- a/smarts/core/__init__.py +++ b/smarts/core/__init__.py @@ -25,9 +25,14 @@ import random import uuid +from functools import lru_cache, partial +from pathlib import Path import numpy as np +import smarts +from smarts.core.configuration import Config + _current_seed = None @@ -48,3 +53,39 @@ def gen_id(): """Generates a unique but deterministic id if `smarts.core.seed` has set the core seed.""" id_ = uuid.UUID(int=random.getrandbits(128)) return str(id_)[:8] + + +@lru_cache(maxsize=1) +def config(default=Path(".") / "smarts_engine.ini") -> Config: + from smarts.core.utils.file import smarts_global_user_dir, smarts_local_user_dir + + def get_file(config_file): + try: + if not config_file.is_file(): + return "" + except PermissionError: + return "" + + return str(config_file) + + conf = partial(Config, environment_prefix="SMARTS") + + file = get_file(default) + if file: + return conf(file) + + try: + local_dir = smarts_local_user_dir() + except PermissionError: + file = "" + else: + file = get_file(Path(local_dir) / "engine.ini") + if file: + return conf(file) + + global_dir = smarts_global_user_dir() + file = get_file(Path(global_dir) / "engine.ini") + if file: + return conf(file) + + return conf(get_file(Path(smarts.__file__).parent.absolute() / "engine.ini")) diff --git a/smarts/core/configuration.py b/smarts/core/configuration.py index e905d93dd8..b4818819c2 100644 --- a/smarts/core/configuration.py +++ b/smarts/core/configuration.py @@ -95,3 +95,15 @@ def get_setting( raise return default return cast(value) + + def __call__( + self, + section: str, + option: str, + default: Any = _UNSET, + cast: Callable[[str], Any] = str, + ) -> Optional[Any]: + return self.get_setting(section, option, default, cast) + + def __repr__(self) -> str: + return f"Config(config_file={ {k: dict(v.items()) for k, v in self._config.items(raw=True)} }, environment_prefix={self._environment_prefix})" diff --git a/smarts/core/smarts.py b/smarts/core/smarts.py index 3faddfb189..cdf1ac36da 100644 --- a/smarts/core/smarts.py +++ b/smarts/core/smarts.py @@ -32,7 +32,7 @@ from smarts.core.plan import Plan from smarts.core.utils.logging import timeit -from . import models +from . import models, config from .actor import ActorRole, ActorState from .agent_interface import AgentInterface from .agent_manager import AgentManager @@ -71,8 +71,6 @@ level=logging.ERROR, ) -MAX_PYBULLET_FREQ = 240 - class SMARTSNotSetupError(Exception): """Represents a case where SMARTS cannot operate because it is not set up yet.""" @@ -728,7 +726,7 @@ def _setup_bullet_client(self, client: bc.BulletClient): client.configureDebugVisualizer( pybullet.COV_ENABLE_GUI, 0 # pylint: disable=no-member ) - + MAX_PYBULLET_FREQ = config()("physics", "max_pybullet_freq", cast=int) # PyBullet defaults the timestep to 240Hz. Several parameters are tuned with # this value in mind. For example the number of solver iterations and the error # reduction parameters (erp) for contact, friction and non-contact joints. @@ -857,10 +855,10 @@ def __del__(self): " go away.", e, ) - except (AttributeError, KeyboardInterrupt): + except (TypeError, KeyboardInterrupt): return raise exception - + def _teardown_vehicles(self, vehicle_ids): self._vehicle_index.teardown_vehicles_by_vehicle_ids(vehicle_ids) self._clear_collisions(vehicle_ids) @@ -1281,7 +1279,9 @@ def fixed_timestep_sec(self) -> float: def fixed_timestep_sec(self, fixed_timestep_sec: float): if not fixed_timestep_sec: # This is the fastest we could possibly run given constraints from pybullet - self._rounder = rounder_for_dt(round(1 / MAX_PYBULLET_FREQ, 6)) + self._rounder = rounder_for_dt( + round(1 / config().get_setting("physics", "max_pybullet_freq", cast=int), 6) + ) else: self._rounder = rounder_for_dt(fixed_timestep_sec) self._fixed_timestep_sec = fixed_timestep_sec diff --git a/smarts/core/tests/test_configuration.py b/smarts/core/tests/test_configuration.py index 678cd8fc43..b0c0806c1e 100644 --- a/smarts/core/tests/test_configuration.py +++ b/smarts/core/tests/test_configuration.py @@ -67,3 +67,17 @@ def test_get_setting_with_environment_variables(config_path): config = Config(config_path, "smarts") assert config.get_setting("nonexistent", "option", default=None) == "now_exists" del os.environ["SMARTS_NONEXISTENT_OPTION"] + + +def test_get_missing_section_and_missing_option(): + from smarts.core import config as core_conf + + core_conf.cache_clear() + + config: Config = core_conf() + + with pytest.raises(KeyError): + config.get_setting("core", "not_a_setting") + + with pytest.raises(KeyError): + config.get_setting("not_a_section", "bar") \ No newline at end of file diff --git a/smarts/core/utils/file.py b/smarts/core/utils/file.py index 21a411fbd0..dd39ad69e3 100644 --- a/smarts/core/utils/file.py +++ b/smarts/core/utils/file.py @@ -142,17 +142,31 @@ def pickle_hash_int(obj) -> int: return c_int64(val).value -def smarts_log_dir() -> str: - """Retrieves the smarts logging directory.""" +def smarts_local_user_dir() -> str: + """Retrieves the smarts logging directory. + + Returns: + str: The smarts local user directory path. + """ ## Following should work for linux and macos smarts_dir = os.path.join(os.path.expanduser("~"), ".smarts") os.makedirs(smarts_dir, exist_ok=True) return smarts_dir +def smarts_global_user_dir() -> str: + """Retrieves the smarts global user directory. + + Returns: + str: The smarts global user directory path. + """ + smarts_dir = os.path.join("/etc", "smarts") + return smarts_dir + + def make_dir_in_smarts_log_dir(dir): """Return a new directory location in the smarts logging directory.""" - return os.path.join(smarts_log_dir(), dir) + return os.path.join(smarts_local_user_dir(), dir) @contextmanager diff --git a/smarts/engine.ini b/smarts/engine.ini new file mode 100644 index 0000000000..eec1cf05c5 --- /dev/null +++ b/smarts/engine.ini @@ -0,0 +1,8 @@ +[benchmark] +[core] +[controllers] +[physics] +max_pybullet_freq = 240 +[providers] +[rendering] +[resources] \ No newline at end of file diff --git a/smarts/env/utils/record.py b/smarts/env/utils/record.py index 7fd76bc9d3..4add9dc1a3 100644 --- a/smarts/env/utils/record.py +++ b/smarts/env/utils/record.py @@ -24,13 +24,13 @@ import numpy as np from smarts.core.sensors import Observation -from smarts.core.utils.file import smarts_log_dir +from smarts.core.utils.file import smarts_local_user_dir from smarts.env.wrappers.utils.rendering import vis_sim_obs Action = Any Operation = Any -default_log_dir = smarts_log_dir() +default_log_dir = smarts_local_user_dir() class AgentCameraRGBRender: From 92963357e223e9f10bd1bd278dc720d0f25c890a Mon Sep 17 00:00:00 2001 From: Tucker Date: Wed, 8 Feb 2023 18:21:09 -0500 Subject: [PATCH 07/14] Ensure the ini config file is included. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index feda46df28..09fa472fa4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include smarts/core/glsl/*.vert smarts/core/glsl/*.frag include smarts/core/models/*.glb smarts/core/models/*.urdf include smarts/core/models/controller_parameters.yaml include envision/web/dist/* +include smarts/*.ini recursive-include smarts/benchmark *.yaml *.yml recursive-include smarts/ros/src *.launch *.msg *.srv package.xml CMakeLists.txt *.py recursive-include smarts/scenarios *.xml *.py From 693683decda6cb19cccb3614a47c8ca680b15bc1 Mon Sep 17 00:00:00 2001 From: Tucker Date: Wed, 8 Feb 2023 18:24:29 -0500 Subject: [PATCH 08/14] Add missing docstring and formatting --- smarts/core/__init__.py | 20 ++++++++++++++++++-- smarts/core/smarts.py | 23 ++++++++++++++++++----- smarts/core/tests/test_configuration.py | 4 ++-- smarts/env/wrappers/format_action.py | 10 ++++++---- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/smarts/core/__init__.py b/smarts/core/__init__.py index 4ae62c92cd..548e7b8ca3 100644 --- a/smarts/core/__init__.py +++ b/smarts/core/__init__.py @@ -56,7 +56,23 @@ def gen_id(): @lru_cache(maxsize=1) -def config(default=Path(".") / "smarts_engine.ini") -> Config: +def config(default=str("./smarts_engine.ini")) -> Config: + """Get the SMARTS environment config for the smarts engine. + + .. note:: + + This searches the following locations and loads the first one it finds: + ./smarts_engine.ini + ~/.smarts/engine.ini + /etc/smarts/engine.ini + $PYTHON_PATH/smarts/engine.ini + + Args: + default (str, optional): The default configurable location. Defaults to "./smarts_engine.ini". + + Returns: + Config: A configuration utility that allows resolving environment and `engine.ini` configuration. + """ from smarts.core.utils.file import smarts_global_user_dir, smarts_local_user_dir def get_file(config_file): @@ -70,7 +86,7 @@ def get_file(config_file): conf = partial(Config, environment_prefix="SMARTS") - file = get_file(default) + file = get_file(Path(default)) if file: return conf(file) diff --git a/smarts/core/smarts.py b/smarts/core/smarts.py index cdf1ac36da..2e0b546784 100644 --- a/smarts/core/smarts.py +++ b/smarts/core/smarts.py @@ -32,7 +32,7 @@ from smarts.core.plan import Plan from smarts.core.utils.logging import timeit -from . import models, config +from . import config, models from .actor import ActorRole, ActorState from .agent_interface import AgentInterface from .agent_manager import AgentManager @@ -71,6 +71,8 @@ level=logging.ERROR, ) +MAX_PYBULLET_FREQ = 240 + class SMARTSNotSetupError(Exception): """Represents a case where SMARTS cannot operate because it is not set up yet.""" @@ -726,7 +728,9 @@ def _setup_bullet_client(self, client: bc.BulletClient): client.configureDebugVisualizer( pybullet.COV_ENABLE_GUI, 0 # pylint: disable=no-member ) - MAX_PYBULLET_FREQ = config()("physics", "max_pybullet_freq", cast=int) + max_pybullet_freq = config()( + "physics", "max_pybullet_freq", default=MAX_PYBULLET_FREQ, cast=int + ) # PyBullet defaults the timestep to 240Hz. Several parameters are tuned with # this value in mind. For example the number of solver iterations and the error # reduction parameters (erp) for contact, friction and non-contact joints. @@ -739,11 +743,11 @@ def _setup_bullet_client(self, client: bc.BulletClient): self._pybullet_period = ( self._fixed_timestep_sec if self._fixed_timestep_sec - else 1 / MAX_PYBULLET_FREQ + else 1 / max_pybullet_freq ) client.setPhysicsEngineParameter( fixedTimeStep=self._pybullet_period, - numSubSteps=int(self._pybullet_period * MAX_PYBULLET_FREQ), + numSubSteps=int(self._pybullet_period * max_pybullet_freq), numSolverIterations=10, solverResidualThreshold=0.001, # warmStartingFactor=0.99 @@ -1280,7 +1284,16 @@ def fixed_timestep_sec(self, fixed_timestep_sec: float): if not fixed_timestep_sec: # This is the fastest we could possibly run given constraints from pybullet self._rounder = rounder_for_dt( - round(1 / config().get_setting("physics", "max_pybullet_freq", cast=int), 6) + round( + 1 + / config()( + "physics", + "max_pybullet_freq", + default=MAX_PYBULLET_FREQ, + cast=int, + ), + 6, + ) ) else: self._rounder = rounder_for_dt(fixed_timestep_sec) diff --git a/smarts/core/tests/test_configuration.py b/smarts/core/tests/test_configuration.py index b0c0806c1e..776550ec86 100644 --- a/smarts/core/tests/test_configuration.py +++ b/smarts/core/tests/test_configuration.py @@ -78,6 +78,6 @@ def test_get_missing_section_and_missing_option(): with pytest.raises(KeyError): config.get_setting("core", "not_a_setting") - + with pytest.raises(KeyError): - config.get_setting("not_a_section", "bar") \ No newline at end of file + config.get_setting("not_a_section", "bar") diff --git a/smarts/env/wrappers/format_action.py b/smarts/env/wrappers/format_action.py index 756d0c21eb..a9caac086a 100644 --- a/smarts/env/wrappers/format_action.py +++ b/smarts/env/wrappers/format_action.py @@ -31,8 +31,8 @@ class FormatAction(gym.ActionWrapper): Note: - (a) Only ``ActionSpaceType.Continuous``, ``ActionSpaceType.Lane``, - ``ActionSpaceType.ActuatorDynamic``, and `ActionSpaceType.TargetPose`` + (a) Only ``ActionSpaceType.Continuous``, ``ActionSpaceType.Lane``, + ``ActionSpaceType.ActuatorDynamic``, and `ActionSpaceType.TargetPose`` are supported by this wrapper now. (b) All agents should have the same action space. @@ -98,7 +98,9 @@ def wrapper(action: Dict[str, int]) -> Dict[str, str]: return wrapper, space -def _actuator_dynamic() -> Tuple[Callable[[Dict[str, np.ndarray]], Dict[str, np.ndarray]], gym.Space]: +def _actuator_dynamic() -> Tuple[ + Callable[[Dict[str, np.ndarray]], Dict[str, np.ndarray]], gym.Space +]: space = gym.spaces.Box( low=np.array([0.0, 0.0, -1.0]), high=np.array([1.0, 1.0, 1.0]), dtype=np.float32 ) @@ -121,4 +123,4 @@ def _target_pose() -> Tuple[ def wrapper(action: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: return {k: v.astype(np.float32) for k, v in action.items()} - return wrapper, space \ No newline at end of file + return wrapper, space From d43d4beb48a5bc3694833ffa59c7310259e00108 Mon Sep 17 00:00:00 2001 From: Montgomery Alban Date: Thu, 9 Feb 2023 16:01:42 +0000 Subject: [PATCH 09/14] Fix ambiguity with smarts and smarts.core.smarts --- smarts/core/__init__.py | 10 +++++----- smarts/core/configuration.py | 16 ++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/smarts/core/__init__.py b/smarts/core/__init__.py index 548e7b8ca3..272df8ca7e 100644 --- a/smarts/core/__init__.py +++ b/smarts/core/__init__.py @@ -30,7 +30,6 @@ import numpy as np -import smarts from smarts.core.configuration import Config _current_seed = None @@ -56,7 +55,7 @@ def gen_id(): @lru_cache(maxsize=1) -def config(default=str("./smarts_engine.ini")) -> Config: +def config(default: str = "./smarts_engine.ini") -> Config: """Get the SMARTS environment config for the smarts engine. .. note:: @@ -75,7 +74,7 @@ def config(default=str("./smarts_engine.ini")) -> Config: """ from smarts.core.utils.file import smarts_global_user_dir, smarts_local_user_dir - def get_file(config_file): + def get_file(config_file: Path): try: if not config_file.is_file(): return "" @@ -86,7 +85,7 @@ def get_file(config_file): conf = partial(Config, environment_prefix="SMARTS") - file = get_file(Path(default)) + file = get_file(Path(default).absolute()) if file: return conf(file) @@ -104,4 +103,5 @@ def get_file(config_file): if file: return conf(file) - return conf(get_file(Path(smarts.__file__).parent.absolute() / "engine.ini")) + default_path = Path(__file__).parents[1].resolve() / "engine.ini" + return conf(get_file(default_path)) diff --git a/smarts/core/configuration.py b/smarts/core/configuration.py index b4818819c2..d756cee409 100644 --- a/smarts/core/configuration.py +++ b/smarts/core/configuration.py @@ -42,17 +42,19 @@ class Config: def __init__( self, config_file: Union[str, Path], environment_prefix: str = "" ) -> None: + self._config = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation() + ) + self._environment_prefix = environment_prefix.upper() + self._environment_variable_format_string = self._environment_prefix + "_{}_{}" + if isinstance(config_file, str): config_file = Path(config_file) + config_file = config_file.resolve() if not config_file.is_file(): raise FileNotFoundError(f"Configuration file not found at {config_file}") - self._config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation() - ) self._config.read(str(config_file.absolute())) - self._environment_prefix = environment_prefix.upper() - self._format_string = self._environment_prefix + "_{}_{}" @property def environment_prefix(self): @@ -84,7 +86,9 @@ def get_setting( KeyError: If the configuration option is not found in the configuration file and no default is provided. configparser.NoSectionError: If the section in the configuration file is not found and no default is provided. """ - env_variable = self._format_string.format(section.upper(), option.upper()) + env_variable = self._environment_variable_format_string.format( + section.upper(), option.upper() + ) setting = os.getenv(env_variable) if setting is not None: return cast(setting) From 949b640d00bc1e370d17148655780826bb962a52 Mon Sep 17 00:00:00 2001 From: Montgomery Alban Date: Thu, 9 Feb 2023 22:02:07 +0000 Subject: [PATCH 10/14] Update documentation --- docs/conf.py | 3 +++ docs/ecosystem/sumo.rst | 2 +- docs/index.rst | 4 +++- docs/resources/contributing.rst | 6 +++++- docs/resources/todo.rst | 8 ++++++++ docs/sim/configuration.rst | 27 +++++++++++++++++++++++++++ 6 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 docs/resources/todo.rst create mode 100644 docs/sim/configuration.rst diff --git a/docs/conf.py b/docs/conf.py index 4196799cd8..b530353480 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,6 +38,7 @@ "sphinx.ext.coverage", # to generate documentation coverage reports "sphinx.ext.extlinks", # shorten external links "sphinx.ext.napoleon", # support Numpy and Google doc style + "sphinx.ext.todo", # support for todo items "sphinx.ext.viewcode", # link to sourcecode from docs "sphinx_rtd_theme", # Read The Docs theme "sphinx_click", # extract documentation from a `click` application @@ -90,6 +91,8 @@ "waymo_open_dataset", ] +todo_include_todos = True + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/ecosystem/sumo.rst b/docs/ecosystem/sumo.rst index bcb832ada6..a01edac9b9 100644 --- a/docs/ecosystem/sumo.rst +++ b/docs/ecosystem/sumo.rst @@ -5,7 +5,7 @@ SUMO Learn SUMO through its user `documentation `_ . -SMARTS currently directly installs SUMO version >=1.12.0 via `pip`. +SMARTS currently directly installs SUMO version >=1.15.0 via `pip`. .. code-block:: bash diff --git a/docs/index.rst b/docs/index.rst index ffdf022393..887e679bc9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -69,6 +69,7 @@ If you use SMARTS in your research, please cite the `paper `_. + +Feel free to further contribute to the documentation and look at :ref:`todo` for sections that may yet need to be filled. + Communication ------------- @@ -141,4 +145,4 @@ Things inevitably become slow, when this happens, Flame Graph is a great tool to $ mkdir -p flamegraph_dir $ curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl > ./utils/third_party/tools/flamegraph.pl $ chmod 777 {$flamegraph_dir}/flamegraph.pl - $ make flamegraph scenario=./scenarios/sumo/loop script=./examples/control/chase_via_points.py + $ make flamegraph scenario=./scenarios/sumo/loop script=./examples/control/chase_via_points.py \ No newline at end of file diff --git a/docs/resources/todo.rst b/docs/resources/todo.rst new file mode 100644 index 0000000000..b3c13e01c5 --- /dev/null +++ b/docs/resources/todo.rst @@ -0,0 +1,8 @@ +.. _todo: + +TODO List +========= + +A list of current documentation TODO. + +.. todolist_:: \ No newline at end of file diff --git a/docs/sim/configuration.rst b/docs/sim/configuration.rst new file mode 100644 index 0000000000..0b6d2a65e3 --- /dev/null +++ b/docs/sim/configuration.rst @@ -0,0 +1,27 @@ +.. _configuration: + +Configuration +============= + +You can change the behavior of the underlying SMARTS engine. + +Configuration of the engine can come from several sources. These locations take precidence as noted: + +1. Individual ``SMARTS_`` prefixed environment variables (e.g. ``SMARTS_SENSOR_WORKER_COUNT``) +2. Local directory engine configuration (./smarts_engine.ini ) +3. Local user engine configuration, ``~/.smarts/engine.ini``, if local directory configuration is not found. +4. Global engine configuration, ``/etc/smarts/engine.ini``, if local configuration is not found. +5. Package default configuration, ``$PYTHON_PATH/smarts/engine.ini``, if global configuration is not found. + +Note that configuration files resolve all settings at the first found configuration file (they do not layer.) + +Options +------- + +All settings demostrated as environment variables are formatted to ``UPPERCASE`` and prefixed with ``SMARTS_`` (e.g. ``[core] logging`` can be configured with ``SMARTS_CORE_LOGGING``) + +These settings are as follows: + +.. todo:: + + List engine settings \ No newline at end of file From dc983c9107c331fb1ae6f3e0d0d5b1181e52a5be Mon Sep 17 00:00:00 2001 From: Tucker Alban Date: Fri, 10 Feb 2023 13:21:15 -0500 Subject: [PATCH 11/14] Update docs/sim/configuration.rst Co-authored-by: adai --- docs/sim/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sim/configuration.rst b/docs/sim/configuration.rst index 0b6d2a65e3..053b605bae 100644 --- a/docs/sim/configuration.rst +++ b/docs/sim/configuration.rst @@ -5,7 +5,7 @@ Configuration You can change the behavior of the underlying SMARTS engine. -Configuration of the engine can come from several sources. These locations take precidence as noted: +Configuration of the engine can come from several sources. These locations take precedence as noted: 1. Individual ``SMARTS_`` prefixed environment variables (e.g. ``SMARTS_SENSOR_WORKER_COUNT``) 2. Local directory engine configuration (./smarts_engine.ini ) From e577d66d83c2fce4baaa4ff4ef77489745430f7e Mon Sep 17 00:00:00 2001 From: Montgomery Alban Date: Fri, 10 Feb 2023 18:41:20 +0000 Subject: [PATCH 12/14] Move sim configuration to next steps. --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 887e679bc9..707088e890 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,6 +46,7 @@ If you use SMARTS in your research, please cite the `paper Date: Fri, 10 Feb 2023 20:25:08 +0000 Subject: [PATCH 13/14] Assign pybullet frequency to make clear. --- smarts/core/smarts.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/smarts/core/smarts.py b/smarts/core/smarts.py index 2e0b546784..f1ea6b697a 100644 --- a/smarts/core/smarts.py +++ b/smarts/core/smarts.py @@ -1282,19 +1282,10 @@ def fixed_timestep_sec(self) -> float: @fixed_timestep_sec.setter def fixed_timestep_sec(self, fixed_timestep_sec: float): if not fixed_timestep_sec: - # This is the fastest we could possibly run given constraints from pybullet - self._rounder = rounder_for_dt( - round( - 1 - / config()( - "physics", - "max_pybullet_freq", - default=MAX_PYBULLET_FREQ, - cast=int, - ), - 6, - ) + max_pybullet_freq = config()( + "physics", "max_pybullet_freq", default=MAX_PYBULLET_FREQ, cast=int ) + self._rounder = rounder_for_dt(round(1 / max_pybullet_freq, 6)) else: self._rounder = rounder_for_dt(fixed_timestep_sec) self._fixed_timestep_sec = fixed_timestep_sec From db8cdc9595d3fd67a242777ecc535392a106bf35 Mon Sep 17 00:00:00 2001 From: Tucker Alban Date: Fri, 10 Feb 2023 15:25:33 -0500 Subject: [PATCH 14/14] Update docs/sim/configuration.rst Co-authored-by: Saul Field --- docs/sim/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sim/configuration.rst b/docs/sim/configuration.rst index 053b605bae..1cb718117d 100644 --- a/docs/sim/configuration.rst +++ b/docs/sim/configuration.rst @@ -18,7 +18,7 @@ Note that configuration files resolve all settings at the first found configurat Options ------- -All settings demostrated as environment variables are formatted to ``UPPERCASE`` and prefixed with ``SMARTS_`` (e.g. ``[core] logging`` can be configured with ``SMARTS_CORE_LOGGING``) +All settings demonstrated as environment variables are formatted to ``UPPERCASE`` and prefixed with ``SMARTS_`` (e.g. ``[core] logging`` can be configured with ``SMARTS_CORE_LOGGING``) These settings are as follows: