Pat/benchmark

.gitignore

@@ -144,4 +144,7 @@ collected_observations/
 *.swp
 
 # Static type check
-.pytype
+.pytype
+
+# Benchmark
+**/benchmark/reports/*

MANIFEST.in

@@ -3,4 +3,5 @@ include smarts/core/models/*.glb smarts/core/models/*.urdf
 include smarts/core/models/controller_parameters.yaml
 include envision/web/dist/*
 recursive-include smarts/ros/src *.launch *.msg *.srv package.xml CMakeLists.txt *.py
-recursive-include smarts/scenarios *.xml *.py
+recursive-include smarts/scenarios *.xml *.py
+recursive-include smarts/benchmark *.xml *.py

cli/benchmark.py

@@ -0,0 +1,40 @@
+# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import click
+
+
+@click.group(
+    name="benchmark",
+    help="Utilities for benchmarking the simulation performance. See `scl benchmark COMMAND --help` for further options.",
+)
+def benchmark_cli():
+    pass
+
+
+@click.command("run", help="Run all benchmarks.")
+@click.argument("scenarios", nargs=-1, metavar="<scenarios>")
+def run(scenarios):
+    from smarts.benchmark import run as _run
+
+    _run.main(scenarios)
+
+
+benchmark_cli.add_command(run)

cli/cli.py

@@ -22,6 +22,7 @@
 
 import click
 
+from cli.benchmark import benchmark_cli
 from cli.envision import envision_cli
 from cli.run import run_experiment
 from cli.studio import scenario_cli
@@ -60,9 +61,8 @@ def waymo_cli(ctx):
             "pip install pathos==0.2.8 tabulate>=0.8.10 waymo-open-dataset-tf-2-4-0"
         )
 
-
 scl.add_command(waymo_cli)
-
+scl.add_command(benchmark_cli)
 
 if __name__ == "__main__":
     scl()

cli/studio.py

@@ -55,28 +55,28 @@ def scenario_cli():
     help="Set the base seed of the scenario.",
 )
 @click.argument("scenario", type=click.Path(exists=True), metavar="<scenario>")
-def build_scenario(clean: bool, allow_offset_map: bool, scenario: str, seed: int):
+def build(clean: bool, allow_offset_map: bool, scenario: str, seed: int):
     click.echo(f"build-scenario {scenario}")
 
-    from smarts.sstudio.scenario_construction import build_single_scenario
+    from smarts.sstudio.scenario_construction import build_scenario
 
     assert seed == None or isinstance(seed, (int))
 
-    build_single_scenario(clean, allow_offset_map, scenario, seed, click.echo)
+    build_scenario(clean, allow_offset_map, scenario, seed, click.echo)
 
 
-def _build_single_scenario_proc(
+def _build_scenario_proc(
     clean: bool,
     allow_offset_map: bool,
     scenario: str,
     semaphore: synchronize.Semaphore,
     seed: int,
 ):
-    from smarts.sstudio.scenario_construction import build_single_scenario
+    from smarts.sstudio.scenario_construction import build_scenario
 
     semaphore.acquire()
     try:
-        build_single_scenario(clean, allow_offset_map, scenario, seed, click.echo)
+        build_scenario(clean, allow_offset_map, scenario, seed, click.echo)
     finally:
         semaphore.release()
 
@@ -117,13 +117,11 @@ def _is_scenario_folder_to_build(path: str) -> bool:
     help="Set the base seed of the scenarios.",
 )
 @click.argument("scenarios", nargs=-1, metavar="<scenarios>")
-def build_all_scenarios(
-    clean: bool, allow_offset_maps: bool, scenarios: List[str], seed: int
-):
-    _build_all_scenarios(clean, allow_offset_maps, scenarios, seed)
+def build_all(clean: bool, allow_offset_maps: bool, scenarios: List[str], seed: int):
+    build_scenarios(clean, allow_offset_maps, scenarios, seed)
 
 
-def _build_all_scenarios(
+def build_scenarios(
     clean: bool,
     allow_offset_maps: bool,
     scenarios: List[str],
@@ -143,7 +141,7 @@ def _build_all_scenarios(
                 p = Path(subdir)
                 scenario = f"{scenarios_path}/{p.relative_to(scenarios_path)}"
                 proc = Process(
-                    target=_build_single_scenario_proc,
+                    target=_build_scenario_proc,
                     args=(clean, allow_offset_maps, scenario, sema, seed),
                 )
                 all_processes.append((scenario, proc))
@@ -185,7 +183,7 @@ def replay(directory: Sequence[str], timestep: float, endpoint: str):
             )
 
 
-scenario_cli.add_command(build_scenario)
-scenario_cli.add_command(build_all_scenarios)
+scenario_cli.add_command(build)
+scenario_cli.add_command(build_all)
 scenario_cli.add_command(clean_scenario)
 scenario_cli.add_command(replay)

cli/tests/test_studio.py

@@ -47,16 +47,16 @@ def test_scenario_generation_unchanged():
             loc2 = temp_dir2 + "/scenarios"
             import re
 
-            from cli.studio import _build_all_scenarios
+            from cli.studio import build_scenarios
 
             _hashseed = os.getenv("PYTHONHASHSEED")
             assert _hashseed not in (None, "random"), f"PYTHONHASHSEED is {_hashseed}"
 
             shutil.copytree("scenarios/sumo", loc1)
-            _build_all_scenarios(True, True, [loc1], 42)
+            build_scenarios(True, True, [loc1], 42)
 
             shutil.copytree("scenarios/sumo", loc2)
-            _build_all_scenarios(True, True, [loc2], 42)
+            build_scenarios(True, True, [loc2], 42)
 
             for dirpath, dirnames, files in os.walk(loc1):
                 if "traffic" in dirpath:

examples/egoless.py

@@ -5,7 +5,7 @@
 
 from smarts.core.utils.episodes import episodes
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(level=logging.WARNING)
 
 
 def main(scenarios, headless, num_episodes, max_episode_steps=None):

examples/env/figure_eight_env.py

@@ -23,9 +23,9 @@ def entry_point(*args, **kwargs):
 
     scenario = str((Path(__file__).parent / "../../scenarios/figure_eight").resolve())
     ## Note: can build the scenario here
-    from smarts.sstudio.scenario_construction import build_single_scenario
+    from smarts.sstudio.scenario_construction import build_scenario
 
-    build_single_scenario(clean=True, allow_offset_map=True, scenario=scenario)
+    build_scenario(clean=True, allow_offset_map=True, scenario=scenario)
     hiwayenv = HiWayEnv(
         agent_specs={"agent-007": agent_spec},
         scenarios=[scenario],

setup.cfg

@@ -64,6 +64,8 @@ console_scripts =
     scl=cli.cli:scl
 
 [options.extras_require]
+benchmark = py-cpuinfo==9.0.0
+    mdutils==1.4.0
 camera-obs = Panda3D==1.10.9
     panda3d-gltf==0.13
 dev = black[jupyter]==22.6.0

smarts/benchmark/10_agents_to_n_roads/10_roads/map.net.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- generated on 2022-08-16 00:05:39 by Eclipse SUMO netedit Version 1.10.0
+<configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://sumo.dlr.de/xsd/netconvertConfiguration.xsd">
+
+    <input>
+        <sumo-net-file value="scenarios/NGSIM/i80/map.net.xml"/>
+    </input>
+
+    <output>
+        <output-file value="scenarios/NGSIM/i80/map.net.xml"/>
+    </output>
+
+    <processing>
+        <geometry.min-radius.fix.railways value="false"/>
+        <geometry.max-grade.fix value="false"/>
+        <offset.disable-normalization value="true"/>
+        <lefthand value="false"/>
+    </processing>
+
+    <junctions>
+        <no-turnarounds value="true"/>
+        <junctions.corner-detail value="5"/>
+        <junctions.limit-turn-speed value="5.5"/>
+        <rectangular-lane-cut value="false"/>
+    </junctions>
+
+    <pedestrian>
+        <walkingareas value="false"/>
+    </pedestrian>
+
+    <report>
+        <aggregate-warnings value="5"/>
+    </report>
+
+</configuration>
+-->
+
+<net version="1.9" junctionCornerDetail="5" limitTurnSpeed="5.50" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://sumo.dlr.de/xsd/net_file.xsd">
+
+    <location netOffset="0.00,0.00" convBoundary="-14.67,1.40,440.62,26.47" origBoundary="-10000000000.00,-10000000000.00,10000000000.00,10000000000.00" projParameter="!"/>
+
+    <edge id=":cluster_J2_J8_0" function="internal">
+        <lane id=":cluster_J2_J8_0_0" index="0" speed="13.89" length="29.82" width="5.00" shape="155.91,2.84 182.29,4.65"/>
+        <lane id=":cluster_J2_J8_0_1" index="1" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 160.24,3.84 165.89,5.93 173.13,8.01 182.25,8.98"/>
+        <lane id=":cluster_J2_J8_0_2" index="2" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 160.16,4.41 165.68,7.76 172.89,11.10 182.22,12.64"/>
+        <lane id=":cluster_J2_J8_0_3" index="3" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 160.05,4.98 165.39,9.59 172.56,14.19 182.18,16.30"/>
+        <lane id=":cluster_J2_J8_0_4" index="4" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 159.92,5.56 165.03,11.42 172.15,17.27 182.15,19.96"/>
+        <lane id=":cluster_J2_J8_0_5" index="5" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 159.77,6.13 164.62,13.25 171.67,20.36 182.12,23.62"/>
+    </edge>
+    <edge id=":cluster_J2_J8_6" function="internal">
+        <lane id=":cluster_J2_J8_6_0" index="0" speed="13.89" length="26.36" width="5.00" shape="155.90,5.49 182.29,4.65"/>
+        <lane id=":cluster_J2_J8_6_1" index="1" speed="13.89" length="26.36" width="3.66" shape="155.88,9.15 182.25,8.98"/>
+        <lane id=":cluster_J2_J8_6_2" index="2" speed="13.89" length="26.36" width="3.66" shape="155.86,12.81 182.22,12.64"/>
+        <lane id=":cluster_J2_J8_6_3" index="3" speed="13.89" length="26.36" width="3.66" shape="155.83,16.47 182.18,16.30"/>
+        <lane id=":cluster_J2_J8_6_4" index="4" speed="13.89" length="26.36" width="3.66" shape="155.81,20.13 182.15,19.96"/>
+        <lane id=":cluster_J2_J8_6_5" index="5" speed="13.89" length="26.36" width="3.66" shape="155.79,23.79 182.12,23.62"/>
+    </edge>
+
+    <edge id="E3" from="J5" to="cluster_J2_J8" priority="-1" shape="20.58,1.40 83.18,3.53 165.66,5.58">
+        <lane id="E3_0" index="0" speed="13.89" length="135.31" width="5.00" shape="20.67,-1.10 83.25,1.03 155.91,2.84"/>
+    </edge>
+    <edge id="gneE01" from="gneJ00" to="cluster_J2_J8" priority="-1" shape="-14.67,25.00 155.78,25.62">
+        <lane id="gneE01_0" index="0" speed="13.89" length="170.48" width="3.66" shape="-14.60,4.87 155.90,5.49"/>
+        <lane id="gneE01_1" index="1" speed="13.89" length="170.48" width="3.66" shape="-14.61,8.53 155.88,9.15"/>
+        <lane id="gneE01_2" index="2" speed="13.89" length="170.48" width="3.66" shape="-14.62,12.19 155.86,12.81"/>
+        <lane id="gneE01_3" index="3" speed="13.89" length="170.48" width="3.66" shape="-14.64,15.85 155.83,16.47"/>
+        <lane id="gneE01_4" index="4" speed="13.89" length="170.48" width="3.66" shape="-14.65,19.51 155.81,20.13"/>
+        <lane id="gneE01_5" index="5" speed="13.89" length="170.48" width="3.66" shape="-14.66,23.17 155.79,23.79"/>
+    </edge>
+    <edge id="gneE01.132" from="cluster_J2_J8" to="gneJ10" priority="-1" shape="164.99,25.38 440.62,26.47">
+        <lane id="gneE01.132_0" index="0" speed="13.89" length="258.46" width="5.00" shape="182.29,4.65 440.70,5.67"/>
+        <lane id="gneE01.132_1" index="1" speed="13.89" length="258.46" width="3.66" shape="182.25,8.98 440.69,10.00"/>
+        <lane id="gneE01.132_2" index="2" speed="13.89" length="258.46" width="3.66" shape="182.22,12.64 440.67,13.66"/>
+        <lane id="gneE01.132_3" index="3" speed="13.89" length="258.46" width="3.66" shape="182.18,16.30 440.66,17.32"/>
+        <lane id="gneE01.132_4" index="4" speed="13.89" length="258.46" width="3.66" shape="182.15,19.96 440.64,20.98"/>
+        <lane id="gneE01.132_5" index="5" speed="13.89" length="258.46" width="3.66" shape="182.12,23.62 440.63,24.64"/>
+    </edge>
+
+    <junction id="J5" type="dead_end" x="20.58" y="1.40" incLanes="" intLanes="" shape="20.58,1.40 20.75,-3.60"/>
+    <junction id="cluster_J2_J8" type="priority" x="159.46" y="16.60" incLanes="E3_0 gneE01_0 gneE01_1 gneE01_2 gneE01_3 gneE01_4 gneE01_5" intLanes=":cluster_J2_J8_0_0 :cluster_J2_J8_0_1 :cluster_J2_J8_0_2 :cluster_J2_J8_0_3 :cluster_J2_J8_0_4 :cluster_J2_J8_0_5 :cluster_J2_J8_6_0 :cluster_J2_J8_6_1 :cluster_J2_J8_6_2 :cluster_J2_J8_6_3 :cluster_J2_J8_6_4 :cluster_J2_J8_6_5" shape="182.10,25.78 182.31,2.27 173.97,1.57 165.94,0.84 155.93,0.16 155.78,25.62" customShape="1">
+        <request index="0"  response="111111000000" foes="111111000000" cont="0"/>
+        <request index="1"  response="111111000000" foes="111111000000" cont="0"/>
+        <request index="2"  response="111111000000" foes="111111000000" cont="0"/>
+        <request index="3"  response="111111000000" foes="111111000000" cont="0"/>
+        <request index="4"  response="111111000000" foes="111111000000" cont="0"/>
+        <request index="5"  response="111111000000" foes="111111000000" cont="0"/>
+        <request index="6"  response="000000000000" foes="000000111111" cont="0"/>
+        <request index="7"  response="000000000000" foes="000000111111" cont="0"/>
+        <request index="8"  response="000000000000" foes="000000111111" cont="0"/>
+        <request index="9"  response="000000000000" foes="000000111111" cont="0"/>
+        <request index="10" response="000000000000" foes="000000111111" cont="0"/>
+        <request index="11" response="000000000000" foes="000000111111" cont="0"/>
+    </junction>
+    <junction id="gneJ00" type="dead_end" x="-14.67" y="25.00" incLanes="" intLanes="" shape="-14.67,25.00 -14.59,3.04"/>
+    <junction id="gneJ10" type="dead_end" x="440.62" y="26.47" incLanes="gneE01.132_0 gneE01.132_1 gneE01.132_2 gneE01.132_3 gneE01.132_4 gneE01.132_5" intLanes="" shape="440.71,3.17 440.62,26.47"/>
+
+    <connection from="E3" to="gneE01.132" fromLane="0" toLane="0" via=":cluster_J2_J8_0_0" dir="s" state="m"/>
+    <connection from="E3" to="gneE01.132" fromLane="0" toLane="1" via=":cluster_J2_J8_0_1" dir="s" state="m"/>
+    <connection from="E3" to="gneE01.132" fromLane="0" toLane="2" via=":cluster_J2_J8_0_2" dir="s" state="m"/>
+    <connection from="E3" to="gneE01.132" fromLane="0" toLane="3" via=":cluster_J2_J8_0_3" dir="s" state="m"/>
+    <connection from="E3" to="gneE01.132" fromLane="0" toLane="4" via=":cluster_J2_J8_0_4" dir="s" state="m"/>
+    <connection from="E3" to="gneE01.132" fromLane="0" toLane="5" via=":cluster_J2_J8_0_5" dir="s" state="m"/>
+    <connection from="gneE01" to="gneE01.132" fromLane="0" toLane="0" via=":cluster_J2_J8_6_0" dir="s" state="M"/>
+    <connection from="gneE01" to="gneE01.132" fromLane="1" toLane="1" via=":cluster_J2_J8_6_1" dir="s" state="M"/>
+    <connection from="gneE01" to="gneE01.132" fromLane="2" toLane="2" via=":cluster_J2_J8_6_2" dir="s" state="M"/>
+    <connection from="gneE01" to="gneE01.132" fromLane="3" toLane="3" via=":cluster_J2_J8_6_3" dir="s" state="M"/>
+    <connection from="gneE01" to="gneE01.132" fromLane="4" toLane="4" via=":cluster_J2_J8_6_4" dir="s" state="M"/>
+    <connection from="gneE01" to="gneE01.132" fromLane="5" toLane="5" via=":cluster_J2_J8_6_5" dir="s" state="M"/>
+
+    <connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="0" toLane="0" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="1" toLane="1" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="2" toLane="2" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="3" toLane="3" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="4" toLane="4" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="5" toLane="5" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="0" toLane="0" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="1" toLane="1" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="2" toLane="2" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="3" toLane="3" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="4" toLane="4" dir="s" state="M"/>
+    <connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="5" toLane="5" dir="s" state="M"/>
+
+</net>

smarts/benchmark/10_agents_to_n_roads/10_roads/scenario.py

@@ -0,0 +1,47 @@
+# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import os
+from pathlib import Path
+
+from smarts.sstudio import gen_scenario
+from smarts.sstudio.types import Mission, Route, Scenario, SocialAgentActor,RandomRoute
+
+actors = [
+    SocialAgentActor(
+        name=f"non-interactive-agent-50-v0",
+        agent_locator="zoo.policies:non-interactive-agent-v0",
+        policy_kwargs={"speed": 50},
+    )
+]
+
+
+def to_missions(agent_num):
+    missions = {}
+    for i in range(0,agent_num):
+        missions[f'group-{i}']=tuple((actors, [Mission(route=RandomRoute())]),)
+    return missions
+
+gen_scenario(
+    Scenario(
+        social_agent_missions=to_missions(10)
+    ),
+    output_dir=Path(__file__).parent,
+)