Skip to content

Commit

Permalink
Pat/benchmark (#1718)
Browse files Browse the repository at this point in the history
* add benchmark scenarios, benchmark script and modify cli.py

* modify logging.py

* update manifest.ini to include the benchmark files

* add scenarios via upload

* add scenarios via upload

* add scenarios via upload

* progress

* Simplify scenario build function names.

* Change build scenario function name in examples.

* Simplify scl benchmark file.

* progress

* progress

* Return full path in scenario.name

* Return only basename in EpisodeLogs.

* Change log level in egoless.

* Cleaned timeit function.

* Functioning benchmark run.

* Make format.

* progress

* Different episode steps for replay scenarios.

* Fix ci.

* intergration

* Add headers.

* progress

* round data to 2 decimals, clean and reformat

* Add Welford's online std.

* Make format.

* Make format.

* Fix ci test and add copyright header.

* Add report writing.

* format and add documentation

* Ignore reports and improved graph plot.

* Remove unnecesary explanations.

* Add Readme

* Remove duplicate dependency

* Improved benchmark.

* Fix type

* Try

* Try2

* Fix ci and type.

* Move welford function and add docstrings.

* Add header.

* Reorganise imports.

* remove unnecessary files

* remove glbs

* change default episode per sec from 5 to 10

* add missing copyright headers

* typo

* change logging.py and run.py

* format

* remove unncessary comment

Co-authored-by: adaickalavan <[email protected]>
  • Loading branch information
qianyi-sun and Adaickalavan authored Nov 24, 2022
1 parent f7ad0f0 commit e4d6b25
Show file tree
Hide file tree
Showing 75 changed files with 4,414 additions and 31 deletions.
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,7 @@ collected_observations/
*.swp

# Static type check
.pytype
.pytype

# Benchmark
**/benchmark/reports/*
3 changes: 2 additions & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ include smarts/core/models/*.glb smarts/core/models/*.urdf
include smarts/core/models/controller_parameters.yaml
include envision/web/dist/*
recursive-include smarts/ros/src *.launch *.msg *.srv package.xml CMakeLists.txt *.py
recursive-include smarts/scenarios *.xml *.py
recursive-include smarts/scenarios *.xml *.py
recursive-include smarts/benchmark *.xml *.py
40 changes: 40 additions & 0 deletions cli/benchmark.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import click


@click.group(
name="benchmark",
help="Utilities for benchmarking the simulation performance. See `scl benchmark COMMAND --help` for further options.",
)
def benchmark_cli():
pass


@click.command("run", help="Run all benchmarks.")
@click.argument("scenarios", nargs=-1, metavar="<scenarios>")
def run(scenarios):
from smarts.benchmark import run as _run

_run.main(scenarios)


benchmark_cli.add_command(run)
4 changes: 2 additions & 2 deletions cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import click

from cli.benchmark import benchmark_cli
from cli.envision import envision_cli
from cli.run import run_experiment
from cli.studio import scenario_cli
Expand Down Expand Up @@ -60,9 +61,8 @@ def waymo_cli(ctx):
"pip install pathos==0.2.8 tabulate>=0.8.10 waymo-open-dataset-tf-2-4-0"
)


scl.add_command(waymo_cli)

scl.add_command(benchmark_cli)

if __name__ == "__main__":
scl()
26 changes: 12 additions & 14 deletions cli/studio.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,28 +55,28 @@ def scenario_cli():
help="Set the base seed of the scenario.",
)
@click.argument("scenario", type=click.Path(exists=True), metavar="<scenario>")
def build_scenario(clean: bool, allow_offset_map: bool, scenario: str, seed: int):
def build(clean: bool, allow_offset_map: bool, scenario: str, seed: int):
click.echo(f"build-scenario {scenario}")

from smarts.sstudio.scenario_construction import build_single_scenario
from smarts.sstudio.scenario_construction import build_scenario

assert seed == None or isinstance(seed, (int))

build_single_scenario(clean, allow_offset_map, scenario, seed, click.echo)
build_scenario(clean, allow_offset_map, scenario, seed, click.echo)


def _build_single_scenario_proc(
def _build_scenario_proc(
clean: bool,
allow_offset_map: bool,
scenario: str,
semaphore: synchronize.Semaphore,
seed: int,
):
from smarts.sstudio.scenario_construction import build_single_scenario
from smarts.sstudio.scenario_construction import build_scenario

semaphore.acquire()
try:
build_single_scenario(clean, allow_offset_map, scenario, seed, click.echo)
build_scenario(clean, allow_offset_map, scenario, seed, click.echo)
finally:
semaphore.release()

Expand Down Expand Up @@ -117,13 +117,11 @@ def _is_scenario_folder_to_build(path: str) -> bool:
help="Set the base seed of the scenarios.",
)
@click.argument("scenarios", nargs=-1, metavar="<scenarios>")
def build_all_scenarios(
clean: bool, allow_offset_maps: bool, scenarios: List[str], seed: int
):
_build_all_scenarios(clean, allow_offset_maps, scenarios, seed)
def build_all(clean: bool, allow_offset_maps: bool, scenarios: List[str], seed: int):
build_scenarios(clean, allow_offset_maps, scenarios, seed)


def _build_all_scenarios(
def build_scenarios(
clean: bool,
allow_offset_maps: bool,
scenarios: List[str],
Expand All @@ -143,7 +141,7 @@ def _build_all_scenarios(
p = Path(subdir)
scenario = f"{scenarios_path}/{p.relative_to(scenarios_path)}"
proc = Process(
target=_build_single_scenario_proc,
target=_build_scenario_proc,
args=(clean, allow_offset_maps, scenario, sema, seed),
)
all_processes.append((scenario, proc))
Expand Down Expand Up @@ -185,7 +183,7 @@ def replay(directory: Sequence[str], timestep: float, endpoint: str):
)


scenario_cli.add_command(build_scenario)
scenario_cli.add_command(build_all_scenarios)
scenario_cli.add_command(build)
scenario_cli.add_command(build_all)
scenario_cli.add_command(clean_scenario)
scenario_cli.add_command(replay)
6 changes: 3 additions & 3 deletions cli/tests/test_studio.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,16 @@ def test_scenario_generation_unchanged():
loc2 = temp_dir2 + "/scenarios"
import re

from cli.studio import _build_all_scenarios
from cli.studio import build_scenarios

_hashseed = os.getenv("PYTHONHASHSEED")
assert _hashseed not in (None, "random"), f"PYTHONHASHSEED is {_hashseed}"

shutil.copytree("scenarios/sumo", loc1)
_build_all_scenarios(True, True, [loc1], 42)
build_scenarios(True, True, [loc1], 42)

shutil.copytree("scenarios/sumo", loc2)
_build_all_scenarios(True, True, [loc2], 42)
build_scenarios(True, True, [loc2], 42)

for dirpath, dirnames, files in os.walk(loc1):
if "traffic" in dirpath:
Expand Down
2 changes: 1 addition & 1 deletion examples/egoless.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from smarts.core.utils.episodes import episodes

logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.WARNING)


def main(scenarios, headless, num_episodes, max_episode_steps=None):
Expand Down
4 changes: 2 additions & 2 deletions examples/env/figure_eight_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ def entry_point(*args, **kwargs):

scenario = str((Path(__file__).parent / "../../scenarios/figure_eight").resolve())
## Note: can build the scenario here
from smarts.sstudio.scenario_construction import build_single_scenario
from smarts.sstudio.scenario_construction import build_scenario

build_single_scenario(clean=True, allow_offset_map=True, scenario=scenario)
build_scenario(clean=True, allow_offset_map=True, scenario=scenario)
hiwayenv = HiWayEnv(
agent_specs={"agent-007": agent_spec},
scenarios=[scenario],
Expand Down
2 changes: 2 additions & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ console_scripts =
scl=cli.cli:scl

[options.extras_require]
benchmark = py-cpuinfo==9.0.0
mdutils==1.4.0
camera-obs = Panda3D==1.10.9
panda3d-gltf==0.13
dev = black[jupyter]==22.6.0
Expand Down
124 changes: 124 additions & 0 deletions smarts/benchmark/10_agents_to_n_roads/10_roads/map.net.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>

<!-- generated on 2022-08-16 00:05:39 by Eclipse SUMO netedit Version 1.10.0
<configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://sumo.dlr.de/xsd/netconvertConfiguration.xsd">
<input>
<sumo-net-file value="scenarios/NGSIM/i80/map.net.xml"/>
</input>
<output>
<output-file value="scenarios/NGSIM/i80/map.net.xml"/>
</output>
<processing>
<geometry.min-radius.fix.railways value="false"/>
<geometry.max-grade.fix value="false"/>
<offset.disable-normalization value="true"/>
<lefthand value="false"/>
</processing>
<junctions>
<no-turnarounds value="true"/>
<junctions.corner-detail value="5"/>
<junctions.limit-turn-speed value="5.5"/>
<rectangular-lane-cut value="false"/>
</junctions>
<pedestrian>
<walkingareas value="false"/>
</pedestrian>
<report>
<aggregate-warnings value="5"/>
</report>
</configuration>
-->

<net version="1.9" junctionCornerDetail="5" limitTurnSpeed="5.50" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://sumo.dlr.de/xsd/net_file.xsd">

<location netOffset="0.00,0.00" convBoundary="-14.67,1.40,440.62,26.47" origBoundary="-10000000000.00,-10000000000.00,10000000000.00,10000000000.00" projParameter="!"/>

<edge id=":cluster_J2_J8_0" function="internal">
<lane id=":cluster_J2_J8_0_0" index="0" speed="13.89" length="29.82" width="5.00" shape="155.91,2.84 182.29,4.65"/>
<lane id=":cluster_J2_J8_0_1" index="1" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 160.24,3.84 165.89,5.93 173.13,8.01 182.25,8.98"/>
<lane id=":cluster_J2_J8_0_2" index="2" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 160.16,4.41 165.68,7.76 172.89,11.10 182.22,12.64"/>
<lane id=":cluster_J2_J8_0_3" index="3" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 160.05,4.98 165.39,9.59 172.56,14.19 182.18,16.30"/>
<lane id=":cluster_J2_J8_0_4" index="4" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 159.92,5.56 165.03,11.42 172.15,17.27 182.15,19.96"/>
<lane id=":cluster_J2_J8_0_5" index="5" speed="13.89" length="29.82" width="3.66" shape="155.91,2.84 159.77,6.13 164.62,13.25 171.67,20.36 182.12,23.62"/>
</edge>
<edge id=":cluster_J2_J8_6" function="internal">
<lane id=":cluster_J2_J8_6_0" index="0" speed="13.89" length="26.36" width="5.00" shape="155.90,5.49 182.29,4.65"/>
<lane id=":cluster_J2_J8_6_1" index="1" speed="13.89" length="26.36" width="3.66" shape="155.88,9.15 182.25,8.98"/>
<lane id=":cluster_J2_J8_6_2" index="2" speed="13.89" length="26.36" width="3.66" shape="155.86,12.81 182.22,12.64"/>
<lane id=":cluster_J2_J8_6_3" index="3" speed="13.89" length="26.36" width="3.66" shape="155.83,16.47 182.18,16.30"/>
<lane id=":cluster_J2_J8_6_4" index="4" speed="13.89" length="26.36" width="3.66" shape="155.81,20.13 182.15,19.96"/>
<lane id=":cluster_J2_J8_6_5" index="5" speed="13.89" length="26.36" width="3.66" shape="155.79,23.79 182.12,23.62"/>
</edge>

<edge id="E3" from="J5" to="cluster_J2_J8" priority="-1" shape="20.58,1.40 83.18,3.53 165.66,5.58">
<lane id="E3_0" index="0" speed="13.89" length="135.31" width="5.00" shape="20.67,-1.10 83.25,1.03 155.91,2.84"/>
</edge>
<edge id="gneE01" from="gneJ00" to="cluster_J2_J8" priority="-1" shape="-14.67,25.00 155.78,25.62">
<lane id="gneE01_0" index="0" speed="13.89" length="170.48" width="3.66" shape="-14.60,4.87 155.90,5.49"/>
<lane id="gneE01_1" index="1" speed="13.89" length="170.48" width="3.66" shape="-14.61,8.53 155.88,9.15"/>
<lane id="gneE01_2" index="2" speed="13.89" length="170.48" width="3.66" shape="-14.62,12.19 155.86,12.81"/>
<lane id="gneE01_3" index="3" speed="13.89" length="170.48" width="3.66" shape="-14.64,15.85 155.83,16.47"/>
<lane id="gneE01_4" index="4" speed="13.89" length="170.48" width="3.66" shape="-14.65,19.51 155.81,20.13"/>
<lane id="gneE01_5" index="5" speed="13.89" length="170.48" width="3.66" shape="-14.66,23.17 155.79,23.79"/>
</edge>
<edge id="gneE01.132" from="cluster_J2_J8" to="gneJ10" priority="-1" shape="164.99,25.38 440.62,26.47">
<lane id="gneE01.132_0" index="0" speed="13.89" length="258.46" width="5.00" shape="182.29,4.65 440.70,5.67"/>
<lane id="gneE01.132_1" index="1" speed="13.89" length="258.46" width="3.66" shape="182.25,8.98 440.69,10.00"/>
<lane id="gneE01.132_2" index="2" speed="13.89" length="258.46" width="3.66" shape="182.22,12.64 440.67,13.66"/>
<lane id="gneE01.132_3" index="3" speed="13.89" length="258.46" width="3.66" shape="182.18,16.30 440.66,17.32"/>
<lane id="gneE01.132_4" index="4" speed="13.89" length="258.46" width="3.66" shape="182.15,19.96 440.64,20.98"/>
<lane id="gneE01.132_5" index="5" speed="13.89" length="258.46" width="3.66" shape="182.12,23.62 440.63,24.64"/>
</edge>

<junction id="J5" type="dead_end" x="20.58" y="1.40" incLanes="" intLanes="" shape="20.58,1.40 20.75,-3.60"/>
<junction id="cluster_J2_J8" type="priority" x="159.46" y="16.60" incLanes="E3_0 gneE01_0 gneE01_1 gneE01_2 gneE01_3 gneE01_4 gneE01_5" intLanes=":cluster_J2_J8_0_0 :cluster_J2_J8_0_1 :cluster_J2_J8_0_2 :cluster_J2_J8_0_3 :cluster_J2_J8_0_4 :cluster_J2_J8_0_5 :cluster_J2_J8_6_0 :cluster_J2_J8_6_1 :cluster_J2_J8_6_2 :cluster_J2_J8_6_3 :cluster_J2_J8_6_4 :cluster_J2_J8_6_5" shape="182.10,25.78 182.31,2.27 173.97,1.57 165.94,0.84 155.93,0.16 155.78,25.62" customShape="1">
<request index="0" response="111111000000" foes="111111000000" cont="0"/>
<request index="1" response="111111000000" foes="111111000000" cont="0"/>
<request index="2" response="111111000000" foes="111111000000" cont="0"/>
<request index="3" response="111111000000" foes="111111000000" cont="0"/>
<request index="4" response="111111000000" foes="111111000000" cont="0"/>
<request index="5" response="111111000000" foes="111111000000" cont="0"/>
<request index="6" response="000000000000" foes="000000111111" cont="0"/>
<request index="7" response="000000000000" foes="000000111111" cont="0"/>
<request index="8" response="000000000000" foes="000000111111" cont="0"/>
<request index="9" response="000000000000" foes="000000111111" cont="0"/>
<request index="10" response="000000000000" foes="000000111111" cont="0"/>
<request index="11" response="000000000000" foes="000000111111" cont="0"/>
</junction>
<junction id="gneJ00" type="dead_end" x="-14.67" y="25.00" incLanes="" intLanes="" shape="-14.67,25.00 -14.59,3.04"/>
<junction id="gneJ10" type="dead_end" x="440.62" y="26.47" incLanes="gneE01.132_0 gneE01.132_1 gneE01.132_2 gneE01.132_3 gneE01.132_4 gneE01.132_5" intLanes="" shape="440.71,3.17 440.62,26.47"/>

<connection from="E3" to="gneE01.132" fromLane="0" toLane="0" via=":cluster_J2_J8_0_0" dir="s" state="m"/>
<connection from="E3" to="gneE01.132" fromLane="0" toLane="1" via=":cluster_J2_J8_0_1" dir="s" state="m"/>
<connection from="E3" to="gneE01.132" fromLane="0" toLane="2" via=":cluster_J2_J8_0_2" dir="s" state="m"/>
<connection from="E3" to="gneE01.132" fromLane="0" toLane="3" via=":cluster_J2_J8_0_3" dir="s" state="m"/>
<connection from="E3" to="gneE01.132" fromLane="0" toLane="4" via=":cluster_J2_J8_0_4" dir="s" state="m"/>
<connection from="E3" to="gneE01.132" fromLane="0" toLane="5" via=":cluster_J2_J8_0_5" dir="s" state="m"/>
<connection from="gneE01" to="gneE01.132" fromLane="0" toLane="0" via=":cluster_J2_J8_6_0" dir="s" state="M"/>
<connection from="gneE01" to="gneE01.132" fromLane="1" toLane="1" via=":cluster_J2_J8_6_1" dir="s" state="M"/>
<connection from="gneE01" to="gneE01.132" fromLane="2" toLane="2" via=":cluster_J2_J8_6_2" dir="s" state="M"/>
<connection from="gneE01" to="gneE01.132" fromLane="3" toLane="3" via=":cluster_J2_J8_6_3" dir="s" state="M"/>
<connection from="gneE01" to="gneE01.132" fromLane="4" toLane="4" via=":cluster_J2_J8_6_4" dir="s" state="M"/>
<connection from="gneE01" to="gneE01.132" fromLane="5" toLane="5" via=":cluster_J2_J8_6_5" dir="s" state="M"/>

<connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="0" toLane="0" dir="s" state="M"/>
<connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="1" toLane="1" dir="s" state="M"/>
<connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="2" toLane="2" dir="s" state="M"/>
<connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="3" toLane="3" dir="s" state="M"/>
<connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="4" toLane="4" dir="s" state="M"/>
<connection from=":cluster_J2_J8_0" to="gneE01.132" fromLane="5" toLane="5" dir="s" state="M"/>
<connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="0" toLane="0" dir="s" state="M"/>
<connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="1" toLane="1" dir="s" state="M"/>
<connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="2" toLane="2" dir="s" state="M"/>
<connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="3" toLane="3" dir="s" state="M"/>
<connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="4" toLane="4" dir="s" state="M"/>
<connection from=":cluster_J2_J8_6" to="gneE01.132" fromLane="5" toLane="5" dir="s" state="M"/>

</net>
47 changes: 47 additions & 0 deletions smarts/benchmark/10_agents_to_n_roads/10_roads/scenario.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
from pathlib import Path

from smarts.sstudio import gen_scenario
from smarts.sstudio.types import Mission, Route, Scenario, SocialAgentActor,RandomRoute

actors = [
SocialAgentActor(
name=f"non-interactive-agent-50-v0",
agent_locator="zoo.policies:non-interactive-agent-v0",
policy_kwargs={"speed": 50},
)
]


def to_missions(agent_num):
missions = {}
for i in range(0,agent_num):
missions[f'group-{i}']=tuple((actors, [Mission(route=RandomRoute())]),)
return missions

gen_scenario(
Scenario(
social_agent_missions=to_missions(10)
),
output_dir=Path(__file__).parent,
)
Loading

0 comments on commit e4d6b25

Please sign in to comment.