Cap the steps metric and weight the score by difficulty

CHANGELOG.md

@@ -33,6 +33,8 @@ Copy and pasting the git commit messages is __NOT__ enough.
 - Changed instances of `hiway-v0` and `gym` to use `hiway-v1` and `gymnasium`, respectively.
 - `RoadMap.Route` now optionally stores the start and end lanes of the route.
 - `DistToDestination` metric is now computed by summing the (i) off-route distance driven by the vehicle from its last on-route position, and (ii) the distance to goal from the vehicle's last on-route position. 
+- `Steps` metric is capped by scenario duration set in the scenario metadata.
+- Overall metric score is weighted by each agent's task difficulty.
 ### Deprecated
 - `visdom` is set to be removed from the SMARTS object parameters.
 - Deprecated `start_time` on missions.

smarts/benchmark/driving_smarts/v2023/config_1.yaml

@@ -2,8 +2,6 @@
 benchmark:
   name: "Driving SMARTS 2023.1"
   message: |
-    This is the Driving SMARTS 2023.1 benchmark.
-
     For context see: 
         - https://smarts-project.github.io/competition/2023_driving_smarts/
         - https://codalab.lisn.upsaclay.fr/competitions/

smarts/benchmark/driving_smarts/v2023/config_2.yaml

@@ -2,8 +2,6 @@
 benchmark:
   name: "Driving SMARTS 2023.2"
   message: |
-    This is the Driving SMARTS 2023.2 benchmark.
-
     For context see: 
         - https://smarts-project.github.io/competition/2023_driving_smarts/
         - https://codalab.lisn.upsaclay.fr/competitions/

smarts/benchmark/driving_smarts/v2023/config_3.yaml

@@ -2,8 +2,6 @@
 benchmark:
   name: "Driving SMARTS 2023.3"
   message: |
-    This is the Driving SMARTS 2023.3 benchmark.
-
     For context see: 
         - https://smarts-project.github.io/competition/2023_driving_smarts/
         - https://codalab.lisn.upsaclay.fr/competitions/

smarts/benchmark/driving_smarts/v2023/metric_formula_drive.py

@@ -25,13 +25,27 @@
 import numpy as np
 
 from smarts.env.gymnasium.wrappers.metric.costs import Costs
-from smarts.env.gymnasium.wrappers.metric.formula import FormulaBase, Score, avg_costs
+from smarts.env.gymnasium.wrappers.metric.formula import (
+    FormulaBase,
+    Score,
+    agent_scores,
+    agent_weights,
+    score_rule_violation,
+    weighted_score,
+)
 from smarts.env.gymnasium.wrappers.metric.params import (
+    Collisions,
     Comfort,
     DistToDestination,
     DistToObstacles,
     JerkLinear,
+    LaneCenterOffset,
+    OffRoad,
     Params,
+    SpeedLimit,
+    Steps,
+    VehicleGap,
+    WrongWay,
 )
 from smarts.env.gymnasium.wrappers.metric.types import Record
 
@@ -51,75 +65,83 @@ def params(self) -> Params:
             Params: Cost function parameters.
         """
         params = Params(
-            comfort=Comfort(
-                active=True,
-            ),
-            dist_to_destination=DistToDestination(
-                active=True,
-            ),
-            dist_to_obstacles=DistToObstacles(
-                active=False,
-            ),
+            collisions=Collisions(active=False),
+            comfort=Comfort(active=True),
+            dist_to_destination=DistToDestination(active=True),
+            dist_to_obstacles=DistToObstacles(active=False),
             jerk_linear=JerkLinear(active=False),
+            lane_center_offset=LaneCenterOffset(active=True),
+            off_road=OffRoad(active=False),
+            speed_limit=SpeedLimit(active=True),
+            steps=Steps(active=True),
+            vehicle_gap=VehicleGap(active=False),
+            wrong_way=WrongWay(active=True),
         )
         return params
 
-    def score(self, records_sum: Dict[str, Dict[str, Record]]) -> Score:
+    def score(self, records: Dict[str, Dict[str, Record]]) -> Score:
         """
         Computes several sub-component scores and one total combined score named
         "Overall" on the wrapped environment.
 
-        +-------------------+--------+-----------------------------------------------------------+
-        |                   | Range  | Remarks                                                   |
-        +===================+========+===========================================================+
-        | Overall           | [0, 1] | Total score. The higher, the better.                      |
-        +-------------------+--------+-----------------------------------------------------------+
-        | DistToDestination | [0, 1] | Remaining distance to destination. The lower, the better. |
-        +-------------------+--------+-----------------------------------------------------------+
-        | Time              | [0, 1] | Time taken to complete scenario. The lower, the better.   |
-        +-------------------+--------+-----------------------------------------------------------+
-        | HumannessError    | [0, 1] | Humanness indicator. The lower, the better.               |
-        +-------------------+--------+-----------------------------------------------------------+
-        | RuleViolation     | [0, 1] | Traffic rules compliance. The lower, the better.          |
-        +-------------------+--------+-----------------------------------------------------------+
+        Args:
+            records (Dict[str, Dict[str, Record]]): Records.       
 
         Returns:
-            Score: Contains "Overall", "DistToDestination", "VehicleGap",
-            "HumannessError", and "RuleViolation" scores.
+            Score: "Overall" score and other sub-component scores.
         """
 
-        costs_final = avg_costs(records_sum=records_sum)
-
-        # Compute sub-components of score.
-        dist_to_destination = costs_final.dist_to_destination
-        humanness_error = _humanness_error(costs=costs_final)
-        rule_violation = _rule_violation(costs=costs_final)
-        time = costs_final.steps
-        overall = (
-            0.25 * (1 - dist_to_destination)
-            + 0.25 * (1 - time)
-            + 0.25 * (1 - humanness_error)
-            + 0.25 * (1 - rule_violation)
-        )
-
-        return Score(
-            {
-                "overall": overall,
-                "dist_to_destination": dist_to_destination,
-                "time": time,
-                "humanness_error": humanness_error,
-                "rule_violation": rule_violation,
-            }
-        )
-
-
-def _humanness_error(costs: Costs) -> float:
+        agent_weight = agent_weights(records=records)
+        agent_score = agent_scores(records=records, func=costs_to_score)
+        return weighted_score(scores=agent_score, weights=agent_weight)
+
+
+def costs_to_score(costs: Costs) -> Score:
+    """Compute score from costs.
+
+    +-------------------+--------+-----------------------------------------------------------+
+    |                   | Range  | Remarks                                                   |
+    +===================+========+===========================================================+
+    | Overall           | [0, 1] | Total score. The higher, the better.                      |
+    +-------------------+--------+-----------------------------------------------------------+
+    | DistToDestination | [0, 1] | Remaining distance to destination. The lower, the better. |
+    +-------------------+--------+-----------------------------------------------------------+
+    | Time              | [0, 1] | Time taken to complete scenario. The lower, the better.   |
+    +-------------------+--------+-----------------------------------------------------------+
+    | HumannessError    | [0, 1] | Humanness indicator. The lower, the better.               |
+    +-------------------+--------+-----------------------------------------------------------+
+    | RuleViolation     | [0, 1] | Traffic rules compliance. The lower, the better.          |
+    +-------------------+--------+-----------------------------------------------------------+
+
+    Args:
+        costs (Costs): Costs.
+
+    Returns:
+        Score: Score.
+    """
+    dist_to_destination = costs.dist_to_destination
+    humanness_error = _score_humanness_error(costs=costs)
+    rule_violation = score_rule_violation(costs=costs)
+    time = costs.steps
+    overall = (
+        0.25 * (1 - dist_to_destination)
+        + 0.25 * (1 - time)
+        + 0.25 * (1 - humanness_error)
+        + 0.25 * (1 - rule_violation)
+    )
+
+    return Score(
+        {
+            "overall": overall,
+            "dist_to_destination": dist_to_destination,
+            "time": time,
+            "humanness_error": humanness_error,
+            "rule_violation": rule_violation,
+        }
+    )
+
+
+def _score_humanness_error(costs: Costs) -> float:
     humanness_error = np.array([costs.comfort, costs.lane_center_offset])
     humanness_error = np.mean(humanness_error, dtype=float)
     return humanness_error
-
-
-def _rule_violation(costs: Costs) -> float:
-    rule_violation = np.array([costs.speed_limit, costs.wrong_way])
-    rule_violation = np.mean(rule_violation, dtype=float)
-    return rule_violation

smarts/benchmark/driving_smarts/v2023/metric_formula_platoon.py

@@ -25,15 +25,27 @@
 import numpy as np
 
 from smarts.env.gymnasium.wrappers.metric.costs import Costs
-from smarts.env.gymnasium.wrappers.metric.formula import FormulaBase, Score, avg_costs
+from smarts.env.gymnasium.wrappers.metric.formula import (
+    FormulaBase,
+    Score,
+    agent_scores,
+    agent_weights,
+    score_rule_violation,
+    weighted_score,
+)
 from smarts.env.gymnasium.wrappers.metric.params import (
+    Collisions,
     Comfort,
     DistToDestination,
     DistToObstacles,
     JerkLinear,
+    LaneCenterOffset,
+    OffRoad,
     Params,
+    SpeedLimit,
     Steps,
     VehicleGap,
+    WrongWay,
 )
 from smarts.env.gymnasium.wrappers.metric.types import Record
 
@@ -53,81 +65,83 @@ def params(self) -> Params:
             Params: Cost function parameters.
         """
         params = Params(
-            comfort=Comfort(
-                active=True,
-            ),
-            dist_to_destination=DistToDestination(
-                active=True,
-            ),
-            dist_to_obstacles=DistToObstacles(
-                active=False,
-            ),
+            collisions=Collisions(active=False),
+            comfort=Comfort(active=True),
+            dist_to_destination=DistToDestination(active=True),
+            dist_to_obstacles=DistToObstacles(active=False),
             jerk_linear=JerkLinear(active=False),
-            vehicle_gap=VehicleGap(
-                active=True,
-            ),
-            steps=Steps(
-                active=False,
-            ),
+            lane_center_offset=LaneCenterOffset(active=True),
+            off_road=OffRoad(active=False),
+            speed_limit=SpeedLimit(active=True),
+            steps=Steps(active=False),
+            vehicle_gap=VehicleGap(active=True),
+            wrong_way=WrongWay(active=True),
         )
         return params
 
-    def score(self, records_sum: Dict[str, Dict[str, Record]]) -> Score:
+    def score(self, records: Dict[str, Dict[str, Record]]) -> Score:
         """
         Computes several sub-component scores and one total combined score named
         "Overall" on the wrapped environment.
 
-        +-------------------+--------+-----------------------------------------------------------+
-        |                   | Range  | Remarks                                                   |
-        +===================+========+===========================================================+
-        | Overall           | [0, 1] | Total score. The higher, the better.                      |
-        +-------------------+--------+-----------------------------------------------------------+
-        | DistToDestination | [0, 1] | Remaining distance to destination. The lower, the better. |
-        +-------------------+--------+-----------------------------------------------------------+
-        | VehicleGap        | [0, 1] | Gap between vehicles in a convoy. The lower, the better.  |
-        +-------------------+--------+-----------------------------------------------------------+
-        | HumannessError    | [0, 1] | Humanness indicator. The lower, the better.               |
-        +-------------------+--------+-----------------------------------------------------------+
-        | RuleViolation     | [0, 1] | Traffic rules compliance. The lower, the better.          |
-        +-------------------+--------+-----------------------------------------------------------+
+        Args:
+            records (Dict[str, Dict[str, Record]]): Records.       
 
         Returns:
-            Score: Contains "Overall", "DistToDestination", "VehicleGap",
-            "HumannessError", and "RuleViolation" scores.
+            Score: "Overall" score and other sub-component scores.
         """
 
-        costs_final = avg_costs(records_sum=records_sum)
-
-        # Compute sub-components of score.
-        dist_to_destination = costs_final.dist_to_destination
-        humanness_error = _humanness_error(costs=costs_final)
-        rule_violation = _rule_violation(costs=costs_final)
-        vehicle_gap = costs_final.vehicle_gap
-        overall = (
-            0.25 * (1 - dist_to_destination)
-            + 0.25 * (1 - vehicle_gap)
-            + 0.25 * (1 - humanness_error)
-            + 0.25 * (1 - rule_violation)
-        )
-
-        return Score(
-            {
-                "overall": overall,
-                "dist_to_destination": dist_to_destination,
-                "vehicle_gap": vehicle_gap,
-                "humanness_error": humanness_error,
-                "rule_violation": rule_violation,
-            }
-        )
-
-
-def _humanness_error(costs: Costs) -> float:
+        agent_weight = agent_weights(records=records)
+        agent_score = agent_scores(records=records, func=costs_to_score)
+        return weighted_score(scores=agent_score, weights=agent_weight)
+
+
+def costs_to_score(costs: Costs) -> Score:
+    """Compute score from costs.
+
+    +-------------------+--------+-----------------------------------------------------------+
+    |                   | Range  | Remarks                                                   |
+    +===================+========+===========================================================+
+    | Overall           | [0, 1] | Total score. The higher, the better.                      |
+    +-------------------+--------+-----------------------------------------------------------+
+    | DistToDestination | [0, 1] | Remaining distance to destination. The lower, the better. |
+    +-------------------+--------+-----------------------------------------------------------+
+    | VehicleGap        | [0, 1] | Gap between vehicles in a convoy. The lower, the better.  |
+    +-------------------+--------+-----------------------------------------------------------+
+    | HumannessError    | [0, 1] | Humanness indicator. The lower, the better.               |
+    +-------------------+--------+-----------------------------------------------------------+
+    | RuleViolation     | [0, 1] | Traffic rules compliance. The lower, the better.          |
+    +-------------------+--------+-----------------------------------------------------------+
+
+    Args:
+        costs (Costs): Costs.
+
+    Returns:
+        Score: Score.
+    """
+    dist_to_destination = costs.dist_to_destination
+    humanness_error = _score_humanness_error(costs=costs)
+    rule_violation = score_rule_violation(costs=costs)
+    vehicle_gap = costs.vehicle_gap
+    overall = (
+        0.25 * (1 - dist_to_destination)
+        + 0.25 * (1 - vehicle_gap)
+        + 0.25 * (1 - humanness_error)
+        + 0.25 * (1 - rule_violation)
+    )
+
+    return Score(
+        {
+            "overall": overall,
+            "dist_to_destination": dist_to_destination,
+            "vehicle_gap": vehicle_gap,
+            "humanness_error": humanness_error,
+            "rule_violation": rule_violation,
+        }
+    )
+
+
+def _score_humanness_error(costs: Costs) -> float:
     humanness_error = np.array([costs.comfort, costs.lane_center_offset])
     humanness_error = np.mean(humanness_error, dtype=float)
     return humanness_error
-
-
-def _rule_violation(costs: Costs) -> float:
-    rule_violation = np.array([costs.speed_limit, costs.wrong_way])
-    rule_violation = np.mean(rule_violation, dtype=float)
-    return rule_violation