Add log error metrics

src/gluonts/ev/__init__.py

@@ -17,6 +17,9 @@
     error,
     absolute_error,
     squared_error,
+    log_error,
+    absolute_log_error,
+    squared_log_error,
     quantile_loss,
     coverage,
     absolute_percentage_error,
@@ -30,6 +33,8 @@
     sum_absolute_label,
     SumAbsoluteError,
     MSE,
+    MALE,
+    MSLE,
     SumQuantileLoss,
     Coverage,
     MAPE,
@@ -38,6 +43,8 @@
     MASE,
     ND,
     RMSE,
+    EMALE,
+    ERMSLE,
     NRMSE,
     WeightedSumQuantileLoss,
     MAECoverage,
@@ -54,6 +61,9 @@
     "error",
     "absolute_error",
     "squared_error",
+    "log_error",
+    "absolute_log_error",
+    "squared_log_error",
     "quantile_loss",
     "coverage",
     "absolute_percentage_error",
@@ -64,6 +74,8 @@
     "sum_absolute_label",
     "SumAbsoluteError",
     "MSE",
+    "MALE",
+    "MSLE",
     "SumQuantileLoss",
     "Coverage",
     "MAPE",
@@ -72,6 +84,8 @@
     "MASE",
     "ND",
     "RMSE",
+    "EMALE",
+    "ERMSLE",
     "NRMSE",
     "WeightedSumQuantileLoss",
     "MAECoverage",

src/gluonts/ev/metrics.py

@@ -24,12 +24,14 @@
     error,
     absolute_error,
     absolute_label,
+    absolute_log_error,
     absolute_percentage_error,
     absolute_scaled_error,
     coverage,
     quantile_loss,
     scaled_interval_score,
     squared_error,
+    squared_log_error,
     symmetric_absolute_percentage_error,
 )
 
@@ -94,6 +96,36 @@ def __call__(self, axis: Optional[int] = None) -> DirectEvaluator:
         )
 
 
+@dataclass
+class MALE:
+    """Mean Absolute Log Error"""
+
+    forecast_type: str = "0.5"
+
+    def __call__(self, axis: Optional[int] = None) -> DirectEvaluator:
+        return DirectEvaluator(
+            name="MALE",
+            stat=partial(absolute_log_error, forecast_type=self.forecast_type),
+            aggregate=Mean(axis=axis),
+        )
+
+
+@dataclass
+class MSLE:
+    """Mean Squared Log Error"""
+
+    forecast_type: str = "0.5"
+    # technically, the forecast type should be "geometric mean",
+    # but forecast objects do not provide this (for now)
+
+    def __call__(self, axis: Optional[int] = None) -> DirectEvaluator:
+        return DirectEvaluator(
+            name="MSLE",
+            stat=partial(squared_log_error, forecast_type=self.forecast_type),
+            aggregate=Mean(axis=axis),
+        )
+
+
 @dataclass
 class SumQuantileLoss:
     q: float
@@ -228,6 +260,54 @@ def __call__(self, axis: Optional[int] = None) -> DerivedEvaluator:
         )
 
 
+@dataclass
+class EMALE:
+    """Exponential Mean Absolute Log Error"""
+
+    forecast_type: str = "0.5"
+
+    @staticmethod
+    def exponential_mean_absolute_log_error(
+        mean_absolute_log_error: np.ndarray,
+    ) -> np.ndarray:
+        return np.exp(mean_absolute_log_error)
+
+    def __call__(self, axis: Optional[int] = None) -> DerivedEvaluator:
+        return DerivedEvaluator(
+            name="EMALE",
+            evaluators={
+                "mean_absolute_log_error": MALE(
+                    forecast_type=self.forecast_type
+                )(axis=axis)
+            },
+            post_process=self.exponential_mean_absolute_log_error,
+        )
+
+
+@dataclass
+class ERMSLE:
+    """Exponential Root Mean Squared Log Error"""
+
+    forecast_type: str = "0.5"
+
+    @staticmethod
+    def exponential_root_mean_squared_log_error(
+        mean_squared_log_error: np.ndarray,
+    ) -> np.ndarray:
+        return np.exp(np.sqrt(mean_squared_log_error))
+
+    def __call__(self, axis: Optional[int] = None) -> DerivedEvaluator:
+        return DerivedEvaluator(
+            name="ERMSLE",
+            evaluators={
+                "mean_squared_log_error": MSLE(
+                    forecast_type=self.forecast_type
+                )(axis=axis)
+            },
+            post_process=self.exponential_root_mean_squared_log_error,
+        )
+
+
 @dataclass
 class NRMSE:
     """RMSE, normalized by the mean absolute label"""

src/gluonts/ev/stats.py

@@ -36,6 +36,22 @@ def squared_error(
     return np.square(error(data, forecast_type))
 
 
+def log_error(data: Dict[str, np.ndarray], forecast_type: str) -> np.ndarray:
+    return np.log(data["label"] / data[forecast_type])
+
+
+def absolute_log_error(
+    data: Dict[str, np.ndarray], forecast_type: str
+) -> np.ndarray:
+    return np.abs(log_error(data, forecast_type))
+
+
+def squared_log_error(
+    data: Dict[str, np.ndarray], forecast_type: str
+) -> np.ndarray:
+    return np.square(log_error(data, forecast_type))
+
+
 def quantile_loss(data: Dict[str, np.ndarray], q: float) -> np.ndarray:
     forecast_type = str(q)
     prediction = data[forecast_type]

src/gluonts/evaluation/_base.py

@@ -42,10 +42,12 @@
     abs_target_sum,
     calculate_seasonal_error,
     coverage,
+    male,
     mape,
     mase,
     mse,
     msis,
+    msle,
     quantile_loss,
     smape,
 )
@@ -400,6 +402,8 @@ def get_metrics_per_ts(
             if mean_fcst is not None
             else None,
             "abs_error": abs_error(pred_target, median_fcst),
+            "MALE": male(pred_target, median_fcst),
+            "MSLE": msle(pred_target, median_fcst),
             "abs_target_sum": abs_target_sum(pred_target),
             "abs_target_mean": abs_target_mean(pred_target),
             "seasonal_error": seasonal_error,
@@ -479,6 +483,8 @@ def get_aggregate_metrics(
         agg_funs = {
             "MSE": "mean",
             "abs_error": "sum",
+            "MALE": "mean",
+            "MSLE": "mean",
             "abs_target_sum": "sum",
             "abs_target_mean": "mean",
             "seasonal_error": "mean",
@@ -512,6 +518,8 @@ def get_aggregate_metrics(
         totals["RMSE"] = np.sqrt(totals["MSE"])
         totals["NRMSE"] = totals["RMSE"] / totals["abs_target_mean"]
         totals["ND"] = totals["abs_error"] / totals["abs_target_sum"]
+        totals["EMALE"] = np.exp(totals["MALE"])
+        totals["ERMSLE"] = np.exp(np.sqrt(totals["MSLE"]))
 
         for quantile in self.quantiles:
             totals[f"wQuantileLoss[{quantile}]"] = (

src/gluonts/evaluation/metrics.py

@@ -72,6 +72,24 @@ def abs_error(target: np.ndarray, forecast: np.ndarray) -> float:
     return np.sum(np.abs(target - forecast))
 
 
+def male(target: np.ndarray, forecast: np.ndarray) -> float:
+    r"""
+    .. math::
+
+        male = mean(|\log(Y / \hat{Y})|)
+    """
+    return np.mean(np.abs(np.log(target / forecast)))
+
+
+def msle(target: np.ndarray, forecast: np.ndarray) -> float:
+    r"""
+    .. math::
+
+        msle = mean(\log(Y / \hat{Y})^2)
+    """
+    return np.mean(np.square(np.log(target / forecast)))
+
+
 def quantile_loss(target: np.ndarray, forecast: np.ndarray, q: float) -> float:
     r"""
     .. math::

test/ev/test_metrics_compared_to_previous_approach.py

@@ -35,6 +35,8 @@
     MAPE,
     MASE,
     MSE,
+    MALE,
+    MSLE,
     MSIS,
     SMAPE,
     Coverage,
@@ -45,6 +47,8 @@
     ND,
     NRMSE,
     RMSE,
+    EMALE,
+    ERMSLE,
     MAECoverage,
     MeanSumQuantileLoss,
     MeanWeightedSumQuantileLoss,
@@ -158,6 +162,8 @@ def get_new_metrics(test_data, predictor, quantile_levels):
         *(SumQuantileLoss(q=quantile.value) for quantile in quantiles),
         mean_absolute_label,
         MSE(),
+        MALE(),
+        MSLE(),
         MASE(),
         MAPE(),
         SMAPE(),
@@ -168,6 +174,8 @@ def get_new_metrics(test_data, predictor, quantile_levels):
         RMSE(),
         NRMSE(),
         ND(),
+        EMALE(),
+        ERMSLE(),
         *(WeightedSumQuantileLoss(q=quantile.value) for quantile in quantiles),
         MeanSumQuantileLoss([quantile.value for quantile in quantiles]),
         MeanWeightedSumQuantileLoss(
@@ -237,10 +245,18 @@ def get_new_metrics(test_data, predictor, quantile_levels):
         "MAE_Coverage": aggregated_metrics["MAE_coverage"],
     }
 
-    for metric_name in ["MSE", "MASE", "MAPE", "sMAPE", "MSIS"]:
+    for metric_name in [
+        "MSE",
+        "MALE",
+        "MSLE",
+        "MASE",
+        "MAPE",
+        "sMAPE",
+        "MSIS",
+    ]:
         all_metrics[metric_name] = np.ma.mean(item_metrics[metric_name])
 
-    for metric_name in ["RMSE", "NRMSE", "ND", "OWA"]:
+    for metric_name in ["RMSE", "NRMSE", "ND", "EMALE", "ERMSLE", "OWA"]:
         all_metrics[metric_name] = aggregated_metrics[metric_name]
 
     return all_metrics

test/ev/test_stats.py

@@ -15,12 +15,15 @@
 
 from gluonts.ev import (
     absolute_error,
+    absolute_log_error,
     absolute_label,
     absolute_percentage_error,
     coverage,
     error,
+    log_error,
     quantile_loss,
     squared_error,
+    squared_log_error,
     symmetric_absolute_percentage_error,
     scaled_interval_score,
     absolute_scaled_error,
@@ -88,6 +91,36 @@ def test_squared_error():
             np.testing.assert_almost_equal(actual, expected)
 
 
+def test_log_error():
+    for label in TIME_SERIES:
+        for forecast in TIME_SERIES:
+            data = {"label": label, "0.5": forecast}
+            actual = log_error(data, forecast_type="0.5")
+            expected = np.log(label / forecast)
+
+            np.testing.assert_almost_equal(actual, expected)
+
+
+def test_abs_log_error():
+    for label in TIME_SERIES:
+        for forecast in TIME_SERIES:
+            data = {"label": label, "0.5": forecast}
+            actual = absolute_log_error(data, forecast_type="0.5")
+            expected = np.abs(np.log(label / forecast))
+
+            np.testing.assert_almost_equal(actual, expected)
+
+
+def test_squared_log_error():
+    for label in TIME_SERIES:
+        for forecast in TIME_SERIES:
+            data = {"label": label, "0.5": forecast}
+            actual = squared_log_error(data, forecast_type="0.5")
+            expected = np.square(np.log(label / forecast))
+
+            np.testing.assert_almost_equal(actual, expected)
+
+
 def test_quantile_loss():
     for label in TIME_SERIES:
         for forecast in TIME_SERIES: