From e01b92a916afc4a9246110394ebd5bbc4617c637 Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Mon, 2 Mar 2020 01:54:28 +0000 Subject: [PATCH 1/3] cudnn expects eps to be >e105 and gamma1,gamma2 renamed in other PR --- benchmark/opperf/rules/default_params.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/opperf/rules/default_params.py b/benchmark/opperf/rules/default_params.py index 2bf80d58e264..aaa095758cf4 100644 --- a/benchmark/opperf/rules/default_params.py +++ b/benchmark/opperf/rules/default_params.py @@ -282,7 +282,7 @@ DEFAULT_LR = [0.1,0.5,0.9] DEFAULT_RHO = [0.1,0.5,0.9] DEFAULT_MOMENTUM = [0.1,0.5,0.9] -DEFAULT_EPSILON = [1e-08] +DEFAULT_EPSILON = [1e-05] DEFAULT_BETA_1 = [0.1, 0.5, 0.9] DEFAULT_BETA_2 = [0.1, 0.5, 0.9] DEFAULT_T = [1, 5] @@ -611,8 +611,8 @@ "lrs" : DEFAULT_LRS, "wds" : DEFAULT_LRS, "wd": DEFAULT_LR, - "gamma1" : DEFAULT_GAMMA_1, - "gamma2" : DEFAULT_GAMMA_2, + "rho" : DEFAULT_RHO, + "momentum" : DEFAULT_MOMENTUM, "epsilon" : DEFAULT_EPSILON, "beta1" : DEFAULT_BETA_1, "beta2" : DEFAULT_BETA_2, From 80a7aacef8177156b23ce40c88ed870dbf7b116c Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Mon, 2 Mar 2020 07:23:49 +0000 Subject: [PATCH 2/3] fix lamb_update_* ops --- benchmark/opperf/rules/default_params.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/benchmark/opperf/rules/default_params.py b/benchmark/opperf/rules/default_params.py index aaa095758cf4..f4f4c9303584 100644 --- a/benchmark/opperf/rules/default_params.py +++ b/benchmark/opperf/rules/default_params.py @@ -423,6 +423,8 @@ "mean" : DEFAULT_MEAN, "var" : DEFAULT_VAR, "mom" : DEFAULT_MOM, + "r1" : DEFAULT_R1, + "r2" : DEFAULT_R2, "n" : DEFAULT_N, "d" : DEFAULT_D, "v" : DEFAULT_V, @@ -431,7 +433,7 @@ "delta" : DEFAULT_DELTA, "lr" : DEFAULT_LR, "lrs" : DEFAULT_LRS, - "wds" : DEFAULT_LRS, + "wd" : DEFAULT_LR, "rho" : DEFAULT_RHO, "momentum" : DEFAULT_MOMENTUM, "epsilon" : DEFAULT_EPSILON, @@ -508,7 +510,6 @@ "gamma_groupnorm": DEFAULT_BETA_GAMMA_GN, "beta_groupnorm": DEFAULT_BETA_GAMMA_GN, "num_groups": DEFAULT_NUM_GROUPS, - "eps": DEFAULT_EPSILON, "data_dropout": DEFAULT_DATA_DROPOUT, "mode_dropout": DEFAULT_MODE_DROPOUT, "p_dropout": DEFAULT_P, @@ -609,7 +610,6 @@ "delta" : DEFAULT_DELTA_LARGE_TENSOR, "lr" : DEFAULT_LR, "lrs" : DEFAULT_LRS, - "wds" : DEFAULT_LRS, "wd": DEFAULT_LR, "rho" : DEFAULT_RHO, "momentum" : DEFAULT_MOMENTUM, @@ -723,7 +723,6 @@ "data_groupnorm": DEFAULT_DATA_GN_LARGE_TENSOR, "gamma_groupnorm": DEFAULT_BETA_GAMMA_GN_LARGE_TENSOR, "beta_groupnorm": DEFAULT_BETA_GAMMA_GN_LARGE_TENSOR, - "eps": DEFAULT_EPSILON, "data_dropout": DEFAULT_DATA_DROPOUT_LARGE_TENSOR, "mode_dropout": DEFAULT_MODE_DROPOUT, "p_dropout": DEFAULT_P_DROPOUT_LARGE_TENSOR, From a527bb0c3ccec06a147f00f2fe4e1415e945e0e0 Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Wed, 4 Mar 2020 18:22:33 +0000 Subject: [PATCH 3/3] add var name for wd --- benchmark/opperf/rules/default_params.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmark/opperf/rules/default_params.py b/benchmark/opperf/rules/default_params.py index f4f4c9303584..81a1ee5859ec 100644 --- a/benchmark/opperf/rules/default_params.py +++ b/benchmark/opperf/rules/default_params.py @@ -279,9 +279,10 @@ DEFAULT_R2 = [(1, 1024), (1, 1), (1, 100)] DEFAULT_DELTA = [(1024, 1024), (10000, 1), (10000, 100)] DEFAULT_LRS = [(0.1,0.1)] -DEFAULT_LR = [0.1,0.5,0.9] -DEFAULT_RHO = [0.1,0.5,0.9] -DEFAULT_MOMENTUM = [0.1,0.5,0.9] +DEFAULT_LR = [0.1, 0.5, 0.9] +DEFAULT_WD = [0.1, 0.5, 0.9] +DEFAULT_RHO = [0.1, 0.5, 0.9] +DEFAULT_MOMENTUM = [0.1, 0.5, 0.9] DEFAULT_EPSILON = [1e-05] DEFAULT_BETA_1 = [0.1, 0.5, 0.9] DEFAULT_BETA_2 = [0.1, 0.5, 0.9] @@ -433,7 +434,7 @@ "delta" : DEFAULT_DELTA, "lr" : DEFAULT_LR, "lrs" : DEFAULT_LRS, - "wd" : DEFAULT_LR, + "wd" : DEFAULT_WD, "rho" : DEFAULT_RHO, "momentum" : DEFAULT_MOMENTUM, "epsilon" : DEFAULT_EPSILON, @@ -610,7 +611,7 @@ "delta" : DEFAULT_DELTA_LARGE_TENSOR, "lr" : DEFAULT_LR, "lrs" : DEFAULT_LRS, - "wd": DEFAULT_LR, + "wd": DEFAULT_WD, "rho" : DEFAULT_RHO, "momentum" : DEFAULT_MOMENTUM, "epsilon" : DEFAULT_EPSILON,