From f4fd9323de4f9040a7d15984284b99b4e6362408 Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Thu, 18 Jul 2019 18:29:52 -0700 Subject: [PATCH 1/5] add array rearrange operators to opperf --- benchmark/opperf/nd_operations/README.md | 5 -- .../opperf/nd_operations/array_rearrange.py | 57 +++++++++++++++++++ .../opperf/nd_operations/binary_operators.py | 4 +- benchmark/opperf/opperf.py | 4 ++ benchmark/opperf/rules/default_params.py | 10 +++- benchmark/opperf/utils/op_registry_utils.py | 20 +++++++ 6 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 benchmark/opperf/nd_operations/array_rearrange.py diff --git a/benchmark/opperf/nd_operations/README.md b/benchmark/opperf/nd_operations/README.md index b98a0d3d0c48..321158c48399 100644 --- a/benchmark/opperf/nd_operations/README.md +++ b/benchmark/opperf/nd_operations/README.md @@ -66,9 +66,7 @@ 46. linalg_extractdiag 47. sgd_mom_update 48. SequenceLast -50. flip 51. SequenceReverse -52. swapaxes 53. SVMOutput 54. linalg_trsm 55. where @@ -82,7 +80,6 @@ 63. mp_sgd_mom_update 64. choose_element_0index 65. tile -66. space_to_depth 67. gather_nd 69. SequenceMask 70. reshape_like @@ -110,14 +107,12 @@ 94. broadcast_like 95. Embedding 96. linalg_makediag -97. transpose 98. linalg_syrk 99. squeeze 101. ROIPooling 102. ftrl_update 103. SliceChannel 104. slice_like -105. depth_to_space 106. linalg_maketrian 108. pad 109. LayerNorm diff --git a/benchmark/opperf/nd_operations/array_rearrange.py b/benchmark/opperf/nd_operations/array_rearrange.py new file mode 100644 index 000000000000..e474f8852a3b --- /dev/null +++ b/benchmark/opperf/nd_operations/array_rearrange.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from benchmark.opperf.utils.benchmark_utils import run_op_benchmarks +from benchmark.opperf.utils.op_registry_utils import get_all_rearrange_operators + +"""Performance benchmark tests for MXNet NDArray Rearrange Operators. + +1. transpose +2. swapaxes +3. flip +4. depth_to_space +5. space_to_depth +""" + + +def run_rearrange_operators_benchmarks(ctx=mx.cpu(), dtype='float32', warmup=25, runs=100): + """Runs benchmarks with the given context and precision (dtype) for all the + rearrange operators in MXNet. + + Parameters + ---------- + ctx: mx.ctx + Context to run benchmarks + dtype: str, default 'float32' + Precision to use for benchmarks + warmup: int, default 25 + Number of times to run for warmup + runs: int, default 100 + Number of runs to capture benchmark results + + Returns + ------- + Dictionary of results. Key -> Name of the operator, Value -> Benchmark results. + + """ + # Fetch all optimizer operators + mx_rearrange_ops = get_all_rearrange_operators() + + # Run benchmarks + mx_rearrange_op_results = run_op_benchmarks(mx_rearrange_ops, dtype, ctx, warmup, runs) + return mx_rearrange_op_results \ No newline at end of file diff --git a/benchmark/opperf/nd_operations/binary_operators.py b/benchmark/opperf/nd_operations/binary_operators.py index cca8f9d1f2ad..1898f5dcbe04 100644 --- a/benchmark/opperf/nd_operations/binary_operators.py +++ b/benchmark/opperf/nd_operations/binary_operators.py @@ -39,7 +39,7 @@ def run_mx_binary_broadcast_operators_benchmarks(ctx=mx.cpu(), dtype='float32', warmup=25, runs=100): - """Runs benchmarks with the given context and precision (dtype)for all the binary + """Runs benchmarks with the given context and precision (dtype) for all the binary broadcast operators in MXNet. Parameters @@ -66,7 +66,7 @@ def run_mx_binary_broadcast_operators_benchmarks(ctx=mx.cpu(), dtype='float32', def run_mx_binary_element_wise_operators_benchmarks(ctx=mx.cpu(), dtype='float32', warmup=25, runs=100): - """Runs benchmarks with the given context and precision (dtype)for all the binary + """Runs benchmarks with the given context and precision (dtype) for all the binary element_wise operators in MXNet. Parameters diff --git a/benchmark/opperf/opperf.py b/benchmark/opperf/opperf.py index b2258afaaaf7..77b16670b443 100755 --- a/benchmark/opperf/opperf.py +++ b/benchmark/opperf/opperf.py @@ -39,6 +39,7 @@ from benchmark.opperf.nd_operations.nn_conv_operators import run_pooling_operators_benchmarks, \ run_convolution_operators_benchmarks, run_transpose_convolution_operators_benchmarks from benchmark.opperf.nd_operations.nn_basic_operators import run_nn_basic_operators_benchmarks +from benchmark.opperf.nd_operations.array_rearrange import run_rearrange_operators_benchmarks from benchmark.opperf.utils.common_utils import merge_map_list, save_to_file from benchmark.opperf.utils.op_registry_utils import get_operators_with_no_benchmark, \ @@ -78,6 +79,9 @@ def run_all_mxnet_operator_benchmarks(ctx=mx.cpu(), dtype='float32'): # Run all Sorting and Searching operations benchmarks with default input values mxnet_operator_benchmark_results.append(run_sorting_searching_operators_benchmarks(ctx=ctx, dtype=dtype)) + # Run all Array Rearrange operations benchmarks with default input values + mxnet_operator_benchmark_results.append(run_rearrange_operators_benchmarks(ctx=ctx, dtype=dtype)) + # ************************ MXNET NN OPERATOR BENCHMARKS **************************** # Run all basic NN operations benchmarks with default input values diff --git a/benchmark/opperf/rules/default_params.py b/benchmark/opperf/rules/default_params.py index 2c8f3d436e0d..caead8517465 100644 --- a/benchmark/opperf/rules/default_params.py +++ b/benchmark/opperf/rules/default_params.py @@ -62,6 +62,11 @@ # NOTE: Data used is DEFAULT_DATA DEFAULT_AXIS = [0] +# For rearrange operators +DEFAULT_DIM_1 = [[0, 1, 5]] +DEFAULT_DIM_2 = [[0, 1, 5]] +DEFAULT_BLOCK_SIZE = [[1, 2, 5]] + # Default Inputs. MXNet Op Param Name to Default Input mapping DEFAULTS_INPUTS = {"data": DEFAULT_DATA, "lhs": DEFAULT_LHS, @@ -81,7 +86,10 @@ "k_nd": DEFAULT_K_ND, "p_nd": DEFAULT_P_ND, "axis_shape": DEFAULT_AXIS_SHAPE, - "axis": DEFAULT_AXIS} + "axis": DEFAULT_AXIS, + "dim1": DEFAULT_DIM_1, + "dim2": DEFAULT_DIM_2, + "block_size": DEFAULT_BLOCK_SIZE} # These are names of MXNet operator parameters that is of type NDArray. # We maintain this list to automatically recognize these parameters are to be diff --git a/benchmark/opperf/utils/op_registry_utils.py b/benchmark/opperf/utils/op_registry_utils.py index 88ea7a16cfe6..e76a46765052 100644 --- a/benchmark/opperf/utils/op_registry_utils.py +++ b/benchmark/opperf/utils/op_registry_utils.py @@ -262,6 +262,26 @@ def get_all_sorting_searching_operators(): return sort_search_mx_operators +def get_all_rearrange_operators(): + """Gets all array rearrange operators registered with MXNet. + + Returns + ------- + {"operator_name": {"has_backward", "nd_op_handle", "params"}} + """ + rearrange_ops = ['transpose','swapaxes','flip','depth_to_space','space_to_depth'] + + # Get all mxnet operators + mx_operators = _get_all_mxnet_operators() + + # Filter for Array Rearrange operators + rearrange_mx_operators = {} + for op_name, op_params in mx_operators.items(): + if op_name in rearrange_ops and op_name not in unique_ops: + rearrange_mx_operators[op_name] = mx_operators[op_name] + return rearrange_mx_operators + + def get_operators_with_no_benchmark(operators_with_benchmark): """Gets all MXNet operators with not benchmark. From 91d0dcb92481755220c15a3ff55b8528da267ea3 Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Fri, 19 Jul 2019 16:03:23 -0700 Subject: [PATCH 2/5] Trigger notification From f20ae8a6b20a0871e485cb961c89f18977e570fe Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Mon, 22 Jul 2019 12:48:13 -0700 Subject: [PATCH 3/5] 4d tensor, param support --- benchmark/opperf/rules/default_params.py | 11 ++++++++--- benchmark/opperf/utils/op_registry_utils.py | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/benchmark/opperf/rules/default_params.py b/benchmark/opperf/rules/default_params.py index caead8517465..b16cd5429f59 100644 --- a/benchmark/opperf/rules/default_params.py +++ b/benchmark/opperf/rules/default_params.py @@ -63,9 +63,13 @@ DEFAULT_AXIS = [0] # For rearrange operators -DEFAULT_DIM_1 = [[0, 1, 5]] -DEFAULT_DIM_2 = [[0, 1, 5]] -DEFAULT_BLOCK_SIZE = [[1, 2, 5]] +# NOTE: Data needs to be a 4D tensor for operators like space_to_depth and depth_to_space +# Hence below we append 4d to mark the difference. +# For depth_to_space, dimension 3 needs to be a multiple of 'block' and 1 should be a multiple of `block^2` +DEFAULT_DATA_4d = [(1, 4, 2, 4), (10,25,10,100)] +DEFAULT_DIM_1 = [0, 1, 2, 3] +DEFAULT_DIM_2 = [1, 2, 3, 0] +DEFAULT_BLOCK_SIZE = [2, 5] # Default Inputs. MXNet Op Param Name to Default Input mapping DEFAULTS_INPUTS = {"data": DEFAULT_DATA, @@ -87,6 +91,7 @@ "p_nd": DEFAULT_P_ND, "axis_shape": DEFAULT_AXIS_SHAPE, "axis": DEFAULT_AXIS, + "data_4d": DEFAULT_DATA_4d, "dim1": DEFAULT_DIM_1, "dim2": DEFAULT_DIM_2, "block_size": DEFAULT_BLOCK_SIZE} diff --git a/benchmark/opperf/utils/op_registry_utils.py b/benchmark/opperf/utils/op_registry_utils.py index e76a46765052..f5e75066cafc 100644 --- a/benchmark/opperf/utils/op_registry_utils.py +++ b/benchmark/opperf/utils/op_registry_utils.py @@ -115,6 +115,8 @@ def prepare_op_inputs(arg_params): arg_params["params"]["arg_types"]): if "NDArray" in arg_type and arg_name + "_nd" in DEFAULTS_INPUTS: arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + "_nd"] + elif "NDArray" in arg_type and arg_name + "_4d" in DEFAULTS_INPUTS: + arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + "_4d"] elif arg_name in DEFAULTS_INPUTS: arg_values[arg_name] = DEFAULTS_INPUTS[arg_name] elif "float" in arg_type and arg_name + "_float" in DEFAULTS_INPUTS: From 2b9ae09a521eeb778fa1383b007bd812563ca8f5 Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Mon, 22 Jul 2019 14:07:06 -0700 Subject: [PATCH 4/5] new line --- benchmark/opperf/nd_operations/array_rearrange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/opperf/nd_operations/array_rearrange.py b/benchmark/opperf/nd_operations/array_rearrange.py index e474f8852a3b..151127c7b49a 100644 --- a/benchmark/opperf/nd_operations/array_rearrange.py +++ b/benchmark/opperf/nd_operations/array_rearrange.py @@ -54,4 +54,4 @@ def run_rearrange_operators_benchmarks(ctx=mx.cpu(), dtype='float32', warmup=25, # Run benchmarks mx_rearrange_op_results = run_op_benchmarks(mx_rearrange_ops, dtype, ctx, warmup, runs) - return mx_rearrange_op_results \ No newline at end of file + return mx_rearrange_op_results From 406896200f04c29f3a10395c9ac8a9ad1165d90b Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Wed, 24 Jul 2019 18:48:14 -0700 Subject: [PATCH 5/5] add alias logic --- benchmark/opperf/utils/profiler_utils.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/benchmark/opperf/utils/profiler_utils.py b/benchmark/opperf/utils/profiler_utils.py index a434d3be1e5c..0df67a00ff26 100644 --- a/benchmark/opperf/utils/profiler_utils.py +++ b/benchmark/opperf/utils/profiler_utils.py @@ -42,14 +42,28 @@ def _get_memory_profile(memory_profile_results): def _get_operator_profile(operator_name, operator_profile_results): operator_profile = {} + + # alias map : dictionary of the form {"alias" : "registered_name"} + # allows to retrieve alias operator profile from the profiler results + # TODO handling - "identity" : "_copy" + alias_map = {"broadcast_plus" : "broadcast_add", "broadcast_minus" : "broadcast_sub", "flatten" : "Flatten", "max_axis" : "max", + "swapaxes" : "SwapAxis", "flip" : "reverse", "reshape" : "Reshape", "crop" : "slice", "sum_axis" : "sum", "min_axis" : "min"} + + op_name = None + + if operator_name in alias_map: + op_name = alias_map[operator_name] + else: + op_name = operator_name + for line in operator_profile_results: - if operator_name in line or operator_name[:3] + " " in line: + if op_name in line or op_name[:3] + " " in line: operation = line.split()[0] operation_avg_time = float(line.split()[-1]) if "_backward" in operation: - operator_profile["avg_time" + operation] = operation_avg_time + operator_profile["avg_time_backward_" + operator_name] = operation_avg_time else: - operator_profile["avg_time_forward_" + operation] = operation_avg_time + operator_profile["avg_time_forward_" + operator_name] = operation_avg_time return operator_profile