From 5c7ea488d3a3cfc5aac77918a613e794cf10f60f Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Tue, 4 Feb 2020 17:41:11 -0800 Subject: [PATCH 1/6] Initial commit - implemented linalg_potrf --- .../opperf/nd_operations/linalg_operators.py | 78 +++++++++++++++++++ benchmark/opperf/opperf.py | 4 + benchmark/opperf/utils/op_registry_utils.py | 20 +++++ 3 files changed, 102 insertions(+) create mode 100644 benchmark/opperf/nd_operations/linalg_operators.py diff --git a/benchmark/opperf/nd_operations/linalg_operators.py b/benchmark/opperf/nd_operations/linalg_operators.py new file mode 100644 index 000000000000..dd026538f041 --- /dev/null +++ b/benchmark/opperf/nd_operations/linalg_operators.py @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Performance benchmark tests for MXNet NDArray Linear Algebra Operations. + +Below 18 Linear Algebra Operators are covered: + +['linalg_potri', 'linalg_gemm2', 'linalg_extractdiag', 'linalg_trsm', 'linalg_gelqf', 'linalg_gemm', 'linalg_sumlogdiag', +'linalg_potrf', 'linalg_makediag', 'linalg_syrk', 'linalg_maketrian', 'linalg_trmm', 'linalg_extracttrian', 'lingalg_makediag', +'linalg_slogdet', 'linalg_det', 'linalg_inverse', 'moments'] + +""" + +import mxnet as mx + +from benchmark.opperf.utils.benchmark_utils import run_op_benchmarks +from benchmark.opperf.utils.op_registry_utils import get_all_linalg_operators + +from benchmark.opperf.utils.benchmark_utils import run_performance_test +from benchmark.opperf.utils.common_utils import merge_map_list +from benchmark.opperf.rules.default_params import MX_OP_MODULE + +def run_linalg_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', warmup=25, runs=100): + """Runs benchmarks with the given context and precision (dtype) for all the linear algebra + operators in MXNet. + + Parameters + ---------- + ctx: mx.ctx + Context to run benchmarks + dtype: str, default 'float32' + Precision to use for benchmarks + profiler: str, default 'native' + Type of Profiler to use (native/python) + warmup: int, default 25 + Number of times to run for warmup + runs: int, default 100 + Number of runs to capture benchmark results + + Returns + ------- + Dictionary of results. Key -> Name of the operator, Value -> Benchmark results. + + """ + # Individual tests for ops with specific requirements on input data + # linalg_potrf requires a positive definite as input + linalg_potrf_benchmark = run_performance_test(getattr(MX_OP_MODULE, "linalg_potrf"), + run_backward=False, + dtype=dtype, + ctx=ctx, + profiler=profiler, + inputs=[{"A": mx.nd.array([[1, 0], + [0, 1]])}, + {"A": mx.nd.array([[2, -1, 0], + [-1, 2, -1], + [0, -1, 2]])}], + warmup=warmup, + runs=runs) + + # Fetch all Linear Algebra Operators + mx_linalg_ops = get_all_linalg_operators() + # Run benchmarks + mx_linalg_op_results = run_op_benchmarks(mx_linalg_ops, dtype, ctx, profiler, warmup, runs) + return merge_map_list(linalg_potrf_benchmark + [mx_linalg_op_results]) diff --git a/benchmark/opperf/opperf.py b/benchmark/opperf/opperf.py index fc6964a6b478..5f4c8ee9cf0e 100755 --- a/benchmark/opperf/opperf.py +++ b/benchmark/opperf/opperf.py @@ -43,6 +43,7 @@ from benchmark.opperf.nd_operations.array_rearrange import run_rearrange_operators_benchmarks from benchmark.opperf.nd_operations.indexing_routines import run_indexing_routines_benchmarks from benchmark.opperf.nd_operations.nn_loss_operators import run_loss_operators_benchmarks +from benchmark.opperf.nd_operations.linalg_operators import run_linalg_operators_benchmarks from benchmark.opperf.utils.common_utils import merge_map_list, save_to_file from benchmark.opperf.utils.op_registry_utils import get_operators_with_no_benchmark, \ @@ -114,6 +115,9 @@ def run_all_mxnet_operator_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='n # Run all NN loss operations benchmarks with default input values mxnet_operator_benchmark_results.append(run_loss_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler)) + # Run all Linear Algebra operations benchmarks with default input values + mxnet_operator_benchmark_results.append(run_linalg_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler)) + # ****************************** PREPARE FINAL RESULTS ******************************** final_benchmark_result_map = merge_map_list(mxnet_operator_benchmark_results) return final_benchmark_result_map diff --git a/benchmark/opperf/utils/op_registry_utils.py b/benchmark/opperf/utils/op_registry_utils.py index 515a51f1f22d..22132e93035b 100644 --- a/benchmark/opperf/utils/op_registry_utils.py +++ b/benchmark/opperf/utils/op_registry_utils.py @@ -267,6 +267,26 @@ def get_all_random_sampling_operators(): return random_sampling_mx_operators +def get_all_linalg_operators(): + """Gets all Linear Algebra operators registered with MXNet. + + Returns + ------- + {"operator_name": {"has_backward", "nd_op_handle", "params"}} + """ + other_linalg_ops = ['moments'] + + # Get all mxnet operators + mx_operators = _get_all_mxnet_operators() + + # Filter for Linear Algebra operators + linalg_mx_operators = {} + for op_name, _ in mx_operators.items(): + if op_name.startswith("linalg_") or op_name in other_linalg_ops: + linalg_mx_operators[op_name] = mx_operators[op_name] + return linalg_mx_operators + + def get_all_reduction_operators(): """Gets all Reduction operators registered with MXNet. From a40b6c2e36b70d46fbb00bf44678bab5033054da Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Wed, 5 Feb 2020 10:10:26 -0800 Subject: [PATCH 2/6] Implemented standardized linalg ops, added relevant data --- .../opperf/nd_operations/linalg_operators.py | 2 +- benchmark/opperf/rules/default_params.py | 17 +++++++++++++++-- benchmark/opperf/utils/benchmark_utils.py | 2 +- benchmark/opperf/utils/op_registry_utils.py | 5 ++++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/benchmark/opperf/nd_operations/linalg_operators.py b/benchmark/opperf/nd_operations/linalg_operators.py index dd026538f041..e3222eca9a8c 100644 --- a/benchmark/opperf/nd_operations/linalg_operators.py +++ b/benchmark/opperf/nd_operations/linalg_operators.py @@ -57,7 +57,7 @@ def run_linalg_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='nat """ # Individual tests for ops with specific requirements on input data - # linalg_potrf requires a positive definite as input + # linalg_potrf requires a positive definite matrix as input linalg_potrf_benchmark = run_performance_test(getattr(MX_OP_MODULE, "linalg_potrf"), run_backward=False, dtype=dtype, diff --git a/benchmark/opperf/rules/default_params.py b/benchmark/opperf/rules/default_params.py index 19caf9e5d9f2..596dceb16480 100644 --- a/benchmark/opperf/rules/default_params.py +++ b/benchmark/opperf/rules/default_params.py @@ -135,6 +135,13 @@ DEFAULT_DATA_SMCE = [(1024, 1024)] DEFAULT_LABEL_SMCE = [(1024,)] +# For linalg operators +DEFAULT_A = [(1024, 1024)] +DEFAULT_B = [(1024, 1024)] +DEFAULT_C = [(1024, 1024)] +DEFAULT_A_MT = [(1024, 1035)] +DEFAULT_AXES = [[0, 1]] + # Default Inputs. MXNet Op Param Name to Default Input mapping DEFAULTS_INPUTS = {"data": DEFAULT_DATA, "sample": DEFAULT_SAMPLE, @@ -206,7 +213,12 @@ "transform_type": DEFAULT_TRANSFORM_TYPE, "data_gridgenerator": DEFAULT_DATA_GRIDGEN, "target_shape_gridgenerator": DEFAULT_TARGET_SHAPE, - "data_sample_multinomial": DEFAULT_DATA_SM} + "data_sample_multinomial": DEFAULT_DATA_SM, + "A": DEFAULT_A, + "B": DEFAULT_B, + "C": DEFAULT_C, + "A_linalg_maketrian": DEFAULT_A_MT, + "axes": DEFAULT_AXES} # These are names of MXNet operator parameters that is of type NDArray. @@ -219,4 +231,5 @@ "low", "high", "weight", "bias", "moving_mean", "moving_var", "weight", "weight32", "grad", "mean", "var", "mom", "n", "d", "v", "z", "g", "delta", "args", "indices", "shape_like", "y", - "x", "condition", "a", "index", "raveL_data", "label", "grid"] + "x", "condition", "a", "index", "raveL_data", "label", "grid", + "A", "B", "C"] diff --git a/benchmark/opperf/utils/benchmark_utils.py b/benchmark/opperf/utils/benchmark_utils.py index 4165e71d16c1..421fe314267e 100644 --- a/benchmark/opperf/utils/benchmark_utils.py +++ b/benchmark/opperf/utils/benchmark_utils.py @@ -26,7 +26,7 @@ from benchmark.opperf.rules.default_params import PARAMS_OF_TYPE_NDARRAY from .profiler_utils import cpp_profile, python_profile -no_backward = ['gather_nd', 'softmax_cross_entropy'] +no_backward = ['gather_nd', 'softmax_cross_entropy', 'linalg_gelqf', 'linalg_slogdet', 'moments'] def _prepare_op_inputs(inputs, run_backward, dtype, ctx): mx.random.seed(41) diff --git a/benchmark/opperf/utils/op_registry_utils.py b/benchmark/opperf/utils/op_registry_utils.py index 22132e93035b..45d8f78801ae 100644 --- a/benchmark/opperf/utils/op_registry_utils.py +++ b/benchmark/opperf/utils/op_registry_utils.py @@ -119,6 +119,8 @@ def prepare_op_inputs(op, arg_params): ops_3d = ['CTCLoss', 'ctc_loss'] custom_data = ['BilinearSampler', 'GridGenerator', 'sample_multinomial'] + custom_data = ['linalg_maketrian'] + # Prepare op to default input mapping arg_values = {} for arg_name, arg_type in zip(arg_params["params"]["arg_names"], @@ -282,7 +284,8 @@ def get_all_linalg_operators(): # Filter for Linear Algebra operators linalg_mx_operators = {} for op_name, _ in mx_operators.items(): - if op_name.startswith("linalg_") or op_name in other_linalg_ops: + # Already tested linalg_potrf independently + if (op_name.startswith("linalg_") and op_name != 'linalg_potrf') or op_name in other_linalg_ops: linalg_mx_operators[op_name] = mx_operators[op_name] return linalg_mx_operators From df750e1b423c015e3a60681665c85d78550f2857 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Wed, 5 Feb 2020 10:20:26 -0800 Subject: [PATCH 3/6] Added independently_tested category --- benchmark/opperf/utils/op_registry_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark/opperf/utils/op_registry_utils.py b/benchmark/opperf/utils/op_registry_utils.py index 45d8f78801ae..99f981277adb 100644 --- a/benchmark/opperf/utils/op_registry_utils.py +++ b/benchmark/opperf/utils/op_registry_utils.py @@ -278,14 +278,16 @@ def get_all_linalg_operators(): """ other_linalg_ops = ['moments'] + # Already tested linalg_potrf independently + independently_tested = ['linalg_potrf'] + # Get all mxnet operators mx_operators = _get_all_mxnet_operators() # Filter for Linear Algebra operators linalg_mx_operators = {} for op_name, _ in mx_operators.items(): - # Already tested linalg_potrf independently - if (op_name.startswith("linalg_") and op_name != 'linalg_potrf') or op_name in other_linalg_ops: + if (op_name.startswith("linalg_") and op_name not in independently_tested) or op_name in other_linalg_ops: linalg_mx_operators[op_name] = mx_operators[op_name] return linalg_mx_operators From 4a2bd225e38c205435e7d47da26f6c737e628315 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Wed, 5 Feb 2020 10:29:51 -0800 Subject: [PATCH 4/6] Dropped unnecessary NDArray cast for potrf --- benchmark/opperf/nd_operations/linalg_operators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmark/opperf/nd_operations/linalg_operators.py b/benchmark/opperf/nd_operations/linalg_operators.py index e3222eca9a8c..c16f3433d621 100644 --- a/benchmark/opperf/nd_operations/linalg_operators.py +++ b/benchmark/opperf/nd_operations/linalg_operators.py @@ -63,11 +63,11 @@ def run_linalg_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='nat dtype=dtype, ctx=ctx, profiler=profiler, - inputs=[{"A": mx.nd.array([[1, 0], - [0, 1]])}, - {"A": mx.nd.array([[2, -1, 0], - [-1, 2, -1], - [0, -1, 2]])}], + inputs=[{"A": [[1, 0], + [0, 1]]}, + {"A": [[2, -1, 0], + [-1, 2, -1], + [0, -1, 2]]}], warmup=warmup, runs=runs) From 023b9d4c8767dde3a376743332f2f54ac00d4948 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Thu, 6 Feb 2020 10:49:14 -0800 Subject: [PATCH 5/6] Removed duplicate makediag, rebased --- benchmark/opperf/nd_operations/linalg_operators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/opperf/nd_operations/linalg_operators.py b/benchmark/opperf/nd_operations/linalg_operators.py index c16f3433d621..d2c1cee0a307 100644 --- a/benchmark/opperf/nd_operations/linalg_operators.py +++ b/benchmark/opperf/nd_operations/linalg_operators.py @@ -17,10 +17,10 @@ """Performance benchmark tests for MXNet NDArray Linear Algebra Operations. -Below 18 Linear Algebra Operators are covered: +Below 17 Linear Algebra Operators are covered: ['linalg_potri', 'linalg_gemm2', 'linalg_extractdiag', 'linalg_trsm', 'linalg_gelqf', 'linalg_gemm', 'linalg_sumlogdiag', -'linalg_potrf', 'linalg_makediag', 'linalg_syrk', 'linalg_maketrian', 'linalg_trmm', 'linalg_extracttrian', 'lingalg_makediag', +'linalg_potrf', 'linalg_makediag', 'linalg_syrk', 'linalg_maketrian', 'linalg_trmm', 'linalg_extracttrian', 'linalg_slogdet', 'linalg_det', 'linalg_inverse', 'moments'] """ From 4127bfc0efc8db1add5ee044ca3b3f478b987f57 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Thu, 6 Feb 2020 17:49:59 -0800 Subject: [PATCH 6/6] Updated group of custom_data ops --- benchmark/opperf/utils/op_registry_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmark/opperf/utils/op_registry_utils.py b/benchmark/opperf/utils/op_registry_utils.py index 99f981277adb..eb2adba05d18 100644 --- a/benchmark/opperf/utils/op_registry_utils.py +++ b/benchmark/opperf/utils/op_registry_utils.py @@ -117,9 +117,8 @@ def prepare_op_inputs(op, arg_params): # 3d tensor is needed by following ops ops_3d = ['CTCLoss', 'ctc_loss'] - custom_data = ['BilinearSampler', 'GridGenerator', 'sample_multinomial'] - - custom_data = ['linalg_maketrian'] + + custom_data = ['BilinearSampler', 'GridGenerator', 'sample_multinomial', 'linalg_maketrian'] # Prepare op to default input mapping arg_values = {}