Skip to content

Improve reshapeability of models with eltwise nodes influencing shapes #2767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Nov 16, 2020
Merged

Improve reshapeability of models with eltwise nodes influencing shapes #2767

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
dea118a
Fix ElementwiseInputReshape transformation
rkazants Oct 21, 2020
3103be6
Merge EltwiseInputNormalization and EltwiseInputReshape transformations
rkazants Oct 29, 2020
143d610
Remove Unsqueeze from Fused_op
rkazants Nov 3, 2020
8cdb85d
Fix code after code review #1
rkazants Nov 9, 2020
3f8d4bc
Fix code after review #2
rkazants Nov 5, 2020
1eaa548
Fix code review #4
rkazants Nov 9, 2020
5bb7b96
Perform full normalization based on shapes of all inputs to eltwise
rkazants Nov 12, 2020
f2df747
Refactor much to avoid old API and edges with unsqueeze_dims attribute
rkazants Nov 13, 2020
fd15383
Fix code after review
rkazants Nov 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions inference-engine/src/legacy_api/src/ie_cnn_layer_builder_ngraph.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,11 +651,11 @@ CNNLayer::Ptr NodeConverter<ngraph::op::Squeeze>::createLayer(const std::shared_
}

template <>
CNNLayer::Ptr NodeConverter<ngraph::op::Unsqueeze>::createLayer(const std::shared_ptr<ngraph::Node>& layer) const {
CNNLayer::Ptr NodeConverter<ngraph::op::v0::Unsqueeze>::createLayer(const std::shared_ptr<ngraph::Node>& layer) const {
LayerParams params = {layer->get_friendly_name(), "Unsqueeze",
details::convertPrecision(layer->get_output_element_type(0))};
auto res = std::make_shared<InferenceEngine::CNNLayer>(params);
auto castedLayer = ngraph::as_type_ptr<ngraph::op::Unsqueeze>(layer);
auto castedLayer = ngraph::as_type_ptr<ngraph::op::v0::Unsqueeze>(layer);
if (castedLayer == nullptr) THROW_IE_EXCEPTION << "Cannot get " << params.type << " layer " << params.name;

return res;
Expand Down
6 changes: 3 additions & 3 deletions ...ine/src/legacy_api/src/transformations/convert_opset1_to_legacy/convert_nms_to_nms_ie.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,22 @@ ngraph::pass::ConvertNMSToNMSIEMatcher::ConvertNMSToNMSIEMatcher() {
if (auto new_max_per_class_const = std::dynamic_pointer_cast<opset1::Constant>(new_max_per_class.get_node_shared_ptr())) {
new_max_per_class = opset1::Constant::create(element::i64, Shape{1}, new_max_per_class_const->cast_vector<int64_t>());
} else {
new_max_per_class = std::make_shared<ngraph::op::Unsqueeze>(
new_max_per_class = std::make_shared<ngraph::op::v0::Unsqueeze>(
nms->input_value(2),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_max_per_class.get_node_shared_ptr());
}
}
auto new_iou_threshold = nms->input_value(3);
if (iou_threshold_rank.get_length() == 0) {
new_iou_threshold = std::make_shared<ngraph::op::Unsqueeze>(
new_iou_threshold = std::make_shared<ngraph::op::v0::Unsqueeze>(
nms->input_value(3),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_iou_threshold.get_node_shared_ptr());
}
auto new_score_threshold = nms->input_value(4);
if (score_threshold_rank.get_length() == 0) {
new_score_threshold = std::make_shared<ngraph::op::Unsqueeze>(
new_score_threshold = std::make_shared<ngraph::op::v0::Unsqueeze>(
nms->input_value(4),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_score_threshold.get_node_shared_ptr());
Expand Down
2 changes: 1 addition & 1 deletion inference-engine/tests/functional/plugin/shared/src/subgraph_tests/matmul_squeeze_add.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ void MatmulSqueezeAddTest::SetUp() {
auto matmul_0 = std::make_shared<ngraph::op::MatMul>(params[0], constant_0, false, true);

auto constant_1 = std::make_shared<ngraph::op::Constant>(ngraph::element::Type_t::i64, ngraph::Shape{ 1 }, std::vector<size_t>{0});
auto unsqueeze_0 = std::make_shared<ngraph::op::Unsqueeze>(matmul_0, constant_1);
auto unsqueeze_0 = std::make_shared<ngraph::op::v0::Unsqueeze>(matmul_0, constant_1);

auto constant_2 = ngraph::builder::makeConstant<float>(ngPrc, { 1, inputShape[0], outputSize },
generateFloatNumbers(0, 1, inputShape[0] * outputSize), false);
Expand Down
8 changes: 4 additions & 4 deletions inference-engine/tests/functional/plugin/shared/src/subgraph_tests/memory_LSTMCell.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ namespace SubgraphTestsDefinitions {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

auto permute_in_params = std::make_shared<ngraph::opset1::Constant>(ngraph::element::i64, ngraph::Shape{3}, ngraph::Shape{{1, 0, 2}});
auto permute_in = std::make_shared<ngraph::opset1::Transpose>(unsqueeze_input, permute_in_params);
Expand All @@ -113,7 +113,7 @@ namespace SubgraphTestsDefinitions {
auto lstm = std::make_shared<ngraph::opset4::LSTMCell>(squeeze, H_t, C_t, weightsNode, reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
// body - outputs
auto H_o = lstm->output(0);
auto C_o = lstm->output(1);
Expand Down Expand Up @@ -171,7 +171,7 @@ namespace SubgraphTestsDefinitions {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

auto cell_memory_constant = ngraph::builder::makeConstant<float>(ngPrc, cell_memory_dims, cell_memory_init);

Expand All @@ -188,7 +188,7 @@ namespace SubgraphTestsDefinitions {
reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);

auto final_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));
Expand Down
12 changes: 6 additions & 6 deletions inference-engine/tests/functional/plugin/shared/src/subgraph_tests/multiple_LSTMCell.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ void MultipleLSTMCellTest::SetUp() {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

auto permute_in_params = std::make_shared<ngraph::opset1::Constant>(ngraph::element::i64, ngraph::Shape{3}, ngraph::Shape{{1, 0, 2}});
auto permute_in = std::make_shared<ngraph::opset1::Transpose>(unsqueeze_input, permute_in_params);
Expand All @@ -113,7 +113,7 @@ void MultipleLSTMCellTest::SetUp() {
auto lstm = std::make_shared<ngraph::opset4::LSTMCell>(squeeze, H_t, C_t, weightsNode, reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
// body - outputs
auto H_o = lstm->output(0);
auto C_o = lstm->output(1);
Expand Down Expand Up @@ -168,7 +168,7 @@ void MultipleLSTMCellTest::SetUp() {
auto lstm_2 = std::make_shared<ngraph::opset4::LSTMCell>(squeeze_2, H_t_2, C_t_2, weightsNode_2, reccurrenceWeightsNode_2, biasNode_2, hiddenSize);

auto unsqueeze_2_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_2 = std::make_shared<ngraph::op::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
auto unsqueeze_2 = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
// body - outputs
auto H_o_2 = lstm_2->output(0);
auto C_o_2 = lstm_2->output(1);
Expand Down Expand Up @@ -232,7 +232,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

// Body 1 - layers
auto cell_memory_constant = ngraph::builder::makeConstant<float>(ngPrc, cell_memory_dims, cell_memory_init);
Expand All @@ -249,7 +249,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);

auto first_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));
Expand All @@ -274,7 +274,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
reccurrenceWeightsNode_2, biasNode_2, hiddenSize);

auto unsqueeze_2_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_2 = std::make_shared<ngraph::op::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
auto unsqueeze_2 = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);

auto final_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));
Expand Down
2 changes: 1 addition & 1 deletion ...ce-engine/tests/ngraph_functions/src/low_precision_transformations/unsqueeze_function.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ std::shared_ptr<ngraph::Function> UnsqueezeFunction::getReference(

const std::shared_ptr<Node> dequantizationOpBefore = makeDequantization(input, dequantizationBefore);
const auto unsqueeze = std::make_shared<ngraph::op::TypeRelaxed<ngraph::opset1::Unsqueeze>>(
op::Unsqueeze(dequantizationOpBefore, std::make_shared<ngraph::opset1::Constant>(element::i64, Shape{ axes.size() }, axes)),
op::v0::Unsqueeze(dequantizationOpBefore, std::make_shared<ngraph::opset1::Constant>(element::i64, Shape{ axes.size() }, axes)),
precisionAfterOperation);
const std::shared_ptr<Node> dequantizationOpAfter = makeDequantization(unsqueeze, dequantizationAfter);
dequantizationOpAfter->set_friendly_name("output");
Expand Down
1 change: 0 additions & 1 deletion model-optimizer/automation/package_BOM.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,6 @@ extensions/middle/DeleteControlFlowEdges.py
extensions/middle/DeleteNotExecutable.py
extensions/middle/DilatedConvolution.py
extensions/middle/EltwiseChecker.py
extensions/middle/EltwiseInputNormalization.py
extensions/middle/EltwiseInputReshape.py
extensions/middle/FakeSplitOutputs.py
extensions/middle/FusedBatchNormNonConstant.py
Expand Down
48 changes: 0 additions & 48 deletions model-optimizer/extensions/middle/EltwiseInputNormalization.py
View file
Open in desktop

This file was deleted.

Loading