Merge pull request #2 from itikhono/itikhono/tf_fe/master

Resolve review comments

Author: itikhono

docs/template_plugin/src/template_executable_network.cpp

@@ -8,11 +8,16 @@
 #include <ie_plugin_config.hpp>
 #include <threading/ie_executor_manager.hpp>
 
+#include "cpp/ie_cnn_network.h"
+#include "ie_icnn_network.hpp"
 #include "ie_icore.hpp"
+#include "ie_ngraph_utils.hpp"
+#include "openvino/core/except.hpp"
 #include "template/template_config.hpp"
 #include "template_itt.hpp"
 #include "template_plugin.hpp"
 #include "transformations/serialize.hpp"
+#include "transformations/utils/utils.hpp"
 
 using namespace TemplatePlugin;
 
@@ -66,18 +71,19 @@ TemplatePlugin::ExecutableNetwork::ExecutableNetwork(std::istream& model,
         model.read(dataBlob->buffer(), dataSize);
     }
 
+    auto cnnnetwork = _plugin->GetCore()->ReadNetwork(xmlString, std::move(dataBlob));
+
     // TODO: implement Import / Export of configuration options and merge with `cfg`
     // TODO: implement Import / Export of network precisions, layouts, preprocessing info
-    InferenceEngine::InputsDataMap inputInfoMap;
-    InferenceEngine::OutputsDataMap outputInfoMap;
+    InferenceEngine::InputsDataMap inputInfoMap = cnnnetwork.getInputsInfo();
+    InferenceEngine::OutputsDataMap outputInfoMap = cnnnetwork.getOutputsInfo();
 
-    auto cnnnetwork = _plugin->GetCore()->ReadNetwork(xmlString, std::move(dataBlob));
-
-    setNetworkInputs(cnnnetwork.getInputsInfo());
-    setNetworkOutputs(cnnnetwork.getOutputsInfo());
+    setNetworkInputs(inputInfoMap);
+    setNetworkOutputs(outputInfoMap);
     SetPointerToPlugin(_plugin->shared_from_this());
 
     try {
+        // TODO: remove compilation, network is already compiled and serialized in compiled form
         CompileNetwork(cnnnetwork.getFunction(), inputInfoMap, outputInfoMap);
         InitExecutor();  // creates thread-based executor using for async requests
     } catch (const InferenceEngine::Exception&) {
@@ -107,11 +113,7 @@ void TemplatePlugin::ExecutableNetwork::CompileNetwork(const std::shared_ptr<con
     // Generate backend specific blob mappings. For example Inference Engine uses not ngraph::Result nodes friendly name
     // as inference request output names but the name of the layer before.
     for (auto&& result : _function->get_results()) {
-        auto previousOutput = result->get_input_source_output(0);
-        auto outputName = previousOutput.get_node()->get_friendly_name();
-        if (previousOutput.get_node()->get_output_size() > 1) {
-            outputName += '.' + std::to_string(previousOutput.get_index());
-        }
+        auto outputName = ngraph::op::util::create_ie_output_name(result->input_value(0));
         _outputIndex.emplace(outputName, _function->get_result_index(result));
     }
     for (auto&& parameter : _function->get_parameters()) {

docs/template_plugin/src/template_executable_network.hpp

@@ -43,6 +43,7 @@ class ExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafeDef
 
 private:
     friend class TemplateInferRequest;
+    friend class Plugin;
 
     void CompileNetwork(const std::shared_ptr<const ngraph::Function>& function,
                         const InferenceEngine::InputsDataMap& inputInfoMap,

docs/template_plugin/src/template_plugin.cpp

@@ -17,6 +17,7 @@
 #include <transformations/rt_info/fused_names_attribute.hpp>
 #include <transformations/convert_precision.hpp>
 
+#include "openvino/core/except.hpp"
 #include "template/template_config.hpp"
 #include "template_itt.hpp"
 #include "template_plugin.hpp"
@@ -68,7 +69,7 @@ std::shared_ptr<ngraph::Function> TransformNetwork(const std::shared_ptr<const n
     // TODO: add post-processing based on outputsInfoMap
     // Example: register CommonOptimizations transformation from transformations library
     passManager.register_pass<ngraph::pass::CommonOptimizations>();
-    // GAPI supports only FP32 networks for pre-processing
+    // G-API supports only FP32 networks for pre-processing
     bool needF16toF32 = false;
     for (const auto& param : function->get_parameters()) {
         if (param->get_element_type() == ngraph::element::f16 &&
@@ -78,9 +79,10 @@ std::shared_ptr<ngraph::Function> TransformNetwork(const std::shared_ptr<const n
             break;
         }
     }
-    if (needF16toF32)
+    if (needF16toF32) {
         passManager.register_pass<ngraph::pass::ConvertPrecision>(
             precisions_array{{ngraph::element::f16, ngraph::element::f32}});
+    }
     // Example: register plugin specific transformation
     passManager.register_pass<ngraph::pass::DecomposeDivideMatcher>();
     passManager.register_pass<ngraph::pass::ReluReluFusionMatcher>();
@@ -119,9 +121,11 @@ InferenceEngine::IExecutableNetworkInternal::Ptr Plugin::ImportNetwork(
     OV_ITT_SCOPED_TASK(itt::domains::TemplatePlugin, "Plugin::ImportNetwork");
 
     auto fullConfig = Configuration{config, _cfg};
-    return std::make_shared<ExecutableNetwork>(modelStream,
-                                               fullConfig,
-                                               std::static_pointer_cast<Plugin>(shared_from_this()));
+    auto exec = std::make_shared<ExecutableNetwork>(modelStream,
+                                                    fullConfig,
+                                                    std::static_pointer_cast<Plugin>(shared_from_this()));
+    SetExeNetworkInfo(exec, exec->_function);
+    return exec;
 }
 // ! [plugin:import_network]
 

docs/template_plugin/tests/functional/op_reference/base_reference_test.cpp

@@ -63,7 +63,7 @@ void CommonReferenceTest::Infer() {
 }
 
 void CommonReferenceTest::Validate() {
-    ASSERT_EQ(executableNetwork.get_results().size(), refOutData.size());
+    ASSERT_EQ(executableNetwork.outputs().size(), refOutData.size());
     std::vector<ov::runtime::Tensor> outputs;
     for (const auto& result : function->get_results()) {
         auto name = ngraph::op::util::create_ie_output_name(result->input_value(0));

docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_exec_network.cpp

@@ -3,6 +3,7 @@
 //
 
 #include "behavior/ov_exec_network.hpp"
+#include <common_test_utils/test_constants.hpp>
 
 #include "ie_plugin_config.hpp"
 
@@ -27,6 +28,9 @@ const std::vector<std::map<std::string, std::string>> configs = {
 const std::vector<std::map<std::string, std::string>> multiConfigs = {
     {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_TEMPLATE}}};
 
+const std::vector<std::map<std::string, std::string>> heteroConfigs = {
+    {{"TARGET_FALLBACK", CommonTestUtils::DEVICE_TEMPLATE}}};
+
 INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
                          OVExecNetwork,
                          ::testing::Combine(::testing::ValuesIn(netPrecisions),
@@ -47,4 +51,12 @@ INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests,
                                             ::testing::Values(CommonTestUtils::DEVICE_AUTO),
                                             ::testing::ValuesIn(multiConfigs)),
                          OVExecNetwork::getTestCaseName);
+
+INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests,
+                         OVExecNetwork,
+                         ::testing::Combine(::testing::ValuesIn(netPrecisions),
+                                            ::testing::Values(CommonTestUtils::DEVICE_HETERO),
+                                            ::testing::ValuesIn(heteroConfigs)),
+                         OVExecNetwork::getTestCaseName);
+
 }  // namespace

docs/template_plugin/tests/functional/skip_tests_config.cpp

@@ -19,5 +19,15 @@ std::vector<std::string> disabledTestPatterns() {
         // CVS-51758
         R"(.*InferRequestPreprocessConversionTest.*oLT=(NHWC|NCHW).*)",
         R"(.*InferRequestPreprocessDynamicallyInSetBlobTest.*oPRC=0.*oLT=1.*)",
+
+        // TODO: execution graph is not supported
+        R"(.*ExecGraph.*)",
+
+        // TODO: support import / export of precisions in template plugin
+        R"(.*smoke_Hetero_BehaviorTests.*OVExecNetwork.ieImportExportedFunction.*)",
+        R"(.*smoke_BehaviorTests.*OVExecNetwork.ieImportExportedFunction.*)",
+
+        // TODO: Round with f16 is not supported
+        R"(.*smoke_Hetero_BehaviorTests.*OVExecNetwork.*readFromV10IR.*)",
     };
 }

inference-engine/ie_bridges/python/wheel/meta/openvino-dev.requirements.txt

@@ -11,7 +11,7 @@ texttable~=1.6.3
 py-cpuinfo>=7.0.0
 PyYAML>=5.4.1
 pillow>=8.1.2
-scikit-image~=0.17.2
+scikit-image>=0.17.2
 scikit-learn>=0.24.1
 yamlloader>=0.5
 shapely>=1.7.1

inference-engine/samples/benchmark_app/benchmark_app.hpp

@@ -337,6 +337,7 @@ static void showUsage() {
     std::cout << "    -l \"<absolute_path>\"      " << custom_cpu_library_message << std::endl;
     std::cout << "          Or" << std::endl;
     std::cout << "    -c \"<absolute_path>\"      " << custom_cldnn_message << std::endl;
+    std::cout << "    -hint \"performance hint (latency or throughput)\"   " << hint_message << std::endl;
     std::cout << "    -api \"<sync/async>\"       " << api_message << std::endl;
     std::cout << "    -niter \"<integer>\"        " << iterations_count_message << std::endl;
     std::cout << "    -nireq \"<integer>\"        " << infer_requests_count_message << std::endl;

inference-engine/src/gna_plugin/gna2_model_debug_log.cpp

@@ -304,11 +304,11 @@ static std::vector<std::string> GetParamaterNames(Gna2OperationType type) {
         operationParamaterNamesMap.find(type)->second : std::vector<std::string> {};
 }
 
-typedef void (*dumpParamaters) (std::ostream&, void**, size_t, const std::vector<std::string>);
+typedef void (*dumpParameters) (std::ostream&, void**, size_t, const std::vector<std::string>);
 
-static dumpParamaters GetParamDumpFunc(Gna2OperationType type) {
+static dumpParameters GetParamDumpFunc(Gna2OperationType type) {
     // This map must be aligned with operationParamaterNamesMap in this file
-    static const std::map<Gna2OperationType, dumpParamaters> dumpParamMap = {
+    static const std::map<Gna2OperationType, dumpParameters> dumpParamMap = {
         {Gna2OperationTypeConvolution, DumpConvolutionParameters},
         {Gna2OperationTypeCopy, DumpCopyParameters},
         {Gna2OperationTypeFullyConnectedAffine, DumpFCAffineParameters},

inference-engine/src/hetero_plugin/hetero_executable_network.cpp

@@ -6,6 +6,14 @@
 #include "hetero_executable_network.hpp"
 #include "hetero_async_infer_request.hpp"
 #include "hetero_itt.hpp"
+#include "ie_precision.hpp"
+#include "openvino/core/dimension.hpp"
+#include "openvino/core/except.hpp"
+#include "openvino/core/type.hpp"
+#include "openvino/core/type/element_type.hpp"
+#include "openvino/op/result.hpp"
+#include "transformations/utils/utils.hpp"
+#include "openvino/op/parameter.hpp"
 #include "xml_parse_utils.h"
 #include <caseless.hpp>
 
@@ -63,6 +71,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::CNNNetwo
 #ifndef NDEBUG
     dumpDotFile  = true;
 #endif
+
     QueryNetworkResult queryNetworkResult;
     auto orderedOps = clonedFunction->get_ordered_ops();
     bool allEmpty = true;
@@ -548,6 +557,44 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream&
         });
     }
 
+    const auto parseNode = [] (const pugi::xml_node & xml_node, bool is_param) ->
+        std::shared_ptr<const ov::Node> {
+        const std::string operation_name = GetStrAttr(xml_node, "operation_name");
+        const auto elementType =
+            ov::EnumNames<ov::element::Type_t>::as_enum(GetStrAttr(xml_node, "element_type"));
+
+        std::vector<ov::Dimension> partialShape;
+        pugi::xml_node partialShapeNode = xml_node.child("partial_shape");
+        FOREACH_CHILD(dimNode, partialShapeNode, "dim") {
+            partialShape.emplace_back(ov::Dimension(GetInt64Attr(dimNode, "value")));
+        }
+
+        pugi::xml_node tensorNamesNode = xml_node.child("tensor_names");
+        std::unordered_set<std::string> tensorNames;
+        FOREACH_CHILD(tensorNameNode, tensorNamesNode, "tensor_name") {
+            tensorNames.insert(GetStrAttr(tensorNameNode, "value"));
+        }
+
+        std::shared_ptr<ov::Node> node = std::make_shared<ov::op::v0::Parameter>(elementType, partialShape);
+        if (!is_param)
+            node = std::make_shared<ov::op::v0::Result>(node);
+        node->set_friendly_name(operation_name);
+        node->output(0).get_tensor().add_names(tensorNames);
+
+        return node;
+    };
+    (void)parseNode;
+
+    pugi::xml_node parametersNode = heteroNode.child("parameters");
+    FOREACH_CHILD(parameterNode, parametersNode, "parameter") {
+        _parameters.emplace_back(parseNode(parameterNode, true));
+    }
+
+    pugi::xml_node resultsNode = heteroNode.child("results");
+    FOREACH_CHILD(resultNode, resultsNode, "result") {
+        _results.emplace_back(parseNode(resultNode, false));
+    }
+
     // save state
     this->_config = importedConfigs;
     this->_networks = std::move(descs);
@@ -559,6 +606,8 @@ void HeteroExecutableNetwork::Export(std::ostream& heteroModel) {
     auto heteroNode = doc.append_child("hetero");
     heteroNode.append_attribute("name").set_value(_name.c_str());
 
+    // CNNNetwork inputs and outputs information
+
     auto inputsNode = heteroNode.append_child("inputs");
     for (auto&& networkInput : _networkInputs) {
         inputsNode.append_child("input").append_attribute("name").set_value(networkInput.first.c_str());
@@ -569,6 +618,46 @@ void HeteroExecutableNetwork::Export(std::ostream& heteroModel) {
         outputsNode.append_child("output").append_attribute("name").set_value(networkInput.first.c_str());
     }
 
+    const auto serializeNode = [&] (const std::shared_ptr<const ov::Node>& node,
+        pugi::xml_node & xml_node) {
+        const bool is_result = ov::is_type<ov::op::v0::Result>(node);
+        const std::string name = is_result ?
+            ngraph::op::util::create_ie_output_name(node->input_value(0)) :
+            node->get_friendly_name();
+        xml_node.append_attribute("operation_name").set_value(name.c_str());
+        xml_node.append_attribute("element_type").set_value(
+            node->get_output_element_type(0).get_type_name().c_str());
+
+        const auto & pShape = node->get_output_partial_shape(0);
+        OPENVINO_ASSERT(pShape.rank().is_static(), "Serialization of shapes with dynamic rank is not supported");
+        auto partialShapeNode = xml_node.append_child("partial_shape");
+        for (auto && dim : node->get_output_partial_shape(0)) {
+            if (dim.is_dynamic())
+                partialShapeNode.append_child("dim").append_attribute("value").set_value("-1");
+            else
+                partialShapeNode.append_child("dim").append_attribute("value").set_value(std::to_string(dim.get_length()).c_str());
+        }
+
+        auto tensorNamesNode = xml_node.append_child("tensor_names");
+        for (auto & tensorName : node->get_output_tensor(0).get_names()) {
+            tensorNamesNode.append_child("tensor_name").append_attribute("value").set_value(tensorName.c_str());
+        }
+    };
+
+    // ngraph parameters info
+    auto subnetworkParamsNode = heteroNode.append_child("parameters");
+    for (auto&& parameter : getInputs()) {
+        auto parameterNode = subnetworkParamsNode.append_child("parameter");
+        serializeNode(parameter, parameterNode);
+    }
+
+    // ngraph results info
+    auto subnetworkResultsNode = heteroNode.append_child("results");
+    for (auto&& result : getOutputs()) {
+        auto parameterNode = subnetworkResultsNode.append_child("result");
+        serializeNode(result, parameterNode);
+    }
+
     auto subnetworksNode = heteroNode.append_child("subnetworks");
     for (auto&& subnetwork : _networks) {
         auto subnet = subnetwork._clonedNetwork;

inference-engine/src/hetero_plugin/hetero_plugin.cpp

@@ -42,8 +42,8 @@ std::vector<std::string> supported_configKeys {
 
 }  // namespace
 
-InferenceEngine::IExecutableNetworkInternal::Ptr Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork&    network,
-                                                                           const Configs&                   config) {
+InferenceEngine::IExecutableNetworkInternal::Ptr Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork& network,
+                                                                            const Configs&                     config) {
     if (GetCore() == nullptr) {
         IE_THROW() << "Please, work with HETERO device via InferencEngine::Core object";
     }

inference-engine/src/inference_engine/include/openvino/runtime/executable_network.hpp

@@ -60,18 +60,58 @@ class OPENVINO_RUNTIME_API ExecutableNetwork {
     std::shared_ptr<const Function> get_runtime_function() const;
 
     /**
-     * @brief Get parameters of executeble graph function
+     * @brief Get inputs of executable graph function
      *
-     * @return vector of paramter nodes
+     * @return vector of inputs
      */
-    ParameterVector get_parameters() const;
+    std::vector<ov::Output<const ov::Node>> inputs() const;
+    /**
+     * @brief Get input of executable graph function
+     *
+     * @return Function input or throw ov::Exception in case of several outputs
+     */
+    ov::Output<const ov::Node> input() const;
+    /**
+     * @brief Get input of executable graph function
+     *
+     * @param i input index
+     * @return Function input or throw ov::Exception if input wasn't found
+     */
+    ov::Output<const ov::Node> input(size_t i) const;
+    /**
+     * @brief Get input of executable graph function
+     *
+     * @param tensor_name The input tensor name
+     * @return Function output or throw ov::Exception if input wasn't found
+     */
+    ov::Output<const ov::Node> input(const std::string& tensor_name) const;
 
     /**
-     * @brief Get results of executeble graph function
+     * @brief Get outputs of executable graph function
+     *
+     * @return vector of outputs
+     */
+    std::vector<ov::Output<const ov::Node>> outputs() const;
+    /**
+     * @brief Get output of executable graph function
+     *
+     * @return Function output or throw ov::Exception in case of several outputs
+     */
+    ov::Output<const ov::Node> output() const;
+    /**
+     * @brief Get output of executable graph function
+     *
+     * @param i output index
+     * @return Function output or throw ov::Exception if output wasn't found
+     */
+    ov::Output<const ov::Node> output(size_t i) const;
+    /**
+     * @brief Get output of executable graph function
      *
-     * @return vector of result nodes
+     * @param tensor_name The output tensor name
+     * @return Function output or throw ov::Exception if output wasn't found
      */
-    ResultVector get_results() const;
+    ov::Output<const ov::Node> output(const std::string& tensor_name) const;
 
     /**
      * @brief Creates an inference request object used to infer the network.