diff --git a/src/bindings/python/src/openvino/frontend/pytorch/__init__.py b/src/bindings/python/src/openvino/frontend/pytorch/__init__.py index 5697f862f76518..f571c31753a28d 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/__init__.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/__init__.py @@ -55,8 +55,8 @@ def __init__ (self, pt_module): def inputs (self): return [x.unique() for x in self.pt_module.inputs()] - def input (self, index): - return self.inputs()[index] # TODO: find specialized method + def input (self, index): # TODO: remove + return self.inputs()[index] # TODO: find specialized method def get_input_shape (self, index): input = self._raw_input(index) @@ -182,6 +182,9 @@ def get_subgraph_decoder (self, index): def get_op_type (self): return self.pt_module.kind() + def get_schema (self): + return self.pt_module.schema() + def outputs (self): return [x.unique() for x in self.pt_module.outputs()] @@ -228,7 +231,7 @@ def as_constant (self): if str(pt_value.type()) in ['torch.int32', 'int']: #print(f'Found int value= {pt_value}, type = {type(pt_value.toIValue())}, ivalue = {pt_value.toIValue()}') return op.Constant(OVType.i32, Shape([]), [pt_value.toIValue()]).outputs() - if str(pt_value.type()) in ['torch.FloatType', 'float']: + if str(pt_value.type()) in ['torch.float', 'torch.FloatType', 'float']: #print(f'Found float value= {pt_value}, type = {type(pt_value.toIValue())}, ivalue = {pt_value.toIValue()}') return op.Constant(OVType.f32, Shape([]), [pt_value.toIValue()]).outputs() if str(pt_value.type()) in ['torch.bool', 'bool']: @@ -272,7 +275,12 @@ def as_constant_list (self, pt_value): return ov_const.outputs() def input_is_none (self, index): - return index >= len(self.inputs()) or self._raw_input(index) is None + if index >= len(self.inputs()) or self._raw_input(index) is None: + return True + else: + r_input = self._raw_input(index) + return str(r_input.type()) in ['torch.NoneType', 'NoneType'] + def debug (self): print(f'DEBUG CALLED FOR {self._raw_output(0)}') diff --git a/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp b/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp index bb02bda072de0c..e69876993683f6 100644 --- a/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp +++ b/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp @@ -58,6 +58,9 @@ namespace py = pybind11; std::string get_op_type() const override { PYBIND11_OVERRIDE_PURE(std::string, Decoder, get_op_type); } + std::string get_schema() const override + { PYBIND11_OVERRIDE_PURE(std::string, Decoder, get_schema); } + size_t num_of_outputs () const override { PYBIND11_OVERRIDE_PURE(size_t, Decoder, num_of_outputs); } diff --git a/src/core/include/openvino/op/util/framework_node.hpp b/src/core/include/openvino/op/util/framework_node.hpp index 64a62b2aff82ee..99a5a9774dd3e9 100644 --- a/src/core/include/openvino/op/util/framework_node.hpp +++ b/src/core/include/openvino/op/util/framework_node.hpp @@ -9,6 +9,7 @@ #include "openvino/core/partial_shape.hpp" #include "openvino/core/strides.hpp" #include "openvino/op/op.hpp" +#include "openvino/op/util/multi_subgraph_base.hpp" namespace ov { namespace op { @@ -74,14 +75,14 @@ class OPENVINO_API FrameworkNodeAttrs { std::unordered_map m_attrs; }; -class OPENVINO_API FrameworkNode : public Op { +class OPENVINO_API FrameworkNode : public MultiSubGraphOp { public: OPENVINO_OP("FrameworkNode", "util"); BWDCMP_RTTI_DECLARATION; FrameworkNode() = default; - explicit FrameworkNode(const OutputVector& inputs, size_t output_size = 1); + explicit FrameworkNode(const OutputVector& inputs, size_t output_size = 1, size_t num_subgraphs = 0); void validate_and_infer_types() override; @@ -101,6 +102,10 @@ class OPENVINO_API FrameworkNode : public Op { std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; void cache_output_descriptor(); + void clone_to(FrameworkNode& dst) const; + +protected: + FrameworkNode(const FrameworkNode&); private: std::vector> m_inputs_desc; diff --git a/src/core/src/node.cpp b/src/core/src/node.cpp index 67f1f543304ab6..f92e955d72b138 100644 --- a/src/core/src/node.cpp +++ b/src/core/src/node.cpp @@ -452,7 +452,7 @@ std::set> ov::Node::get_output_target_inputs(size_t i) const } ov::descriptor::Tensor& ov::Node::get_output_tensor(size_t i) const { - NGRAPH_CHECK(i < m_outputs.size(), "index '", i, "' out of range in get_output_tensor(size_t i)"); + NGRAPH_CHECK(i < m_outputs.size(), "index '", i, "' out of range in get_output_tensor(size_t i) for node ", *this); return m_outputs[i].get_tensor(); } diff --git a/src/core/src/op/util/framework_node.cpp b/src/core/src/op/util/framework_node.cpp index acfb1ff5be148a..2899c25f4b9c38 100644 --- a/src/core/src/op/util/framework_node.cpp +++ b/src/core/src/op/util/framework_node.cpp @@ -4,25 +4,50 @@ #include "openvino/op/util/framework_node.hpp" +#include "ngraph/graph_util.hpp" #include "itt.hpp" BWDCMP_RTTI_DEFINITION(ov::op::util::FrameworkNode); -ov::op::util::FrameworkNode::FrameworkNode(const OutputVector& inputs, size_t output_size) : Op(inputs) { +ov::op::util::FrameworkNode::FrameworkNode(const OutputVector& inputs, size_t output_size, size_t num_subgraphs) + : MultiSubGraphOp(inputs, num_subgraphs) { set_output_size(output_size); constructor_validate_and_infer_types(); } +ov::op::util::FrameworkNode::FrameworkNode(const ov::op::util::FrameworkNode& other) : MultiSubGraphOp() { + set_arguments(other.input_values()); + other.clone_to(*this); +} + +void ov::op::util::FrameworkNode::clone_to(ov::op::util::FrameworkNode& dst) const { + dst.set_output_size(m_output_descriptions.size()); + + for (size_t i = 0; i < get_output_size(); ++i) { + dst.set_output_type(i, get_output_element_type(i), get_output_partial_shape(i)); + } + dst.m_inputs_desc = m_inputs_desc; + dst.m_output_desc = m_output_desc; + dst.m_attrs = m_attrs; + + for (int i = 0; i < dst.m_bodies.size(); i++) { + dst.m_bodies.push_back(ov::clone_model(*get_function(i))); + } + + for (auto& input_description : m_input_descriptions[0]) { + dst.m_input_descriptions[0].push_back(input_description->copy()); + } + for (auto& output_description : m_output_descriptions[0]) { + dst.m_output_descriptions[0].push_back(output_description->copy()); + } + dst.validate_and_infer_types(); +} + std::shared_ptr ov::op::util::FrameworkNode::clone_with_new_inputs(const OutputVector& new_args) const { NGRAPH_OP_SCOPE(FrameworkNode_clone_with_new_inputs); check_new_args_count(this, new_args); auto node = std::make_shared(new_args); - for (size_t i = 0; i < get_output_size(); ++i) { - node->set_output_type(i, get_output_element_type(i), get_output_partial_shape(i)); - } - node->m_inputs_desc = m_inputs_desc; - node->m_output_desc = m_output_desc; - node->m_attrs = m_attrs; + clone_to(*node); return node; } diff --git a/src/core/src/pass/serialize.cpp b/src/core/src/pass/serialize.cpp index ac1efb30f5c80a..2d4a43b34573e8 100644 --- a/src/core/src/pass/serialize.cpp +++ b/src/core/src/pass/serialize.cpp @@ -501,7 +501,14 @@ class XmlSerializer : public ngraph::AttributeVisitor { m_xml_node.append_attribute(name.c_str()).set_value(create_atribute_list(adapter).c_str()); } void on_adapter(const std::string& name, ngraph::ValueAccessor>& adapter) override { - if (name == "body" || name == "then_body" || name == "else_body") { + if (name == "net") { + ngfunction_2_ir(m_xml_node, + *adapter.get(), + m_custom_opsets, + m_constant_write_handler, + m_version, + m_deterministic); + } else { // TI, Loop do not have attributtes as regular ops, it is necessary to append "body" // to layer above (m_xml_node.parent()) as in ngfunction_2_ir() layer (m_xml_node) with empty attributes // is removed. @@ -514,15 +521,6 @@ class XmlSerializer : public ngraph::AttributeVisitor { m_deterministic); xml_body.remove_attribute("name"); xml_body.remove_attribute("version"); - } else if (name == "net") { - ngfunction_2_ir(m_xml_node, - *adapter.get(), - m_custom_opsets, - m_constant_write_handler, - m_version, - m_deterministic); - } else { - NGRAPH_CHECK(false, "Unsupported Model name."); } } }; diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/c_torchscript_decoder.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/c_torchscript_decoder.hpp index 5538f07cad4cde..417b4336e7e6e8 100644 --- a/src/frontends/pytorch/include/openvino/frontend/pytorch/c_torchscript_decoder.hpp +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/c_torchscript_decoder.hpp @@ -1,6 +1,8 @@ // This is torch jit dependent piece of code for decoding TS jit graph inside Torch runtime // This code was copied from inside PT source tree from POC branch, it cannot be compiled withou torch dependencies +#pragma once + #include #include #include diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp index 34174827d69ff0..e29a6d5fe98a43 100644 --- a/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp @@ -114,6 +114,9 @@ struct Decoder { // TODO: Is it required to be enable_shared_from_this? // Decide whether we need an equivalent member for integer representation (in this case a map is required to understand what it means) virtual std::string get_op_type() const = 0; + // Returns PT node schema as a string + virtual std::string get_schema() const = 0; + // TODO: use canonical name output_size virtual size_t num_of_outputs () const = 0; diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp new file mode 100644 index 00000000000000..91d052d45d3a8b --- /dev/null +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp @@ -0,0 +1,182 @@ +#pragma once + +#include +#include +#include +#include + +#include "exception.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { + +typedef std::map> TensorMap; + +class NodeContext : public frontend::NodeContext { +public: + NodeContext(std::shared_ptr decoder, + TensorMap* tensor_map, + ParameterVector* external_parameters, + const TensorMap& ext_tensor_map) + : // TODO: why the following ctor is explicit? + frontend::NodeContext(decoder->get_op_type()), + m_decoder(decoder), + m_tensor_map(tensor_map), + m_ext_tensor_map(ext_tensor_map), + m_external_parameters(external_parameters) {} + + // Do not search for input in tensor map; try to access it as a constant of specified type T and return its value + template + T const_input(size_t index) const; + + size_t get_input_size() const override { + return m_decoder->inputs().size(); + }; + + // Search for input in tensor map and return an output port for already converted op + // TODO: int due to base class uses it, but naturally it should be size_t for PT + Output get_input(int index) const override { + // std::cerr << "Trying to map input to ngraph..."; + OV_FRONTEND_REQUIRE(!m_decoder->input_is_none(index)); + auto input = m_decoder->input(index); + OV_FRONTEND_REQUIRE(m_tensor_map->count(input)); + return m_tensor_map->at(input); + } + + // TODO: upstream to base class + OutputVector inputs() const { + OutputVector res; + for (size_t input : m_decoder->inputs()) { + // std::cerr << "Searching for input: " << input->unique() << "\n"; + OV_FRONTEND_REQUIRE(m_tensor_map->find(input) != m_tensor_map->end()); + res.push_back(m_tensor_map->at(input)); + } + return res; + } + + bool input_is_none(size_t index) const { + return m_decoder->input_is_none(index); + } + + // Convert the resulting value of this node to ngraph Constant; works correctly only for nodes that produce + // constant value, naturally for prim::Constant + OutputVector as_constant() const { + return m_decoder->as_constant(); + } + + /* + TODO: Should be uncommented when explicit NodeContext ctor won't require passing op_type + const std::string& get_op_type() const override { + return m_decoder->get_op_type(); + } + */ + + std::string get_schema() const { + return m_decoder->get_schema(); + } + + size_t num_of_outputs() const { + return m_decoder->num_of_outputs(); + } + + std::vector outputs() const { + return m_decoder->outputs(); + } + + std::shared_ptr mark_node(std::shared_ptr ov_node) const { + return m_decoder->mark_node(ov_node); + } + + void mark_nodes(std::vector> ov_nodes) const { + return m_decoder->mark_nodes(ov_nodes); + } + + Output mark_output(Output ov_output) const { + return m_decoder->mark_node(ov_output.get_node_shared_ptr()); + } + + Any get_attribute_as_any(const std::string&) const override { + throw std::runtime_error( + "There is no any named attributes in Pytorch node, query by attribute name is not implemented"); + } + + void mutate_input(size_t index, Output ov_output) { + OV_FRONTEND_REQUIRE(!m_decoder->input_is_none(index)); + auto input = m_decoder->input(index); + OV_FRONTEND_REQUIRE(m_tensor_map->count(input)); + m_tensor_map->at(input).get_tensor().set_names({std::to_string(input) + "_"}); + // TODO: find out why this doesn't work + ov_output.get_tensor().add_names({std::to_string(input)}); + (*m_tensor_map)[input] = ov_output; + m_mutated_tensors.insert(input); + } + + std::set get_mutated_tensors() const { + return m_mutated_tensors; + } + + std::shared_ptr get_decoder() const { + return m_decoder; + } + + void add_tensor_to_context(size_t index, Output ov_output) { + if (m_tensor_map->count(index)) { + std::cerr << "[ WARNING ] Current context has tensor. Rewriting." << std::endl; + } + ov_output.get_tensor().add_names({std::to_string(index)}); + (*m_tensor_map)[index] = ov_output; + } + + Output get_tensor_from_model(size_t index) { + if (m_tensor_map->find(index) != m_tensor_map->end()) { + return m_tensor_map->at(index); + } else { + return Output(); + } + } + + Output get_tensor_from_model_or_create_input(size_t index) { + if (m_tensor_map->find(index) != m_tensor_map->end()) { + return m_tensor_map->at(index); + } else { + // nested subgraphs case + auto parameter = std::make_shared(element::dynamic, PartialShape::dynamic()); + parameter->get_output_tensor(0).add_names({std::to_string(index)}); + (*m_tensor_map)[index] = parameter; + m_external_parameters->push_back(parameter); + std::cout << "Nested case, created: " << parameter << std::endl; + return parameter; + } + } + + Output get_input_from_visible_context(size_t index) { + OV_FRONTEND_REQUIRE(index < get_input_size()); + auto input_tensor = get_input(index); + auto input_node = input_tensor.get_node_shared_ptr(); + if (std::dynamic_pointer_cast(input_node)) { + // We need to look into external context for inputs that would be feed into this parameter + auto name = input_node->get_output_tensor(0).get_any_name(); + size_t tensor_idx = (size_t)std::stoll(name); + if (m_ext_tensor_map.count(tensor_idx)) { + input_tensor = m_ext_tensor_map.at(tensor_idx); + } + } + return input_tensor; + } + + std::shared_ptr convert_subgraph(size_t index); + +private: + std::shared_ptr get_constant_at_input(size_t index) const; + + std::shared_ptr m_decoder; + std::set m_mutated_tensors; + TensorMap* m_tensor_map; + const TensorMap& m_ext_tensor_map; + ParameterVector* m_external_parameters; +}; + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/exception.hpp b/src/frontends/pytorch/src/exception.hpp new file mode 100644 index 00000000000000..33f734e27f0df6 --- /dev/null +++ b/src/frontends/pytorch/src/exception.hpp @@ -0,0 +1,18 @@ + +#pragma once + +namespace ov { +namespace frontend { +namespace pytorch { + +#define OV_FRONTEND_REQUIRE(X) \ + do \ + if (!(X)) { \ + throw std::runtime_error(std::string("[ ERROR ] Failed: ") + #X + " at " + __FILE__ + ":" + \ + std::to_string(__LINE__)); \ + } \ + while (false) + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/frontend.cpp b/src/frontends/pytorch/src/frontend.cpp index 90d82be93a6b2c..f22b3a9305ae60 100644 --- a/src/frontends/pytorch/src/frontend.cpp +++ b/src/frontends/pytorch/src/frontend.cpp @@ -1,1064 +1,48 @@ -#include +#include "openvino/frontend/pytorch/frontend.hpp" + #include -#include #include +#include +#include #include -#include "openvino/opsets/opset7.hpp" -#include -#include -#include "openvino/frontend/node_context.hpp" - -#include "openvino/frontend/pytorch/frontend.hpp" - +#include "exception.hpp" #include "input_model.hpp" #include "transforms.hpp" +#include "openvino/frontend/pytorch/node_context.hpp" +#include "op_table.hpp" +#include "openvino/frontend/exception.hpp" +#include "pt_framework_node.hpp" +#include "utils.hpp" namespace ov { namespace frontend { namespace pytorch { - -#define OV_FRONTEND_REQUIRE(X) \ - do if(!(X)) { \ - throw std::runtime_error(std::string("[ ERROR ] Failed: ") + #X + " at " + __FILE__ + ":" + std::to_string(__LINE__)); \ - } while(false) - - -typedef std::map> TensorMap; - -class NodeContext : public frontend::NodeContext { -public: - - NodeContext (std::shared_ptr decoder, const TensorMap& tensor_map) : - // TODO: why the following ctor is explicit? - frontend::NodeContext(decoder->get_op_type()), - m_decoder(decoder), - m_tensor_map(tensor_map) {} - - // Do not search for input in tensor map; try to access it as a constant of specified type T and return its value - template - T const_input (size_t index) const; - - // Search for input in tensor map and return an output port for already converted op - // TODO: int due to base class uses it, but naturally it should be size_t for PT - Output get_input (int index) const override { - //std::cerr << "Trying to map input to ngraph..."; - OV_FRONTEND_REQUIRE(!m_decoder->input_is_none(index)); - return m_tensor_map.at(m_decoder->input(index)); - } - - // TODO: upstream to base class - OutputVector inputs () const { - OutputVector res; - for (size_t input : m_decoder->inputs()) { - //std::cerr << "Searching for input: " << input->unique() << "\n"; - OV_FRONTEND_REQUIRE(m_tensor_map.find(input) != m_tensor_map.end()); - res.push_back(m_tensor_map.at(input)); - } - return res; - } - - bool input_is_none (size_t index) const { - return m_decoder->input_is_none(index); - } - - // Convert the resulting value of this node to ngraph Constant; works correctly only for nodes that produce - // constant value, naturally for prim::Constant - OutputVector as_constant () { - return m_decoder->as_constant(); - } - - /* - TODO: Should be uncommented when explicit NodeContext ctor won't require passing op_type - const std::string& get_op_type() const override { - return m_decoder->get_op_type(); - } - */ - - size_t num_of_outputs () const { - return m_decoder->num_of_outputs(); - } - - std::vector outputs () const { - return m_decoder->outputs(); - } - - std::shared_ptr mark_node (std::shared_ptr ov_node) const { - return m_decoder->mark_node(ov_node); - } - - void mark_nodes (std::vector> ov_nodes) const { - return m_decoder->mark_nodes(ov_nodes); - } - - Output mark_output (Output ov_output) const { - return m_decoder->mark_node(ov_output.get_node_shared_ptr()); - } - - Any get_attribute_as_any(const std::string&) const override { - throw std::runtime_error("There is no any named attributes in Pytorch node, query by attribute name is not implemented"); - } - - void debug () const { - m_decoder->debug(); - } - -private: - - std::shared_ptr get_constant_at_input (size_t index) const; - - std::shared_ptr m_decoder; - const TensorMap& m_tensor_map; -}; - -std::shared_ptr NodeContext::get_constant_at_input (size_t index) const { - OV_FRONTEND_REQUIRE(!input_is_none(index)); - auto input = std::dynamic_pointer_cast(get_input(index).get_node_shared_ptr()); - OV_FRONTEND_REQUIRE(input); - return input; -} - -template <> -std::vector NodeContext::const_input> (size_t index) const { - return get_constant_at_input(index)->cast_vector(); -} - -template <> -std::string NodeContext::const_input (size_t index) const { - throw std::runtime_error("Cannot represent string as OV constant: lack of strings support"); - //return get_constant_at_input(index)->cast_vector()[0]; -} - -template <> -ngraph::Strides NodeContext::const_input (size_t index) const { - return get_constant_at_input(index)->cast_vector(); -} - -template <> -ngraph::CoordinateDiff NodeContext::const_input (size_t index) const { - return get_constant_at_input(index)->cast_vector(); -} - -template <> -ngraph::Shape NodeContext::const_input (size_t index) const { - return get_constant_at_input(index)->cast_vector(); -} - -template <> -int64_t NodeContext::const_input (size_t index) const { - return get_constant_at_input(index)->cast_vector()[0]; -} - -template <> -bool NodeContext::const_input (size_t index) const { - return get_constant_at_input(index)->cast_vector()[0]; -} - -template <> -double NodeContext::const_input (size_t index) const { - return get_constant_at_input(index)->cast_vector()[0]; -} - -template <> -float NodeContext::const_input (size_t index) const { - return get_constant_at_input(index)->cast_vector()[0]; -} - - -class PtFrameworkNode : public ov::op::util::FrameworkNode { -public: - OPENVINO_OP("PtFrameworkNode", "util", ::ov::op::util::FrameworkNode); - - PtFrameworkNode(const std::shared_ptr& decoder, const OutputVector& inputs) - : ov::op::util::FrameworkNode(inputs, decoder->num_of_outputs()), - m_decoder(decoder) { - ov::op::util::FrameworkNodeAttrs attrs; - //std::cerr << "[ DEBUG ] Making PtFrameworkNode for " << m_decoder->get_op_type() << "\n"; - attrs.set_type_name("PTFrameworkNode"); - attrs["PtTypeName"] = m_decoder->get_op_type(); - set_attrs(attrs); - - //std::cout << attrs["PtTypeName"] << std::endl; - - // Set output shapes and types if recognized - - for(size_t i = 0; i < m_decoder->num_of_outputs(); ++i) { - PartialShape ps; - // TODO: Try to decode PT type as a custom type - Any type = element::dynamic; - // FIXME: ROUGH - try { - ps = m_decoder->get_output_shape(i); - } catch (...) { - // nothing, means the info cannot be queried and remains unknown - } - // FIXME: ROUGH - try { - type = m_decoder->get_output_type(i); - } catch (std::runtime_error& e) { - // nothing, means the info cannot be queried and remains unknown - std::cerr << "[ ERROR ] Cannot retrieve type\n" << e.what() << std::endl; - } - // Let's see what type we have - //std::cout << "Can be represented as element::Type: " << type.is() << std::endl; - //std::cout << "element::Type value: " << type.as() << "\n"; - //std::exit(0); - set_custom_output_type(i, type, ps); - } - } - - std::shared_ptr clone_with_new_inputs(const OutputVector& inputs) const override { - return std::make_shared(m_decoder, inputs); - } - - std::string get_op_type() const { - return m_decoder->get_op_type(); - } - - Decoder* get_decoder() const { - return m_decoder.get(); - } - - void add_subgraph (std::shared_ptr subgraph) { - m_subgraphs.push_back(subgraph); - } - - bool visit_attributes(AttributeVisitor& visitor) override { - bool parent_visit_result = FrameworkNode::visit_attributes(visitor); - for(size_t i = 0; i < m_subgraphs.size(); ++i) { - std::string name = "subgraph_" + std::to_string(i); - visitor.on_attribute(name, m_subgraphs[i]); - } - return parent_visit_result; - } - -private: - std::shared_ptr m_decoder; - std::vector> m_subgraphs; -}; - - -Output make_optional_bias( - Output base_op, - const NodeContext& context, - size_t bias_input_idx, - std::vector unsqueeze_dims = {}) -{ - using namespace ngraph; - using std::make_shared; - - if(!context.input_is_none(bias_input_idx)) { - auto bias = context.get_input(bias_input_idx); - if(!unsqueeze_dims.empty()) { - auto indices = opset7::Constant::create(element::i32, { unsqueeze_dims.size() }, unsqueeze_dims); - context.mark_node(indices); - bias = make_shared(bias, indices); - context.mark_output(bias); - } - return make_shared(context.mark_output(base_op), bias); - } else { - return base_op; - } -} - -std::shared_ptr get_rank_node(ov::Output node) { - auto shape = std::make_shared(node); - return std::make_shared(shape); -} - -Output reshape_kernel_for_group( - const NodeContext& context, - Output input, - Output kernel, - int64_t groups) -{ - using namespace ngraph; - using std::make_shared; - - auto in_shape = std::make_shared(input); - auto c_in_idx = opset8::Constant::create(element::i64, Shape{}, {1}); - auto axis_0 = opset8::Constant::create(element::i64, Shape{}, {0}); - auto in_shape_1 = make_shared(in_shape, c_in_idx, axis_0); - auto in_shape_1_uns = make_shared(in_shape_1, axis_0); - auto groups_const = opset8::Constant::create(element::i64, Shape{1}, {groups}); - auto c_in_value = make_shared(in_shape_1_uns, groups_const); - - auto kernel_shape = std::make_shared(kernel); - auto c_out_idx = opset8::Constant::create(element::i64, Shape{}, {0}); - auto kernel_shape_0 = make_shared(kernel_shape, c_out_idx, axis_0); - auto kernel_shape_0_uns = make_shared(kernel_shape_0, axis_0); - auto c_out_value = make_shared(kernel_shape_0_uns, groups_const); - - auto start = opset8::Constant::create(element::i64, Shape{1}, {2}); - auto stop = opset8::Constant::create(element::i64, Shape{1}, {std::numeric_limits::max()}); - auto step = opset8::Constant::create(element::i64, Shape{1}, {1}); - auto remaining_shape = make_shared(kernel_shape, start, stop, step); - - auto new_kernel_shape = make_shared(OutputVector{groups_const, c_out_value, c_in_value, remaining_shape}, 0); - context.mark_nodes({in_shape, c_in_idx, axis_0, in_shape_1, in_shape_1_uns, groups_const, c_in_value, - kernel_shape, c_out_idx, kernel_shape_0, kernel_shape_0_uns, c_out_value, start, stop, step, remaining_shape, new_kernel_shape}); - return make_shared(kernel, new_kernel_shape, false); -} - -// TODO: Remove this super-hack, tensor_map should be local for each conversion activity -// Now it is global to simplify passing it to along the call stack - - -std::shared_ptr convert_pytorch_model(std::shared_ptr pytorch_model, const TensorMap& external_tensor_map); - -OutputVector convert_node(const std::shared_ptr decoder, TensorMap& tensor_map) { - //std::cout << "[ ---- DEBUG ---- ] convert_node\n"; - using ints = std::vector; - using namespace ngraph; - using std::make_shared; - - //std::cerr << "---\nAttempting to convert " << node->kind().toQualString() << "\n"; - //node->dump(); - - - auto context = NodeContext(decoder, tensor_map); - - //std::cerr << "[ DEBUG ] Attempting to convert " << context.get_op_type() << "\n"; - - - auto relu = [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }; - - // TODO: there is also the 3rd input for add in some cases, involve it to the conversion - auto add = [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1))) }; - }; - +std::shared_ptr FrontEnd::convert(const ov::frontend::InputModel::Ptr& model) const { try { - std::map > converters = { - - { "aten::relu", relu}, - - // TODO: Handle inplace semantics correctly - { "aten::relu_", relu}, // inplace version of relu, for us it is identical to regular relu - - { "aten::conv2d", [&]() -> OutputVector { - auto strides = context.const_input(3); - auto pads_begin = context.const_input(4); // FIXME: The same 4 is used twice - auto pads_end = context.const_input(4); // FIXME: The same 4 is used twice - auto dilations = context.const_input(5); - auto groups = context.const_input(6); - - std::shared_ptr conv; - if (groups == 1) { - conv = make_shared( - context.get_input(0), - context.get_input(1), - strides, - pads_begin, - pads_end, - dilations - ); - } else { - conv = make_shared( - context.get_input(0), - reshape_kernel_for_group(context, context.get_input(0), context.get_input(1), groups), - strides, - pads_begin, - pads_end, - dilations - ); - } - - // FIXME: Doesn't work for dynamic rank - // FIXME: Works for 2D convolutions only - return { context.mark_output(make_optional_bias(conv, context, 2, {-2, -1})) }; - }}, - - { "aten::_convolution", [&]() -> OutputVector { - bool transposed = context.const_input(6); - // TODO: Handle this temporary limitation - OV_FRONTEND_REQUIRE(!transposed); - - auto strides = context.const_input(3); - auto pads_begin = context.const_input(4); // FIXME: The same 4 is used twice - auto pads_end = context.const_input(4); // FIXME: The same 4 is used twice - auto dilations = context.const_input(5); - // TODO: Handle skipped input 7 (6 was used above) -- what is it for? - auto groups = context.const_input(8); - - std::shared_ptr conv; - if (groups == 1) { - conv = make_shared( - context.get_input(0), - context.get_input(1), - strides, - pads_begin, - pads_end, - dilations - ); - } else { - conv = make_shared( - context.get_input(0), - context.mark_output(reshape_kernel_for_group(context, context.get_input(0), context.get_input(1), groups)), - strides, - pads_begin, - pads_end, - dilations - ); - } - - // FIXME: Doesn't work for dynamic rank -- why? It supports only 2D convs, but rank value can be not known in static - // FIXME: Works for 2D convolutions only - return { context.mark_output(make_optional_bias(conv, context, 2, {-2, -1})) }; - }}, - - { "aten::batch_norm", [&]() -> OutputVector { - auto training = context.const_input(5); - OV_FRONTEND_REQUIRE(!training); // TODO: support bn training - return { context.mark_node(make_shared( - context.get_input(0), - context.get_input(1), - context.get_input(2), - context.get_input(3), - context.get_input(4), - context.const_input(7) // epsilon - )) }; - }}, - - {"aten::layer_norm", [&]() -> OutputVector { - auto normalized_shape = context.const_input(1); - auto in_pshape_last_dim = *context.get_input(0).get_partial_shape().rbegin(); - OV_FRONTEND_REQUIRE( - normalized_shape.size() == 1 && in_pshape_last_dim.is_static() && - static_cast(in_pshape_last_dim.get_length()) == normalized_shape.back()); - auto eps = context.const_input(4); - auto axes = context.mark_node(opset7::Constant::create(element::i64, Shape{1}, {-1})); // TODO: support any dimention - auto mvn = context.mark_node(make_shared(context.get_input(0), axes, true, eps, op::MVNEpsMode::INSIDE_SQRT)); - std::shared_ptr out_node = std::dynamic_pointer_cast(mvn); - if (!context.input_is_none(2)) { - auto mul = make_shared(out_node, context.get_input(2)); - out_node = std::dynamic_pointer_cast(mul); - } - if (!context.input_is_none(3)) { - auto add = make_shared(out_node, context.get_input(3)); - out_node = std::dynamic_pointer_cast(add); - } - return { context.mark_node(out_node) }; - }}, - - { "aten::add", add}, - - // TODO: Handle inplace semantics correctly - { "aten::add_", add}, // inplace version of add, for us it is identical to regular add - - { "aten::mul", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1))) }; - }}, - - { "aten::div", [&]() -> OutputVector { - auto pythondiv = false; - if (!context.input_is_none(2)) { - auto rounding_mode = context.const_input(2); - if (rounding_mode == "floor") { - pythondiv = true; - } else if (rounding_mode == "trunc") { - pythondiv = true; - //break; - } - } - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1), pythondiv)) }; - }}, - - { "aten::tanh", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }}, - - { "aten::elu", [&]() -> OutputVector { - auto alpha = context.const_input(1); - return { context.mark_node(make_shared(context.get_input(0), alpha)) }; - }}, - - { "aten::sigmoid", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }}, - - { "aten::silu_", [&]() -> OutputVector { - // TODO: Handle inplace semantics correctly - auto sigmoid = context.mark_node(make_shared(context.get_input(0))); - auto silu = context.mark_node(make_shared(context.get_input(0), sigmoid)); - return { silu }; - }}, - - { "aten::gelu", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }}, - - { "aten::sqrt", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }}, - - { "aten::abs", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }}, - - { "aten::square", [&]() -> OutputVector { - auto input_0 = context.get_input(0); - auto const_2 = context.mark_node(opset7::Constant::create(input_0.get_element_type(), Shape{1}, {2})); - return { context.mark_node(make_shared(input_0, const_2)) }; - }}, - - { "aten::hardtanh", [&]() -> OutputVector { - auto min = context.const_input(1); - auto max = context.const_input(2); - return { context.mark_node(make_shared(context.get_input(0), min, max)) }; - }}, - - { "aten::hardsigmoid", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }}, - - { "aten::hardswish", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0))) }; - }}, - - { "aten::relu6", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), 0., 6.)) }; - }}, - - { "aten::softmax", [&]() -> OutputVector { - auto axis = context.const_input(1); - if (axis < 0) { - auto in_rank = context.get_input(0).get_partial_shape().rank(); - OV_FRONTEND_REQUIRE(in_rank.is_static()); - axis = in_rank.get_length() + axis; - } - return { context.mark_node(make_shared(context.get_input(0), static_cast(axis))) }; - }}, - - { "aten::cat", [&]() -> OutputVector { - // aten::cat needs a special handling since it takes a Tensor[] as - // input. We set the inputs of ListConstruct as the inputs of cat. - // - // Pytorch IR: LLGA sees: - // %a %b %c %dim %a %b %c - // \ | / | \ | / - // prim::ListConstruct prim::Constant llga::Concat[axis=%dim] - // \ / - // aten::cat - auto listConstruct = context.get_input(0).get_node(); - auto listConstruct_fw_node = dynamic_cast(listConstruct); - OV_FRONTEND_REQUIRE(listConstruct_fw_node); - OV_FRONTEND_REQUIRE(listConstruct_fw_node->get_decoder()->get_op_type() == "prim::ListConstruct"); - std::cerr << "POINT 1\n"; - auto axis = context.const_input(1); - std::cerr << "POINT 2\n"; - OutputVector inputs; - for (auto& input : listConstruct->inputs()) { - inputs.push_back(input.get_source_output()); - } - std::cerr << "POINT 3\n"; - auto result = context.mark_node(make_shared(inputs, axis)); - //auto list_set = listConstruct_fw_node->get_rt_info()["pt_node"].as>(); // TODO: fails if marking doesn't really mark anything - std::cerr << "POINT 4\n"; - //result->get_rt_info()["pt_node"].as>().insert(list_set.begin(), list_set.end()); - std::cerr << "POINT 5\n"; - return { result }; - }}, - - { "aten::matmul", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1))) }; - }}, - - { "aten::mm", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1))) }; - }}, - - { "aten::linear", [&]() -> OutputVector { - auto matmul = make_shared(context.get_input(0), context.get_input(1), false, true); - return { context.mark_output(make_optional_bias(matmul, context, 2)) }; - }}, - - { "aten::max_pool2d", [&]() -> OutputVector { - auto kernel = context.const_input(1); - auto strides = context.const_input(2); - auto pads_begin = context.const_input(3); // FIXME: The same 3 is used twice - auto pads_end = context.const_input(3); // FIXME: The same 3 is used twice - auto dilations = context.const_input(4); - auto rounding_type = context.const_input(5) ? op::RoundingType::CEIL : op::RoundingType::FLOOR; - - // TODO: Upgrade to opset8::MaxPool to use dilations; for now we suppose they are all zeros - return { context.mark_node(make_shared( - context.get_input(0), strides, /*dilations,*/ pads_begin, pads_end, kernel, rounding_type)) }; - }}, - - { "aten::avg_pool2d", [&]() -> OutputVector { - auto kernel = context.const_input(1); - auto strides = context.const_input(2); - auto pads_begin = context.const_input(3); // FIXME: The same 3 is used twice - auto pads_end = context.const_input(3); // FIXME: The same 3 is used twice - auto rounding_type = context.const_input(4) ? op::RoundingType::CEIL : op::RoundingType::FLOOR; - auto exclude_pad = !context.const_input(5); - // TODO: support divisor override - // auto divisor_override = context.const_input(6); - - return { context.mark_node(make_shared( - context.get_input(0), strides, pads_begin, pads_end, kernel, exclude_pad, rounding_type)) }; - }}, - - { "aten::adaptive_avg_pool2d", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1))) }; - }}, - - { "aten::adaptive_max_pool2d", [&]() -> OutputVector { - auto adaptive_max_pool = context.mark_node(make_shared(context.get_input(0), context.get_input(1))); - auto return_indices = context.const_input(2); - OutputVector res{adaptive_max_pool->output(0)}; - if (return_indices) { - res.push_back(adaptive_max_pool->output(1)); - } - return res; - }}, - - { "aten::mean", [&]() -> OutputVector { - auto keep_dims = context.const_input(2); - OV_FRONTEND_REQUIRE(context.input_is_none(3)); - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1), keep_dims)) }; - }}, - - { "aten::flatten", [&]() -> OutputVector { - auto start_dim = context.const_input(1); - auto end_dim = context.const_input(2); - auto data_pshape = context.get_input(0).get_partial_shape(); - OV_FRONTEND_REQUIRE(data_pshape.rank().is_static()); // TODO: support dynamic rank - auto rank = data_pshape.rank().get_length(); - if (start_dim < 0) { - start_dim = rank + start_dim; - } - if (end_dim < 0) { - end_dim = rank + end_dim; - } - OV_FRONTEND_REQUIRE(start_dim < end_dim); - auto delta = end_dim - start_dim; - std::vector new_shape(rank - delta, 0); - new_shape[start_dim] = -1; - auto new_shape_const = context.mark_node(opset7::Constant::create(element::i64, {new_shape.size()}, new_shape)); - return { context.mark_node(make_shared(context.get_input(0), new_shape_const, true)) }; - }}, - - { "prim::NumToTensor", [&]() -> OutputVector { - // Do nothing // TODO: Really? Should we produce scalar tensor with shape [] instead of custom PT type? - return { context.mark_node(context.get_input(0).get_node_shared_ptr()) }; - }}, - - { "aten::contiguous", [&]() -> OutputVector { - // Do nothing - return { context.mark_node(context.get_input(0).get_node_shared_ptr()) }; - }}, - - { "aten::as_tensor", [&]() -> OutputVector { - OV_FRONTEND_REQUIRE(context.const_input(1) == 6); - OV_FRONTEND_REQUIRE(context.input_is_none(2)); - //auto new_shape_const = context.mark_node(opset7::Constant::create(element::i64, {1}, {1})); - //return { context.mark_node(std::make_shared(context.get_input(0), new_shape_const->output(0), true)) }; - return { context.mark_output(context.get_input(0)) }; - }}, - - { "aten::Int", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), element::i64)) }; - }}, - - { "aten::to", [&]() -> OutputVector { - auto dtype = element::f32; - // TODO: figure out all inputs meaning - OV_FRONTEND_REQUIRE(context.const_input(1) == 6); - OV_FRONTEND_REQUIRE(context.const_input(2) == false); - OV_FRONTEND_REQUIRE(context.const_input(3) == false); - OV_FRONTEND_REQUIRE(context.input_is_none(4)); - return { context.mark_node(make_shared(context.get_input(0), dtype)) }; - }}, - - { "aten::permute", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1))) }; - }}, - - { "aten::embedding", [&]() -> OutputVector { - // TODO: find out the meaning of input idx 2 - OV_FRONTEND_REQUIRE(context.const_input(3) == false); - OV_FRONTEND_REQUIRE(context.const_input(4) == false); - auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1), axis_0)) }; - }}, - - { "aten::transpose", [&]() -> OutputVector { - auto dim0 = context.const_input(1); - auto dim1 = context.const_input(2); - auto data_pshape = context.get_input(0).get_partial_shape(); - auto rank = data_pshape.rank(); - OV_FRONTEND_REQUIRE(rank.is_static()); - auto _rank = rank.get_length(); - if (dim0 < 0) { - dim0 = _rank + dim0; - } - if (dim1 < 0) { - dim1 = _rank + dim1; - } - OV_FRONTEND_REQUIRE(dim0 > 0 && dim1 > 0); - OV_FRONTEND_REQUIRE(dim0 < _rank && dim1 < _rank); - std::vector order(_rank, 0); - std::iota(order.begin(), order.end(), 0); - std::swap(order[dim0], order[dim1]); - auto order_const = context.mark_node(opset7::Constant::create(element::i64, {order.size()}, order)); - return { context.mark_node(make_shared(context.get_input(0), order_const)) }; - }}, - - { "aten::size", [&]() -> OutputVector { - auto shape = context.mark_node(make_shared(context.get_input(0), element::i32)); - if(context.input_is_none(1)) { - return shape->outputs(); - } else { - auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); - return { context.mark_node(make_shared(shape, context.get_input(1), axis_0)) }; - } - }}, - - { "aten::view", [&]() -> OutputVector { - auto shape_node = context.get_input(1).get_node(); - auto shape_node_fw_node = dynamic_cast(shape_node); - std::shared_ptr reshape; - if (shape_node_fw_node && shape_node_fw_node->get_decoder()->get_op_type() == "prim::ListConstruct") { - // TODO: maybe use pt shape instead of whole shape subgraph, because it may be more efficent - OutputVector inputs; - auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); - for (auto& input : shape_node->inputs()) { - auto rank = input.get_partial_shape().rank(); - OV_FRONTEND_REQUIRE(rank.is_dynamic() || rank.get_length() == 0); - auto unsqueeze = context.mark_node(make_shared(input.get_source_output(), axis_0)); - inputs.push_back(unsqueeze); - } - auto concat = context.mark_node(make_shared(inputs, 0)); - reshape = context.mark_node(make_shared(context.get_input(0), concat, false)); - // TODO: temporary disabled - //auto list_set = shape_node_fw_node->get_rt_info()["pt_node"].as>(); - //reshape->get_rt_info()["pt_node"].as>().insert(list_set.begin(), list_set.end()); + // std::cerr << "[ HERE ]\n"; + auto pytorch_model = std::dynamic_pointer_cast(model); + // TODO: Remove this super-hack, tensor_map should be local for each conversion activity, see more info where + // tensor_map is defined now + auto model = convert_pytorch_model(pytorch_model->m_model); + + // TODO: Propose better solution for the next code block + // Usually if nn.Module.forward is given as a source model for conversion, there is the first Parameter + // that represents original `self` argument in forward(self, ...). `self` shouldn't play any role in model + // inference if model is completelly frozed and all methods are inlined. So we check if it doesn't have any + // consumers in the finally converted model and remove this parameter. This parameter should have index 0. + if (model->get_parameters().size() > 0) { + auto self = model->get_parameters()[0]; + if (self->output(0).get_target_inputs().empty()) { + // There is no consumers: safe to remove + std::cout << "[ WARNING ] Removing parameter[0] in converted Pytorch model, because it is never " + "used and treated as `self`\n"; + model->remove_parameter(self); } else { - reshape = context.mark_node(make_shared(context.get_input(0), context.get_input(1), false)); - } - return { reshape }; - }}, - - { "prim::ListUnpack", [&]() -> OutputVector { - auto split_with_sizes = dynamic_cast(context.get_input(0).get_node()); - if (split_with_sizes && split_with_sizes->get_decoder()->get_op_type() == "aten::split_with_sizes") { - auto split = make_shared( - split_with_sizes->get_input_source_output(0), - split_with_sizes->get_input_source_output(2), - split_with_sizes->get_input_source_output(1)); - return context.mark_node(split)->outputs(); - } - throw std::runtime_error("Cannot match prim::ListUnpack with expected aten::split_with_sizes as an input, left prim::ListUnpack not converted"); - }}, - - { "aten::unsqueeze", [&]() -> OutputVector { - return { context.mark_node(make_shared(context.get_input(0), context.get_input(1))) }; - }}, - - { "aten::rsub", [&]() -> OutputVector { - // reverse aten::sub other - self * alpha - auto alpha_casted = context.mark_node(make_shared(context.get_input(2), context.get_input(0).get_element_type())); - auto alpha_mul = context.mark_node(make_shared(context.get_input(0), alpha_casted)); - return { context.mark_node(make_shared(context.get_input(1), alpha_mul)) }; - }}, - - { "aten::slice", [&]() -> OutputVector { - ov::Output dim = context.get_input(1); - ov::Output start = context.get_input(2); - ov::Output end = context.get_input(3); - ov::Output step = context.get_input(4); - - auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); - if (dim.get_partial_shape().rank().is_static() && dim.get_partial_shape().rank().get_length() == 0) { - dim = context.mark_node(make_shared(dim, axis_0)); + std::cout << "[ WARNING ] Couldn't remove parameter[0] in converted Pytorch model\n"; } - if (start.get_partial_shape().rank().is_static() && start.get_partial_shape().rank().get_length() == 0) { - start = context.mark_node(make_shared(start, axis_0)); - } - if (end.get_partial_shape().rank().is_static() && end.get_partial_shape().rank().get_length() == 0) { - end = context.mark_node(make_shared(end, axis_0)); - } - if (step.get_partial_shape().rank().is_static() && step.get_partial_shape().rank().get_length() == 0) { - step = context.mark_node(make_shared(step, axis_0)); - } - return { context.mark_node(make_shared(context.get_input(0), start, end, step, dim)) }; - }}, - - /* TODO: Don't know how to change it quickly to be compiled, consult with Maxim - { "prim::ConstantChunk", [&]() -> OutputVector { - auto chunks = node->i(attr::chunks); // FIXME: create actual attribute function - auto dim = node->i(attr::dim); - auto dim_const = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {dim})); - auto split = context.mark_node(make_shared(context.get_input(0), dim_const, chunks)); - return split->outputs(); - }}, - */ - - /* TODO: Don't need a special way to handle prim::ListConstruct as it doesn't provide any extra service extra to FW Node at this moment - { "prim::ListConstruct", [&]() -> OutputVector { - // TODO. Probably need to replace by a constant of size 0 in one of the following transformations that embed Lists to Tensors for OV. - }}, - */ - - /* Only makes sence if output type is not deducable, but it is not our case - { "aten::__getitem__", [&]() -> OutputVector { - // Should be handled in a special way: returned tensor is an alias for a part of input list - // Temporary we replace this semantics loosing aliasing and just returning a part of input tensor. - // Represent __getitem__ as generic node with only exception for returned type - // We can infer output type base on the input type. Also we can verify that input type is really a list (TODO: is only list acceptable for __getitem__?) - - auto fw_node = make_shared(decoder, context.inputs()); - OV_FRONTEND_REQUIRE(fw_node->outputs().size() == 1); - // TODO: Deduce output type here; do we need it? Looks like PT has already derived all the types, just need to carefully recognize it in original graph - }}, - */ - - { "aten::append", [&]() -> OutputVector { - // Returns a modified list but also modifies the original list as well. So it should replace original list entry point in tensor_map - // with a new modified value. So returned value becomes a complete alise of input value with a list. - // We replace the original entry point and produces new one for new entry point. It helps to maintain correct data dependencies and keep - // graph connected. - - // We still using FW node to represent this op and going to call transformation that remakes it to supported sub graph - - auto fw_node = make_shared(decoder, context.inputs()); - - // Expect only a single output from aten::append - OV_FRONTEND_REQUIRE(fw_node->outputs().size() == 1); - - // Next code is a hack to make alias for a value; in the final version it should be handled via something similar to AliasDB from PT - auto new_alias = fw_node->output(0); - auto input_list = decoder->input(0); - OV_FRONTEND_REQUIRE(tensor_map.find(input_list) != tensor_map.end()); // for debug only, should be guaranteed - tensor_map[input_list] = new_alias; // replacing the original value which comes to append by the resulting list - // TODO: this code won't work incorrectly if it is in a loop and list comes from outer scope - return decoder->mark_node(fw_node)->outputs(); - }}, - - { "prim::Constant", [&]() -> OutputVector { - return context.as_constant(); - }} - - }; - - auto it = converters.find(context.get_op_type()); - if(it != converters.end()) { - //std::cout << "FOUND converter for " << context.get_op_type() << "\n"; - return it->second(); - } else { - std::cout << "DIDN'T FIND converter for " << context.get_op_type() << "\n"; - } - - } - // catch(pybind11::error_already_set& e) { - // std::cout << "Python exception: " << e << "\n"; - // } - catch (std::runtime_error& e) { - std::cout << std::flush; - std::cout << "Exception happened during conversion: " << e.what() << " during conversion of node of type " << context.get_op_type() << '\n'; - std::cout << "Debug for node: " << std::endl; - context.debug(); - std::cout << std::endl; - std::cout << "End of debug output for node" << std::endl; - //throw; - } - catch (...) { - std::cout << "Some exception happened during convertion of node of type: " << context.get_op_type() << std::endl; - std::cout << "Debug for node: " << std::endl; - context.debug(); - std::cout << std::endl; - std::cout << "End of debug output for node" << std::endl; - //throw; - } - //if (node->kind() != prim::ListConstruct) { - // std::cout << "Making unsupported " << node->kind().toQualString() << std::endl; - // node->dump(); - //} - - // Create PtFrameworkNode for everything that wasn't able to be converted normally - // Pay attention to subgraphs that may appear in the node - //std::cerr << "[ DEBUG ] Before PtFramewokNode creation\n"; - auto fw_node = make_shared(decoder, context.inputs()); - //std::cerr << "[ DEBUG ] After PtFramewokNode creation\n"; - - for(size_t i = 0; i < decoder->get_subgraph_size(); ++i) { - //std::cout << "Start converting subgraph\n"; - fw_node->add_subgraph(convert_pytorch_model(decoder->get_subgraph_decoder(i), tensor_map)); // tensor_map should contain both local context and all external contexts - //std::cout << "End converting subgraph\n"; - } - - return decoder->mark_node(fw_node)->outputs(); -} - - -std::shared_ptr convert_pytorch_model(std::shared_ptr pytorch_model, const TensorMap& external_tensor_map) { - std::shared_ptr resulting_model; // define here to make a conversion in a nested scope - { - ParameterVector parameters; - - TensorMap tensor_map; // tensor map of the current context - - //std::cerr << "+++++before++++\n"; - // Go over all pytorch_model inputs and register them in the tensor map: - auto inputs = pytorch_model->inputs(); - //std::cerr << "+++++after++++++\n"; - std::cout << "[ --- DEBUG --- ] convert_pytorch_model: number of inputs: " << inputs.size() << '\n'; - for (int i = 0; i < inputs.size(); ++i) { - std::cout << "Input: " << i << ": " << inputs[i] << "\n"; - PartialShape ps = pytorch_model->get_input_shape(i); - std::cout << "PartialShape = " << ps << "\n"; - Any type = pytorch_model->get_input_type(i); - std::cout << "Custom Type = " << type.type_info().name() << std::endl; - auto parameter = std::make_shared(ov::element::custom, type, ps); - std::cout << "Parameter: " << parameter << std::endl; - parameters.push_back(parameter); - auto order = pytorch_model->get_input_transpose_order(i); - if (order.size() > 0 && !std::is_sorted(order.begin(), order.end())) { - OV_FRONTEND_REQUIRE(ps.is_static()); // TODO: make dynamic - auto sh = ps.get_shape(); - Shape new_shape(sh.size()); - for (int i = 0; i < sh.size(); i++) { - new_shape[order[i]] = sh[i]; - } - auto shape_const = opset7::Constant::create(element::i64, { new_shape.size() }, new_shape); - auto reshape = std::make_shared(parameter, shape_const, false); - auto order_const = opset7::Constant::create(element::i32, { order.size() }, order); - auto transpose = std::make_shared(reshape, order_const); - tensor_map[pytorch_model->input(i)] = transpose; - } else { - tensor_map[pytorch_model->input(i)] = parameter; } - } - - - auto node_visitor = [&](std::shared_ptr node) - { - //std::cerr << "Node convert start" << std::endl; - - // Explore all inputs of node. Node may refer to input value that hasn't been created in the current scope. - // But this value can be found in the outer scope, for this purpose we need to search node in external_tensor_map as well - - auto raw_inputs = node->inputs(); - for(size_t i = 0; i < raw_inputs.size(); ++i) { - auto input = node->input(i); - if(tensor_map.find(input) == tensor_map.end()) { - //std::cout << "Trampoline for input index " << i << " with value " << input << "\n"; - // input refers value in the outer scope, need to create a new Parameter in the current scope - // TODO: Connect outer scope and inner scope properly -- should be handled at the level of that operation that introduced this nest of scopes (e.g. loop or if) - // TODO: Eliminate duplication with the main code for Parameters creation - // TODO: There is no real search for values in outer scope because we don't need to link the usage and definition together at this point -- need to do that otherwise graph will fall apart - PartialShape ps = node->get_input_shape(i); - auto parameter = std::make_shared(node->get_input_type(i), ps); - parameters.push_back(parameter); - // TODO: Missing get_input_transpose_order handling for not trivial layouts - tensor_map[input] = parameter; - //std::cout << "Parameter created\n"; - } - } - - auto converted_outputs = convert_node(node, tensor_map); - //std::cerr << "Node convert before outputs" << std::endl; - - auto fw_outputs = node->outputs(); - - // TODO: Make sure that mapping of fw_outputs to converted_outputs does always work - // FIXME: Now it is not true for at least prim::Constant - for(size_t i = 0; i < converted_outputs.size(); ++i) { - size_t fw_tensor_id = node->output(i); - if(tensor_map.find(fw_tensor_id) != tensor_map.end()) { - //std::cerr << "Duplicated producer for tensor with id = " << fw_tensor_id << " discovered at output " - // << "port " << i << " of node " << node->kind().toQualString() << "\n"; - throw std::runtime_error("Duplicated producer for PT value with unique ID: " + std::to_string(fw_tensor_id)); - } - - // Output shape of converted node should match the original output shape - //std::cerr << "[ DEBUG ] PT output shape = " << get_ov_shape(fw_outputs[i]) << '\n'; - //std::cerr << "[ DEBUG ] OV output shape = " << converted_outputs[i].get_partial_shape() << '\n'; - //OV_FRONTEND_REQUIRE(get_ov_shape(fw_outputs[i]) == converted_outputs[i].get_partial_shape()); - - tensor_map[fw_tensor_id] = converted_outputs[i]; - } - //std::cout << "Node convert end" << std::endl; - }; - - OV_FRONTEND_REQUIRE(pytorch_model->get_subgraph_size() == 1); - pytorch_model->visit_subgraph(0, node_visitor); - //std::cout << "All nodes convert end" << std::endl; - - ResultVector results; - //std::cerr << "Outputs:\n"; - for (size_t i = 0; i < pytorch_model->num_of_outputs(); ++i) { - //std::cerr << i << "\n"; - size_t id = pytorch_model->output(i); - //std::cout << "value = " << id << '\n'; - //std::cout << "X\n"; - if(tensor_map.find(id) == tensor_map.end()) { - // Not found in this scope, searching in the outer scope - // TODO: do real search here, skipped for now - - auto parameter = std::make_shared(element::dynamic, PartialShape::dynamic()); - parameters.push_back(parameter); - tensor_map[id] = parameter; - //std::cout << "Added new parameter based on external value\n"; - } - auto ov_output = tensor_map[id]; - //std::cout << "X\n"; - auto order = pytorch_model->get_output_transpose_order(i); - //std::cout << "X\n"; - if (order.size() > 0 && !std::is_sorted(order.begin(), order.end())) { - throw "Output strides have wrong order."; - } - //std::cout << "X\n"; - //std::cout << ov_output << '\n'; - auto result = std::make_shared(ov_output); - //std::cout << "X\n"; - results.push_back(result); - //std::cerr << "Cluster result " << value->unique() << " with shape " << result->get_output_partial_shape(0) << "\n"; - } - //std::cout << "Y\n"; - - for(size_t i = 0; i < parameters.size(); ++i) { - auto parameter = parameters[i]; - //std::cerr << "parameter[" << i << "].shape = " - // << parameter->get_output_shape(0) << ", consumers: " << parameter->output(0).get_target_inputs().size() << "\n"; - } - //std::cout << "Convert end" << std::endl; - //std::cout << "Number of values collected: " << tensor_map.size() << "\n"; - - resulting_model = std::make_shared(results, parameters); - - // Did a conversion in a nested scope to automatically remove any holders of nodes except those in the graph - } - - // TODO: Propose better solution for the next code block - // Usually if nn.Module.forward is given as a source model for conversion, there is the first Parameter - // that represents original `self` argument in forward(self, ...). `self` shouldn't play any role in model inference - // if model is completelly frozed and all methods are inlined. So we check if it doesn't have any consumers - // in the finally converted model and remove this parameter. This parameter should have index 0. - if(resulting_model->get_parameters().size() > 0) { - auto self = resulting_model->get_parameters()[0]; - if(self->output(0).get_target_inputs().empty()) { - // There is no consumers: safe to remove - std::cout << "[ WARNING ] Removing parameter[0] in converted Pytorch model, because it is never used and treated as `self`\n"; - resulting_model->remove_parameter(self); - } - } - - return resulting_model; -} - -std::shared_ptr FrontEnd::convert(const ov::frontend::InputModel::Ptr& model) const { - try { - //std::cerr << "[ HERE ]\n"; - auto pytorch_model = std::dynamic_pointer_cast(model); - // TODO: Remove this super-hack, tensor_map should be local for each conversion activity, see more info where tensor_map is defined now - TensorMap external_tensor_map; // intentionally empty because there is no external context - auto model = convert_pytorch_model(pytorch_model->m_model, external_tensor_map); apply_pytorch_conversion_transforms(model); return model; } catch (const std::runtime_error& e) { @@ -1069,21 +53,21 @@ std::shared_ptr FrontEnd::convert(const ov::frontend::InputModel::Ptr& mo } bool FrontEnd::supported_impl(const std::vector& variants) const { - //std::cout << "[ ----- DEBUG ------ ] supported_impl with " << variants.size() << " arguments\n"; + // std::cout << "[ ----- DEBUG ------ ] supported_impl with " << variants.size() << " arguments\n"; return false; } ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { - //std::cout << "[ ----- DEBUG ----- ] load_impl with " << variants.size() << " parameters\n"; - if(variants.size() != 1) { + // std::cout << "[ ----- DEBUG ----- ] load_impl with " << variants.size() << " parameters\n"; + if (variants.size() != 1) { throw std::runtime_error("Pytorch frontend supports exactly one parameter in model representation, got " + - std::to_string(variants.size()) + "instead."); + std::to_string(variants.size()) + "instead."); } auto decoder = variants[0].as>(); - //std::cout << "Recognized decoder: " << decoder << "\n"; + // std::cout << "Recognized decoder: " << decoder << "\n"; return std::make_shared(decoder); } -} -} -} +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/node_context.cpp b/src/frontends/pytorch/src/node_context.cpp new file mode 100644 index 00000000000000..bc49bbd7b82b59 --- /dev/null +++ b/src/frontends/pytorch/src/node_context.cpp @@ -0,0 +1,108 @@ +#include "openvino/frontend/pytorch/node_context.hpp" + +#include +#include +#include + +#include "exception.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { + +std::shared_ptr NodeContext::get_constant_at_input(size_t index) const { + OV_FRONTEND_REQUIRE(!input_is_none(index)); + auto input_node = get_input(index).get_node_shared_ptr(); + auto input = std::dynamic_pointer_cast(input_node); + auto param = std::dynamic_pointer_cast(input_node); + if (!input && param) { + // We need to look into external context for inputs that would be feed into this parameter + auto name = param->get_output_tensor(0).get_any_name(); + size_t tensor_idx = (size_t)std::stoll(name); + if (m_ext_tensor_map.count(tensor_idx)) { + auto tensor = m_ext_tensor_map.at(tensor_idx); + input_node = tensor.get_node_shared_ptr(); + input = std::dynamic_pointer_cast(input_node); + } + } + FRONT_END_GENERAL_CHECK(input, "Input with index ", index, " cannot be interpreted as Constant: ", input_node); + return input; +} + +std::shared_ptr NodeContext::convert_subgraph(size_t index) { + auto subgraph_decoder = m_decoder->get_subgraph_decoder(index); + + // Extend external context with internal tensors except Parameter nodes, because internal Parameters are created to + // link internal context with external + TensorMap ext_map(m_ext_tensor_map); + for (auto tensor : *m_tensor_map) { + auto node = tensor.second.get_node_shared_ptr(); + if (!std::dynamic_pointer_cast(node)) + ext_map[tensor.first] = tensor.second; + } + + auto model = convert_pytorch_model(subgraph_decoder, ext_map); + // Remove unused parameters, they could be created as inputs to the parts of graph that weren't + // used for generating output. + for (int i = subgraph_decoder->inputs().size(); i < model->get_parameters().size(); i++) { + auto parameter = model->get_parameters()[i]; + if (parameter->output(0).get_target_inputs().empty()) { + // There is no consumers: safe to remove + std::cout << "[ WARNING ] Removing parameter " << parameter + << " in converted Pytorch model, because it is never used" << std::endl; + model->remove_parameter(parameter); + } + } + return model; +} + +template <> +std::vector NodeContext::const_input>(size_t index) const { + return get_constant_at_input(index)->cast_vector(); +} + +template <> +std::string NodeContext::const_input(size_t index) const { + throw std::runtime_error("Cannot represent string as OV constant: lack of strings support"); + // return get_constant_at_input(index)->cast_vector()[0]; +} + +template <> +ngraph::Strides NodeContext::const_input(size_t index) const { + return get_constant_at_input(index)->cast_vector(); +} + +template <> +ngraph::CoordinateDiff NodeContext::const_input(size_t index) const { + return get_constant_at_input(index)->cast_vector(); +} + +template <> +ngraph::Shape NodeContext::const_input(size_t index) const { + return get_constant_at_input(index)->cast_vector(); +} + +template <> +int64_t NodeContext::const_input(size_t index) const { + return get_constant_at_input(index)->cast_vector()[0]; +} + +template <> +bool NodeContext::const_input(size_t index) const { + return get_constant_at_input(index)->cast_vector()[0]; +} + +template <> +double NodeContext::const_input(size_t index) const { + return get_constant_at_input(index)->cast_vector()[0]; +} + +template <> +float NodeContext::const_input(size_t index) const { + return get_constant_at_input(index)->cast_vector()[0]; +} + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/op/if.cpp b/src/frontends/pytorch/src/op/if.cpp new file mode 100644 index 00000000000000..ee385b3d79ddcb --- /dev/null +++ b/src/frontends/pytorch/src/op/if.cpp @@ -0,0 +1,137 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/opsets/opset8.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +OutputVector translate_if(NodeContext& context) { + auto if_node = std::make_shared(context.get_input(0)); + context.mark_node(if_node); + auto decoder = context.get_decoder(); + OV_FRONTEND_REQUIRE(decoder->get_subgraph_size() == 2); + + auto then_decoder = decoder->get_subgraph_decoder(0); + auto then_body = context.convert_subgraph(0); + if_node->set_then_body(then_body); + auto then_inputs = then_decoder->inputs(); + + auto else_decoder = decoder->get_subgraph_decoder(1); + auto else_body = context.convert_subgraph(1); + if_node->set_else_body(else_body); + auto else_inputs = else_decoder->inputs(); + + std::set input_idxs; + input_idxs.insert(then_inputs.begin(), then_inputs.end()); + input_idxs.insert(else_inputs.begin(), else_inputs.end()); + + std::map inputs_map; + std::map outputs_map; + for (auto param : then_body->get_parameters()) { + auto name = param->get_output_tensor(0).get_any_name(); + size_t input_idx = (size_t)std::stoll(name); + FRONT_END_OP_CONVERSION_CHECK(inputs_map.count(input_idx) == 0, + "More then one then_body input with same tensor name: ", + inputs_map.at(input_idx)[0], + " adding: ", + param); + inputs_map[input_idx] = {param, nullptr}; + } + for (auto param : else_body->get_parameters()) { + auto name = param->get_output_tensor(0).get_any_name(); + size_t input_idx = (size_t)std::stoll(name); + if (inputs_map.count(input_idx)) { + inputs_map[input_idx][1] = param; + } else { + inputs_map[input_idx] = {nullptr, param}; + } + } + std::map> then_body_results; + std::map> else_body_results; + std::set output_idxs; + for (auto result : then_body->get_results()) { + auto name = result->input(0).get_tensor().get_any_name(); + size_t output_idx = (size_t)std::stoll(name); + FRONT_END_OP_CONVERSION_CHECK(then_body_results.count(output_idx) == 0, + "More then one then_body output with same tensor name: ", + then_body_results.at(output_idx), + " adding: ", + result); + then_body_results[output_idx] = result; + output_idxs.insert(output_idx); + } + for (auto result : else_body->get_results()) { + auto name = result->input(0).get_tensor().get_any_name(); + size_t output_idx = (size_t)std::stoll(name); + FRONT_END_OP_CONVERSION_CHECK(else_body_results.count(output_idx) == 0, + "More then one then_body output with same tensor name: ", + else_body_results.at(output_idx), + " adding: ", + result); + then_body_results[output_idx] = result; + output_idxs.insert(output_idx); + } + OutputVector res; + for (int i = 0; i < context.num_of_outputs(); i++) { + res.push_back(if_node->set_output(then_body->get_results()[i], else_body->get_results()[i])); + OV_FRONTEND_REQUIRE(output_idxs.erase(then_decoder->output(i))); + OV_FRONTEND_REQUIRE(output_idxs.erase(else_decoder->output(i))); + } + for (auto output_idx : output_idxs) { + if (!then_body_results.count(output_idx)) { + // Need to add Parameter->Result construction in then body + auto new_parameter = std::make_shared(element::dynamic, PartialShape::dynamic()); + new_parameter->get_output_tensor(0).add_names({std::to_string(output_idx)}); + auto new_result = std::make_shared(new_parameter); + then_body->add_parameters({new_parameter}); + then_body->add_results({new_result}); + then_body->validate_nodes_and_infer_types(); + FRONT_END_OP_CONVERSION_CHECK(inputs_map.count(output_idx), "Input must exist in else body"); + inputs_map[output_idx][0] = new_parameter; + then_body_results[output_idx] = new_result; + std::cout << "[ WARNING ] Modified then body: " << if_node << std::endl; + } else if (!else_body_results.count(output_idx)) { + // Need to add Parameter->Result construction in else body + auto new_parameter = std::make_shared(element::dynamic, PartialShape::dynamic()); + new_parameter->get_output_tensor(0).add_names({std::to_string(output_idx)}); + auto new_result = std::make_shared(new_parameter); + else_body->add_parameters({new_parameter}); + else_body->add_results({new_result}); + else_body->validate_nodes_and_infer_types(); + FRONT_END_OP_CONVERSION_CHECK(inputs_map.count(output_idx), "Input must exist in then body"); + inputs_map[output_idx][1] = new_parameter; + else_body_results[output_idx] = new_result; + std::cout << "[ WARNING ] Modified else body: " << if_node << std::endl; + } + } + // Create prim::If inputs and outputs + for (auto input : inputs_map) { + if (!input_idxs.count(input.first)) { + auto external_output = context.get_tensor_from_model_or_create_input(input.first); + if_node->set_input(external_output, input.second[0], input.second[1]); + } else { + auto external_output = context.get_tensor_from_model(input.first); + if (external_output.get_node()) { + if_node->set_input(external_output, input.second[0], input.second[1]); + } + } + } + for (auto output_idx : output_idxs) { + context.add_tensor_to_context( + output_idx, + if_node->set_output(then_body_results.at(output_idx), else_body_results.at(output_idx))); + } + if_node->validate_and_infer_types(); + return res; +}; + +} // namespace op +} // namespace pytorch +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/src/frontends/pytorch/src/op/loop.cpp b/src/frontends/pytorch/src/op/loop.cpp new file mode 100644 index 00000000000000..0979249665ac52 --- /dev/null +++ b/src/frontends/pytorch/src/op/loop.cpp @@ -0,0 +1,72 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/opsets/opset8.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +OutputVector translate_loop(NodeContext& context) { + auto loop = std::make_shared(context.get_input(0), context.get_input(1)); + auto decoder = context.get_decoder(); + OV_FRONTEND_REQUIRE(decoder->get_subgraph_size() == 1); + auto subgraph_decoder = decoder->get_subgraph_decoder(0); + auto body = context.convert_subgraph(0); + loop->set_function(body); + opset8::Loop::SpecialBodyPorts spec_ports{0, 0}; + loop->set_special_body_ports(spec_ports); + + auto inputs = subgraph_decoder->inputs(); + std::set input_idxs(inputs.begin(), inputs.end()); + std::map inputs_map; + + auto body_parameters = body->get_parameters(); + // #0 parameter is counter + for (int i = 1; i < body_parameters.size(); i++) { + auto param = body_parameters[i]; + auto name = param->get_output_tensor(0).get_any_name(); + size_t input_idx = (size_t)std::stoll(name); + if (inputs_map.count(input_idx)) { + inputs_map[input_idx] = {param}; + } else { + inputs_map[input_idx].push_back(param); + } + } + for (auto input : inputs_map) { + if (!input_idxs.count(input.first)) { + auto external_output = context.get_tensor_from_model_or_create_input(input.first); + loop->set_invariant_inputs(external_output, input.second); + } else { + auto external_output = context.get_tensor_from_model(input.first); + if (external_output.get_node()) { + loop->set_invariant_inputs(external_output, input.second); + } + } + } + // TODO: Connect back edges (merged inputs) + auto body_results = body->get_results(); + FRONT_END_OP_CONVERSION_CHECK(body_results.size() > 0, "At least one output from loop is required - condition."); + std::set output_idxs; + // 0 output is condition, do not need to connect it + for (int i = 1; i < body_results.size(); i++) { + auto result = body_results[i]; + auto name = result->input(0).get_tensor().get_any_name(); + size_t out_idx = (size_t)std::stoll(name); + FRONT_END_OP_CONVERSION_CHECK(output_idxs.count(out_idx) == 0, + "More then one body output with same tensor name."); + output_idxs.insert(out_idx); + context.add_tensor_to_context(out_idx, loop->get_iter_value(result, -1)); + } + loop->validate_and_infer_types(); + return {context.mark_node(loop)->outputs()}; +}; + +} // namespace op +} // namespace pytorch +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/src/frontends/pytorch/src/op/slice.cpp b/src/frontends/pytorch/src/op/slice.cpp new file mode 100644 index 00000000000000..8e215e6950b97f --- /dev/null +++ b/src/frontends/pytorch/src/op/slice.cpp @@ -0,0 +1,74 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/opsets/opset8.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +OutputVector translate_slice(NodeContext& context) { + // aten::slice.t(t[] l, int? start=None, int? end=None, int step=1) -> (t[]) + // aten::slice.Tensor(Tensor(a) self, int dim=0, int? start=None, int? end=None, int step=1) -> (Tensor(a)) + ov::Output dim; + int start_idx; + int end_idx; + int step_idx; + auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); + if (context.get_input_size() == 5) { + dim = context.get_input(1); + if (dim.get_partial_shape().rank().is_dynamic() || dim.get_partial_shape().rank().get_length() == 0) { + dim = context.mark_node(std::make_shared(dim, axis_0)); + } + start_idx = 2; + end_idx = 3; + step_idx = 4; + } else if (context.get_input_size() == 4) { + start_idx = 1; + end_idx = 2; + step_idx = 3; + dim = context.mark_node(opset8::Constant::create(element::i64, Shape{1}, {0})); + } else { + FRONT_END_OP_CONVERSION_CHECK(false, "Slice must have either 4 or 5 inputs."); + } + // TODO: support default start/end with negative step + ov::Output start; + if (!context.input_is_none(start_idx)) { + start = context.get_input(start_idx); + if (start.get_partial_shape().rank().is_dynamic() || start.get_partial_shape().rank().get_length() == 0) { + start = context.mark_node(std::make_shared(start, axis_0)); + } + } else { + start = context.mark_node(opset8::Constant::create(element::i64, Shape{1}, {0})); + } + + ov::Output end; + if (!context.input_is_none(end_idx)) { + end = context.get_input(end_idx); + if (end.get_partial_shape().rank().is_dynamic() || end.get_partial_shape().rank().get_length() == 0) { + end = context.mark_node(std::make_shared(end, axis_0)); + } + } else { + end = context.mark_node(opset8::Constant::create(element::i64, Shape{1}, {INT_MAX})); + } + ov::Output step; + if (!context.input_is_none(step_idx)) { + step = context.get_input(step_idx); + if (step.get_partial_shape().rank().is_dynamic() || step.get_partial_shape().rank().get_length() == 0) { + step = context.mark_node(std::make_shared(step, axis_0)); + } + } else { + step = context.mark_node(opset8::Constant::create(element::i64, Shape{1}, {1})); + } + return {context.mark_node(std::make_shared(context.get_input(0), start, end, step, dim))}; +}; + +} // namespace op +} // namespace pytorch +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/src/frontends/pytorch/src/op_table.cpp b/src/frontends/pytorch/src/op_table.cpp new file mode 100644 index 00000000000000..6840a2136ec927 --- /dev/null +++ b/src/frontends/pytorch/src/op_table.cpp @@ -0,0 +1,700 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "op_table.hpp" + +#include + +#include "pt_framework_node.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +#define OP_CONVERTER(op) OutputVector op(NodeContext& node) + +OP_CONVERTER(translate_if); +OP_CONVERTER(translate_loop); +OP_CONVERTER(translate_slice); + +OutputVector relu(NodeContext& context) { + return {context.mark_node(std::make_shared(context.get_input(0)))}; +}; +OutputVector add(NodeContext& context) { + // TODO: there is also the 3rd input for add in some cases, involve it to the conversion + return {context.mark_node(std::make_shared(context.get_input(0), context.get_input(1)))}; +}; + +OutputVector mul(NodeContext& context) { + return {context.mark_node(std::make_shared(context.get_input(0), context.get_input(1)))}; +} + +OutputVector hardtanh(NodeContext& context) { + float min = -1; + float max = 1; + if (!context.input_is_none(1)) { + min = context.const_input(1); + } + if (!context.input_is_none(2)) { + max = context.const_input(2); + } + return {context.mark_node(std::make_shared(context.get_input(0), min, max))}; +} + +OutputVector hardswish(NodeContext& context) { + return {context.mark_node(std::make_shared(context.get_input(0)))}; +} + +const std::map type_map{ + {0, element::u8}, + {6, element::f32}, +}; + +OutputVector translate_aten_to(NodeContext& context) { + int dtype_idx; + int non_blocking_idx; + int copy_idx; + int memory_format_idx; + if (context.get_input_size() == 5) { + // aten::to.dtype(Tensor(a) self, int dtype, bool non_blocking=False, bool copy=False, int? memory_format=None) + // -> (Tensor(a)) + dtype_idx = 1; + non_blocking_idx = 2; + copy_idx = 3; + memory_format_idx = 4; + } else if (context.get_input_size() == 6) { + // aten::to.device(Tensor(a) self, Device device, int dtype, bool non_blocking=False, bool copy=False, int? + // memory_format=None) -> (Tensor(a)). + // Skipping "device" input. + dtype_idx = 2; + non_blocking_idx = 3; + copy_idx = 4; + memory_format_idx = 5; + } else { + FRONT_END_OP_CONVERSION_CHECK(false, "Unknown aten::to format"); + } + // TODO: do we need to check these inputs? + // OV_FRONTEND_REQUIRE(context.const_input(non_blocking_idx) == false); + // OV_FRONTEND_REQUIRE(context.const_input(copy_idx) == false); + // OV_FRONTEND_REQUIRE(context.input_is_none(memory_format_idx)); + auto dtype_ext_node = context.get_input_from_visible_context(dtype_idx).get_node_shared_ptr(); + auto dtype_tensor = context.get_input(dtype_idx); + auto dtype_fw_node = std::dynamic_pointer_cast(dtype_tensor.get_node_shared_ptr()); + Output cast; + if (dtype_fw_node && dtype_fw_node->get_op_type() == "prim::dtype") { + auto type_input = dtype_fw_node->input(0).get_source_output(); + cast = context.mark_node(std::make_shared(context.get_input(0), type_input)); + } else if (std::dynamic_pointer_cast(dtype_ext_node)) { + auto pt_type = context.const_input(dtype_idx); + FRONT_END_OP_CONVERSION_CHECK(type_map.count(pt_type), "Unknown type in aten::to: ", pt_type); + auto dtype = type_map.at(pt_type); + cast = context.mark_node(std::make_shared(context.get_input(0), dtype)); + } else { + cast = context.mark_node(std::make_shared(context.get_input(0), dtype_tensor)); + } + return {cast}; +} + +OutputVector translate_as_tensor(NodeContext& context) { + auto dtype_ext_node = context.get_input_from_visible_context(1).get_node_shared_ptr(); + auto dtype_tensor = context.get_input(1); + auto dtype_fw_node = std::dynamic_pointer_cast(dtype_tensor.get_node_shared_ptr()); + Output cast; + if (dtype_fw_node && dtype_fw_node->get_op_type() == "prim::dtype") { + auto type_input = dtype_fw_node->input(0).get_source_output(); + cast = context.mark_node(std::make_shared(context.get_input(0), type_input)); + } else if (std::dynamic_pointer_cast(dtype_ext_node)) { + auto pt_type = context.const_input(1); + FRONT_END_OP_CONVERSION_CHECK(type_map.count(pt_type), "Unknown type in aten::as_tensor: ", pt_type); + auto dtype = type_map.at(pt_type); + cast = context.mark_node(std::make_shared(context.get_input(0), dtype)); + } + // OV_FRONTEND_REQUIRE(context.input_is_none(2)); // device: no need to check + // auto new_shape_const = context.mark_node(opset8::Constant::create(element::i64, {1}, {1})); + // return { context.mark_node(std::make_shared(cast, + // new_shape_const->output(0), true)) }; + return {cast}; +} + +} // namespace op + +const std::map get_supported_ops() { + return { + {"aten::relu", op::relu}, + {"aten::relu_", inplace_op}, + + {"aten::conv2d", + [](NodeContext& context) -> OutputVector { + auto strides = context.const_input(3); + auto pads_begin = context.const_input(4); // FIXME: The same 4 is used twice + auto pads_end = context.const_input(4); // FIXME: The same 4 is used twice + auto dilations = context.const_input(5); + auto groups = context.const_input(6); + + std::shared_ptr conv; + if (groups == 1) { + conv = std::make_shared(context.get_input(0), + context.get_input(1), + strides, + pads_begin, + pads_end, + dilations); + } else { + conv = std::make_shared( + context.get_input(0), + reshape_kernel_for_group(context, context.get_input(0), context.get_input(1), groups), + strides, + pads_begin, + pads_end, + dilations); + } + + // FIXME: Doesn't work for dynamic rank// FIXME: Doesn't work for dynamic rank -- why? It supports only 2D convs, but rank value can be not known in static + // FIXME: Works for 2D convolutions only + return {context.mark_output(make_optional_bias(conv, context, 2, {-2, -1}))}; + }}, + + {"aten::_convolution", + [](NodeContext& context) -> OutputVector { + bool transposed = context.const_input(6); + // TODO: Handle this temporary limitation + OV_FRONTEND_REQUIRE(!transposed); + + auto strides = context.const_input(3); + auto pads_begin = context.const_input(4); // FIXME: The same 4 is used twice + auto pads_end = context.const_input(4); // FIXME: The same 4 is used twice + auto dilations = context.const_input(5); + // TODO: Handle skipped input 7 (6 was used above) -- what is it for? + auto groups = context.const_input(8); + + std::shared_ptr conv; + if (groups == 1) { + conv = std::make_shared(context.get_input(0), + context.get_input(1), + strides, + pads_begin, + pads_end, + dilations); + } else { + conv = std::make_shared( + context.get_input(0), + context.mark_output( + reshape_kernel_for_group(context, context.get_input(0), context.get_input(1), groups)), + strides, + pads_begin, + pads_end, + dilations); + } + + // FIXME: Doesn't work for dynamic rank + // FIXME: Works for 2D convolutions only + return {context.mark_output(make_optional_bias(conv, context, 2, {-2, -1}))}; + }}, + + {"aten::batch_norm", + [](NodeContext& context) -> OutputVector { + auto training = context.const_input(5); + OV_FRONTEND_REQUIRE(!training); // TODO: support bn training + return {context.mark_node(std::make_shared( + context.get_input(0), + context.get_input(1), + context.get_input(2), + context.get_input(3), + context.get_input(4), + context.const_input(7) // epsilon + ))}; + }}, + + {"aten::layer_norm", + [](NodeContext& context) -> OutputVector { + auto normalized_shape = context.const_input(1); + auto in_pshape_last_dim = *context.get_input(0).get_partial_shape().rbegin(); + OV_FRONTEND_REQUIRE(normalized_shape.size() == 1 && in_pshape_last_dim.is_static() && + static_cast(in_pshape_last_dim.get_length()) == normalized_shape.back()); + auto eps = context.const_input(4); + auto axes = context.mark_node( + opset8::Constant::create(element::i64, Shape{1}, {-1})); // TODO: support any dimention + auto mvn = context.mark_node( + std::make_shared(context.get_input(0), axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT)); + std::shared_ptr out_node = std::dynamic_pointer_cast(mvn); + if (!context.input_is_none(2)) { + auto mul = std::make_shared(out_node, context.get_input(2)); + out_node = std::dynamic_pointer_cast(mul); + } + if (!context.input_is_none(3)) { + auto add = std::make_shared(out_node, context.get_input(3)); + out_node = std::dynamic_pointer_cast(add); + } + return {context.mark_node(out_node)}; + }}, + + {"aten::add", op::add}, + {"aten::add_", inplace_op}, + + {"aten::mul", op::mul}, + {"aten::mul_", inplace_op}, + + {"aten::div", + [](NodeContext& context) -> OutputVector { + auto pythondiv = false; + if (!context.input_is_none(2)) { + auto rounding_mode = context.const_input(2); + if (rounding_mode == "floor") { + pythondiv = true; + } else if (rounding_mode == "trunc") { + pythondiv = true; + // break; + } + } + return {context.mark_node( + std::make_shared(context.get_input(0), context.get_input(1), pythondiv))}; + }}, + + {"aten::tanh", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0)))}; + }}, + + {"aten::elu", + [](NodeContext& context) -> OutputVector { + auto alpha = context.const_input(1); + return {context.mark_node(std::make_shared(context.get_input(0), alpha))}; + }}, + + {"aten::sigmoid", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0)))}; + }}, + + {"aten::gelu", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0)))}; + }}, + + {"aten::sqrt", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0)))}; + }}, + + {"aten::abs", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0)))}; + }}, + + {"aten::square", + [](NodeContext& context) -> OutputVector { + auto input_0 = context.get_input(0); + auto const_2 = context.mark_node(opset8::Constant::create(input_0.get_element_type(), Shape{1}, {2})); + return {context.mark_node(std::make_shared(input_0, const_2))}; + }}, + + {"aten::hardtanh", op::hardtanh}, + {"aten::hardtanh_", inplace_op}, + + {"aten::hardsigmoid", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0)))}; + }}, + + {"aten::hardswish", op::hardswish}, + {"aten::hardswish_", inplace_op}, + + {"aten::silu_", + [](NodeContext& context) -> OutputVector { + auto swish = std::make_shared(context.get_input(0)); + context.mutate_input(0, swish); + return {context.mark_node(swish)}; + }}, + + {"aten::relu6", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0), 0., 6.))}; + }}, + + {"aten::softmax", + [](NodeContext& context) -> OutputVector { + auto axis = context.const_input(1); + if (axis < 0) { + auto in_rank = context.get_input(0).get_partial_shape().rank(); + OV_FRONTEND_REQUIRE(in_rank.is_static()); + axis = in_rank.get_length() + axis; + } + return { + context.mark_node(std::make_shared(context.get_input(0), static_cast(axis)))}; + }}, + + {"aten::cat", + [](NodeContext& context) -> OutputVector { + // aten::cat needs a special handling since it takes a Tensor[] as + // input. We set the inputs of ListConstruct as the inputs of cat. + // + // Pytorch IR: LLGA sees: + // %a %b %c %dim %a %b %c + // \ | / | \ | / + // prim::ListConstruct prim::Constant llga::Concat[axis=%dim] + // \ / + // aten::cat + auto listConstruct = context.get_input(0).get_node(); + auto listConstruct_fw_node = dynamic_cast(listConstruct); + OV_FRONTEND_REQUIRE(listConstruct_fw_node); + OV_FRONTEND_REQUIRE(listConstruct_fw_node->get_op_type() == "prim::ListConstruct"); + auto axis = context.const_input(1); + OutputVector inputs; + for (auto& input : listConstruct->inputs()) { + inputs.push_back(input.get_source_output()); + } + auto result = context.mark_node(std::make_shared(inputs, axis)); + // TODO: do we really need to do that? + // auto list_set = listConstruct_fw_node->get_rt_info()["pt_node"].as>(); + // result->get_rt_info()["pt_node"].as>().insert(list_set.begin(), list_set.end()); + return {result}; + }}, + + {"aten::matmul", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0), context.get_input(1)))}; + }}, + + {"aten::mm", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0), context.get_input(1)))}; + }}, + + {"aten::linear", + [](NodeContext& context) -> OutputVector { + auto matmul = std::make_shared(context.get_input(0), context.get_input(1), false, true); + return {context.mark_output(make_optional_bias(matmul, context, 2))}; + }}, + + {"aten::max_pool2d", + [](NodeContext& context) -> OutputVector { + auto kernel = context.const_input(1); + auto strides = context.const_input(2); + auto pads_begin = context.const_input(3); // FIXME: The same 3 is used twice + auto pads_end = context.const_input(3); // FIXME: The same 3 is used twice + auto dilations = context.const_input(4); + auto rounding_type = + context.const_input(5) ? ov::op::RoundingType::CEIL : ov::op::RoundingType::FLOOR; + + // TODO: Upgrade to opset8::MaxPool to use dilations; for now we suppose they are all zeros + return {context.mark_node(std::make_shared(context.get_input(0), + strides, + dilations, + pads_begin, + pads_end, + kernel, + rounding_type))}; + }}, + + {"aten::avg_pool2d", + [](NodeContext& context) -> OutputVector { + auto kernel = context.const_input(1); + auto strides = context.const_input(2); + auto pads_begin = context.const_input(3); // FIXME: The same 3 is used twice + auto pads_end = context.const_input(3); // FIXME: The same 3 is used twice + auto rounding_type = + context.const_input(4) ? ov::op::RoundingType::CEIL : ov::op::RoundingType::FLOOR; + auto exclude_pad = !context.const_input(5); + // TODO: support divisor override + // auto divisor_override = context.const_input(6); + + return {context.mark_node(std::make_shared(context.get_input(0), + strides, + pads_begin, + pads_end, + kernel, + exclude_pad, + rounding_type))}; + }}, + + {"aten::adaptive_avg_pool2d", + [](NodeContext& context) -> OutputVector { + return {context.mark_node( + std::make_shared(context.get_input(0), context.get_input(1)))}; + }}, + + {"aten::adaptive_max_pool2d", + [](NodeContext& context) -> OutputVector { + auto adaptive_max_pool = context.mark_node( + std::make_shared(context.get_input(0), context.get_input(1))); + auto return_indices = context.const_input(2); + OutputVector res{adaptive_max_pool->output(0)}; + if (return_indices) { + res.push_back(adaptive_max_pool->output(1)); + } + return res; + }}, + + {"aten::mean", + [](NodeContext& context) -> OutputVector { + auto keep_dims = context.const_input(2); + OV_FRONTEND_REQUIRE(context.input_is_none(3)); + return {context.mark_node( + std::make_shared(context.get_input(0), context.get_input(1), keep_dims))}; + }}, + + {"aten::flatten", + [](NodeContext& context) -> OutputVector { + auto start_dim = context.const_input(1); + auto end_dim = context.const_input(2); + auto data_pshape = context.get_input(0).get_partial_shape(); + OV_FRONTEND_REQUIRE(data_pshape.rank().is_static()); // TODO: support dynamic rank + auto rank = data_pshape.rank().get_length(); + if (start_dim < 0) { + start_dim = rank + start_dim; + } + if (end_dim < 0) { + end_dim = rank + end_dim; + } + OV_FRONTEND_REQUIRE(start_dim < end_dim); + auto delta = end_dim - start_dim; + std::vector new_shape(rank - delta, 0); + new_shape[start_dim] = -1; + auto new_shape_const = + context.mark_node(opset8::Constant::create(element::i64, {new_shape.size()}, new_shape)); + return {context.mark_node(std::make_shared(context.get_input(0), new_shape_const, true))}; + }}, + + {"prim::NumToTensor", + [](NodeContext& context) -> OutputVector { + // Do nothing // TODO: Really? Should we produce scalar tensor with shape [] instead of custom PT type? + return {context.mark_node(context.get_input(0).get_node_shared_ptr())}; + }}, + + {"aten::contiguous", + [](NodeContext& context) -> OutputVector { + // Do nothing + return {context.mark_node(context.get_input(0).get_node_shared_ptr())}; + }}, + + {"aten::as_tensor", op::translate_as_tensor}, + + {"aten::Int", + [](NodeContext& context) -> OutputVector { + return {context.mark_node(std::make_shared(context.get_input(0), element::i64))}; + }}, + + {"aten::to", op::translate_aten_to}, + + {"aten::permute", + [](NodeContext& context) -> OutputVector { + return { + context.mark_node(std::make_shared(context.get_input(0), context.get_input(1)))}; + }}, + + {"aten::embedding", + [](NodeContext& context) -> OutputVector { + // TODO: find out the meaning of input idx 2 + OV_FRONTEND_REQUIRE(context.const_input(3) == false); + OV_FRONTEND_REQUIRE(context.const_input(4) == false); + auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); + return {context.mark_node( + std::make_shared(context.get_input(0), context.get_input(1), axis_0))}; + }}, + + {"aten::transpose", + [](NodeContext& context) -> OutputVector { + auto dim0 = context.const_input(1); + auto dim1 = context.const_input(2); + auto data_pshape = context.get_input(0).get_partial_shape(); + auto rank = data_pshape.rank(); + OV_FRONTEND_REQUIRE(rank.is_static()); + auto _rank = rank.get_length(); + if (dim0 < 0) { + dim0 = _rank + dim0; + } + if (dim1 < 0) { + dim1 = _rank + dim1; + } + OV_FRONTEND_REQUIRE(dim0 > 0 && dim1 > 0); + OV_FRONTEND_REQUIRE(dim0 < _rank && dim1 < _rank); + std::vector order(_rank, 0); + std::iota(order.begin(), order.end(), 0); + std::swap(order[dim0], order[dim1]); + auto order_const = context.mark_node(opset8::Constant::create(element::i64, {order.size()}, order)); + return {context.mark_node(std::make_shared(context.get_input(0), order_const))}; + }}, + + {"aten::size", + [](NodeContext& context) -> OutputVector { + auto shape = context.mark_node(std::make_shared(context.get_input(0))); + if (context.input_is_none(1)) { + return shape->outputs(); + } else { + auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); + return {context.mark_node(std::make_shared(shape, context.get_input(1), axis_0))}; + } + }}, + + {"aten::view", + [](NodeContext& context) -> OutputVector { + auto shape_node = context.get_input(1).get_node(); + auto shape_node_fw_node = dynamic_cast(shape_node); + std::shared_ptr reshape; + if (shape_node_fw_node && shape_node_fw_node->get_decoder()->get_op_type() == "prim::ListConstruct") { + // TODO: maybe use pt shape instead of whole shape subgraph, because it may be more efficent + OutputVector inputs; + auto axis_0 = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {0})); + for (auto& input : shape_node->inputs()) { + auto rank = input.get_partial_shape().rank(); + OV_FRONTEND_REQUIRE(rank.is_dynamic() || rank.get_length() == 0); + auto unsqueeze = + context.mark_node(std::make_shared(input.get_source_output(), axis_0)); + inputs.push_back(unsqueeze); + } + auto concat = context.mark_node(std::make_shared(inputs, 0)); + reshape = context.mark_node(std::make_shared(context.get_input(0), concat, false)); + auto list_set = shape_node_fw_node->get_rt_info()["pt_node"].as>(); + reshape->get_rt_info()["pt_node"].as>().insert(list_set.begin(), list_set.end()); + } else { + reshape = context.mark_node( + std::make_shared(context.get_input(0), context.get_input(1), false)); + } + return {reshape}; + }}, + + {"aten::unsqueeze", + [](NodeContext& context) -> OutputVector { + return { + context.mark_node(std::make_shared(context.get_input(0), context.get_input(1)))}; + }}, + + {"aten::rsub", + [](NodeContext& context) -> OutputVector { + // reverse aten::sub other - self * alpha + auto alpha_casted = context.mark_node( + std::make_shared(context.get_input(2), context.get_input(0).get_element_type())); + auto alpha_mul = context.mark_node(std::make_shared(context.get_input(0), alpha_casted)); + return {context.mark_node(std::make_shared(context.get_input(1), alpha_mul))}; + }}, + + {"aten::slice", op::translate_slice}, + {"prim::Loop", op::translate_loop}, + {"prim::If", op::translate_if}, + + {"prim::Constant", + [](NodeContext& context) -> OutputVector { + return context.as_constant(); + }}, + + {"aten::dim", + [](NodeContext& context) -> OutputVector { + auto shape = std::make_shared(context.get_input(0), element::i32); + auto rank = std::make_shared(shape, element::i32); + auto squeeze = std::make_shared(rank); + return {context.mark_node(squeeze)}; + }}, + + {"aten::reciprocal", + [](NodeContext& context) -> OutputVector { + auto x = context.get_input(0); + auto const_neg_1 = opset8::Constant::create(element::i32, Shape{}, {-1}); + auto cast = std::make_shared(const_neg_1, x); + auto power = std::make_shared(x, cast); + return {context.mark_node(power)}; + }}, + + {"aten::sub", + [](NodeContext& context) -> OutputVector { + auto x = context.get_input(0); + auto y = context.get_input(1); + // default is 1 so no need to multiply by alpha + if (!context.input_is_none(2)) { + auto alpha = context.get_input(2); + auto casted_alpha = std::make_shared(alpha, y); + y = std::make_shared(casted_alpha, y); + } + return {context.mark_node(std::make_shared(x, y))}; + }}, + + {"aten::eq", + [](NodeContext& context) -> OutputVector { + auto x = context.get_input(0); + auto y = context.get_input(1); + return {context.mark_node(std::make_shared(x, y))}; + }}, + + {"aten::ne", + [](NodeContext& context) -> OutputVector { + auto x = context.get_input(0); + auto y = context.get_input(1); + return {context.mark_node(std::make_shared(x, y))}; + }}, + + {"aten::gt", + [](NodeContext& context) -> OutputVector { + auto x = context.get_input(0); + auto y = context.get_input(1); + return {context.mark_node(std::make_shared(x, y))}; + }}, + + {"aten::lt", + [](NodeContext& context) -> OutputVector { + auto x = context.get_input(0); + auto y = context.get_input(1); + return {context.mark_node(std::make_shared(x, y))}; + }}, + + {"aten::neg", + [](NodeContext& context) -> OutputVector { + auto x = context.get_input(0); + auto const_neg_1 = opset8::Constant::create(element::i32, Shape{}, {-1}); + auto cast = std::make_shared(const_neg_1, x); + return {context.mark_node(std::make_shared(x, cast))}; + }}, + + /* TODO: Don't need a special way to handle prim::ListConstruct as it doesn't provide any extra service extra to FW Node at this moment + { "prim::ListConstruct", [](NodeContext& context) -> OutputVector { + // TODO. Probably need to replace by a constant of size 0 in one of the following transformations that embed Lists to Tensors for OV. + }}, + */ + + /* Only makes sence if output type is not deducable, but it is not our case + { "aten::__getitem__", [](NodeContext& context) -> OutputVector { + // Should be handled in a special way: returned tensor is an alias for a part of input list + // Temporary we replace this semantics loosing aliasing and just returning a part of input tensor. + // Represent __getitem__ as generic node with only exception for returned type + // We can infer output type base on the input type. Also we can verify that input type is really a list (TODO: is only list acceptable for __getitem__?) + auto fw_node = make_shared(context.get_decoder(), context.inputs()); + OV_FRONTEND_REQUIRE(fw_node->outputs().size() == 1); + // TODO: Deduce output type here; do we need it? Looks like PT has already derived all the types, just need to carefully recognize it in original graph + }}, + */ + + { "aten::append", [](NodeContext& context) -> OutputVector { + // Returns a modified list but also modifies the original list as well. So it should replace original list entry point in tensor_map + // with a new modified value. So returned value becomes a complete alise of input value with a list. + // We replace the original entry point and produces new one for new entry point. It helps to maintain correct data dependencies and keep + // graph connected. + + // We still using FW node to represent this op and going to call transformation that remakes it to supported sub graph + + auto fw_node = std::make_shared(context.get_decoder(), context.inputs()); + + // Expect only a single output from aten::append + OV_FRONTEND_REQUIRE(fw_node->outputs().size() == 1); + + // Next code is a hack to make alias for a value; in the final version it should be handled via something similar to AliasDB from PT + context.mutate_input(0, fw_node->output(0)); + + // TODO: this code won't work incorrectly if it is in a loop and list comes from outer scope + return context.mark_node(fw_node)->outputs(); + }}, + + + // TODO: Don't know how to change it quickly to be compiled, consult with Maxim + /*{ "prim::ConstantChunk", [&]() -> OutputVector { + auto chunks = node->i(attr::chunks); // FIXME: create actual attribute function + auto dim = node->i(attr::dim); + auto dim_const = context.mark_node(opset8::Constant::create(element::i64, Shape{}, {dim})); + auto split = context.mark_node(std::make_shared(context.get_input(0), dim_const, chunks)); + return split->outputs(); + }},*/ + }; +}; + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/op_table.hpp b/src/frontends/pytorch/src/op_table.hpp new file mode 100644 index 00000000000000..d4f472c5102a66 --- /dev/null +++ b/src/frontends/pytorch/src/op_table.hpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "openvino/frontend/pytorch/node_context.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +using CreatorFunction = std::function; + +const std::map get_supported_ops(); + +} // namespace paddle +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/pt_framework_node.hpp b/src/frontends/pytorch/src/pt_framework_node.hpp new file mode 100644 index 00000000000000..0fe52fe21a412b --- /dev/null +++ b/src/frontends/pytorch/src/pt_framework_node.hpp @@ -0,0 +1,136 @@ +#include + +#pragma once + +namespace ov { +namespace frontend { +namespace pytorch { +class PtFrameworkNode : public ov::op::util::FrameworkNode { +public: + OPENVINO_OP("PtFrameworkNode", "util", ::ov::op::util::FrameworkNode); + + PtFrameworkNode(const std::shared_ptr& decoder, const OutputVector& inputs, size_t output_size) + : ov::op::util::FrameworkNode(inputs, output_size, decoder->get_subgraph_size()), + m_decoder(decoder) { + ov::op::util::FrameworkNodeAttrs attrs; + // std::cerr << "[ DEBUG ] Making PtFrameworkNode for " << m_decoder->get_op_type() << "\n"; + attrs.set_type_name("PTFrameworkNode"); + attrs["PtTypeName"] = m_decoder->get_op_type(); + attrs["PtSchema"] = m_decoder->get_schema(); + set_attrs(attrs); + + // std::cout << attrs["PtTypeName"] << std::endl; + + // Set output shapes and types if recognized + + for (size_t i = 0; i < output_size; ++i) { + PartialShape ps; + // TODO: Try to decode PT type as a custom type + Any type = element::dynamic; + // FIXME: ROUGH + if (i < decoder->num_of_outputs()) { + try { + ps = m_decoder->get_output_shape(i); + } catch (...) { + // nothing, means the info cannot be queried and remains unknown + } + // FIXME: ROUGH + try { + type = m_decoder->get_output_type(i); + } catch (std::runtime_error& e) { + // nothing, means the info cannot be queried and remains unknown + std::cerr << "[ ERROR ] Cannot retrieve type\n" << e.what() << std::endl; + } + } else { + std::cerr << "[ WARNING ] Cannot retrieve type for output not existent in pt node: " + << m_decoder->get_op_type() << " with 0 input: " << m_decoder->input(0) << std::endl; + } + // Let's see what type we have + // std::cout << "Can be represented as element::Type: " << type.is() << std::endl; + // std::cout << "element::Type value: " << type.as() << "\n"; + // std::exit(0); + set_custom_output_type(i, type, ps); + } + } + + PtFrameworkNode(const std::shared_ptr& decoder, const OutputVector& inputs) : + PtFrameworkNode(decoder, inputs, decoder->num_of_outputs()) {} + + std::shared_ptr clone_with_new_inputs(const OutputVector& inputs) const override { + auto op = std::make_shared(m_decoder, inputs, get_output_size()); + + for (auto body_index = 0; body_index < m_bodies.size(); ++body_index) { + op->set_function(body_index, clone_model(*get_function(body_index))); + for (const auto& m_input_descr : m_input_descriptions[body_index]) { + op->m_input_descriptions[body_index].push_back(m_input_descr->copy()); + } + for (const auto& m_output_descr : m_output_descriptions[body_index]) { + op->m_output_descriptions[body_index].push_back(m_output_descr->copy()); + } + } + op->validate_and_infer_types(); + + return op; + } + + std::string get_op_type() const { + return m_decoder->get_op_type(); + } + + Decoder* get_decoder() const { + return m_decoder.get(); + } + + bool visit_attributes(AttributeVisitor& visitor) override { + bool parent_visit_result = FrameworkNode::visit_attributes(visitor); + // TODO: serialize bodies and descriptors + /*for (size_t i = 0; i < m_bodies.size(); ++i) { + //visitor.on_attribute("body", m_bodies[i]); + //visitor.on_attribute("input_descriptions", m_input_descriptions[i]); + //visitor.on_attribute("output_descriptions", m_output_descriptions[i]); + }*/ + return parent_visit_result; + } + + void validate_and_infer_types() override { + for (int i = 0; i < m_bodies.size(); i++) { + // Input + for (const auto& input_description : m_input_descriptions[i]) { + auto index = input_description->m_input_index; + if (auto invariant_input_description = + ov::as_type_ptr(input_description)) { + auto body_parameter = + m_bodies[i]->get_parameters().at(invariant_input_description->m_body_parameter_index); + + auto body_param_partial_shape = body_parameter->get_partial_shape(); + auto input_partial_shape = input(index).get_partial_shape(); + + body_parameter->set_partial_shape(input_partial_shape); + } + } + + // Body + m_bodies[i]->validate_nodes_and_infer_types(); + + // Output + for (const auto& output_description : m_output_descriptions[i]) { + auto index = output_description->m_output_index; + + auto body_value = m_bodies[i]->get_results().at(output_description->m_body_value_index)->input_value(0); + + if (auto body_output_description = + ov::as_type_ptr(output_description)) { + const ov::PartialShape& ps = body_value.get_partial_shape(); + set_output_type(index, body_value.get_element_type(), ps); + } + } + } + } + +private: + std::shared_ptr m_decoder; +}; + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/utils.cpp b/src/frontends/pytorch/src/utils.cpp new file mode 100644 index 00000000000000..07f3b4f50e4304 --- /dev/null +++ b/src/frontends/pytorch/src/utils.cpp @@ -0,0 +1,420 @@ +#include "utils.hpp" + +#include +#include + +#include "exception.hpp" +#include "op_table.hpp" +#include "pt_framework_node.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +int LEVEL = 0; +int NUMBER = 0; +int COUNTER = 0; + +Output make_optional_bias(Output base_op, + const NodeContext& context, + size_t bias_input_idx, + std::vector unsqueeze_dims) { + using namespace ngraph; + using std::make_shared; + + if (!context.input_is_none(bias_input_idx)) { + auto bias = context.get_input(bias_input_idx); + if (!unsqueeze_dims.empty()) { + auto indices = opset8::Constant::create(element::i32, {unsqueeze_dims.size()}, unsqueeze_dims); + context.mark_node(indices); + bias = make_shared(bias, indices); + context.mark_output(bias); + } + return make_shared(context.mark_output(base_op), bias); + } else { + return base_op; + } +} + +std::shared_ptr get_rank_node(ov::Output node) { + auto shape = std::make_shared(node); + return std::make_shared(shape); +} + +Output reshape_kernel_for_group(const NodeContext& context, + Output input, + Output kernel, + int64_t groups) { + using namespace ngraph; + using std::make_shared; + + auto in_shape = std::make_shared(input); + auto c_in_idx = opset8::Constant::create(element::i64, Shape{}, {1}); + auto axis_0 = opset8::Constant::create(element::i64, Shape{}, {0}); + auto in_shape_1 = make_shared(in_shape, c_in_idx, axis_0); + auto in_shape_1_uns = make_shared(in_shape_1, axis_0); + auto groups_const = opset8::Constant::create(element::i64, Shape{1}, {groups}); + auto c_in_value = make_shared(in_shape_1_uns, groups_const); + + auto kernel_shape = std::make_shared(kernel); + auto c_out_idx = opset8::Constant::create(element::i64, Shape{}, {0}); + auto kernel_shape_0 = make_shared(kernel_shape, c_out_idx, axis_0); + auto kernel_shape_0_uns = make_shared(kernel_shape_0, axis_0); + auto c_out_value = make_shared(kernel_shape_0_uns, groups_const); + + auto start = opset8::Constant::create(element::i64, Shape{1}, {2}); + auto stop = opset8::Constant::create(element::i64, Shape{1}, {std::numeric_limits::max()}); + auto step = opset8::Constant::create(element::i64, Shape{1}, {1}); + auto remaining_shape = make_shared(kernel_shape, start, stop, step); + + auto new_kernel_shape = + make_shared(OutputVector{groups_const, c_out_value, c_in_value, remaining_shape}, 0); + context.mark_nodes({in_shape, + c_in_idx, + axis_0, + in_shape_1, + in_shape_1_uns, + groups_const, + c_in_value, + kernel_shape, + c_out_idx, + kernel_shape_0, + kernel_shape_0_uns, + c_out_value, + start, + stop, + step, + remaining_shape, + new_kernel_shape}); + return make_shared(kernel, new_kernel_shape, false); +} + +OutputVector convert_node(NodeContext* context) { + // std::cout << "[ ---- DEBUG ---- ] convert_node\n"; + + // std::cerr << "---\nAttempting to convert " << node->kind().toQualString() << "\n"; + // node->dump(); + + // std::cerr << "[ DEBUG ] Attempting to convert " << context.get_op_type() << "\n"; + + try { + auto CONVERTERS_MAP = get_supported_ops(); + auto it = CONVERTERS_MAP.find(context->get_op_type()); + if (it != CONVERTERS_MAP.end()) { + // std::cout << "FOUND converter for " << context.get_op_type() << "\n"; + return it->second(*context); + } else { + const std::set known_skips{"prim::RaiseException", + "aten::warn", + // remove all that above + "prim::TupleConstruct", + "prim::ListConstruct", + "aten::format", + "aten::append", + "aten::update", + "aten::dict", + "aten::list", + "aten::_set_item", + "aten::__getitem__", + "aten::__isnot__", + "aten::__contains__", + "prim::unchecked_cast", + "prim::Uninitialized", + "prim::SetAttr", + "prim::GetAttr", + "prim::ListUnpack", + "aten::__not__"}; + if (!known_skips.count(context->get_op_type())) { + std::cout << "DIDN'T FIND converter for " << context->get_op_type() << " with inputs:"; + if (context->inputs().size() == 0) { + std::cout << " None"; + } + for (auto input : context->inputs()) { + std::cout << " " << input; + } + std::cout << " with schema: " << context->get_schema() << std::endl; + } + } + + } + // catch(pybind11::error_already_set& e) { + // std::cout << "Python exception: " << e << "\n"; + // } + catch (std::runtime_error& e) { + std::cout << "Exception happened during conversion of op: " << context->get_op_type() + << " with schema: " << context->get_schema() << ": " << e.what() << '\n'; + // throw; + } catch (...) { + std::cout << "Some exception happened during conversion of node of type: " << context->get_op_type() + << std::endl; + // throw; + } + // if (node->kind() != prim::ListConstruct) { + // std::cout << "Making unsupported " << node->kind().toQualString() << std::endl; + // node->dump(); + // } + + // Create PtFrameworkNode for everything that wasn't able to be converted normally + // Pay attention to subgraphs that may appear in the node + // std::cerr << "[ DEBUG ] Before PtFramewokNode creation\n"; + + auto schema = context->get_schema(); + if (schema.find('!') != std::string::npos) { + // Hack. Can indicate mutable inputs, but can it be reliable? + auto fw_node = std::make_shared(context->get_decoder(), + context->inputs(), + context->get_decoder()->num_of_outputs() + 1); + fw_node->set_friendly_name(context->get_op_type() + ":" + std::to_string(COUNTER++)); + auto outputs = fw_node->outputs(); + // update writes to input 0, so we need to replace this input with output from update + context->mutate_input(0, outputs.back()); + std::cerr << "[ WARNING ] Created node with mutated 0 input. Schema: " << schema << std::endl; + context->get_decoder()->mark_node(fw_node); + return outputs; + } + auto fw_node = std::make_shared(context->get_decoder(), + context->inputs(), + context->get_decoder()->num_of_outputs()); + fw_node->set_friendly_name(context->get_op_type() + ":" + std::to_string(COUNTER++)); + + std::map inputs_map; + std::map outputs_map; + std::set input_idxs; + for (size_t i = 0; i < context->get_decoder()->get_subgraph_size(); ++i) { + auto subgraph_decoder = context->get_decoder()->get_subgraph_decoder(i); + auto inputs = subgraph_decoder->inputs(); + input_idxs.insert(inputs.begin(), inputs.end()); + auto body = context->convert_subgraph(i); + fw_node->set_function(i, body); + for (auto param : body->get_parameters()) { + auto name = param->get_output_tensor(0).get_any_name(); + size_t input_idx = (size_t)std::stoll(name); + inputs_map[input_idx].push_back(param); + } + for (auto result : body->get_results()) { + auto name = result->input(0).get_tensor().get_any_name(); + size_t out_idx = (size_t)std::stoll(name); + FRONT_END_OP_CONVERSION_CHECK(outputs_map.count(out_idx) == 0, + "More then one body output with same tensor name."); + outputs_map[out_idx].push_back(result); + } + } + for (auto input : inputs_map) { + if (!input_idxs.count(input.first)) { + auto external_output = context->get_tensor_from_model_or_create_input(input.first); + fw_node->set_invariant_inputs(external_output, input.second); + } else { + auto external_output = context->get_tensor_from_model(input.first); + if (external_output.get_node()) { + fw_node->set_invariant_inputs(external_output, input.second); + } + } + } + for (auto output : outputs_map) { + context->add_tensor_to_context(output.first, fw_node->set_body_outputs(output.second)); + } + return context->get_decoder()->mark_node(fw_node)->outputs(); +} + +std::shared_ptr convert_pytorch_model(std::shared_ptr pytorch_model, + const TensorMap& external_tensor_map) { + LEVEL++; + // std::cout << "=====Convert model:" << LEVEL << " start=====" << std::endl; + std::shared_ptr resulting_model; // define here to make a conversion in a nested scope + { + ParameterVector parameters; + TensorMap tensor_map; // tensor map of the current context + std::set mutated_tensors; + + // Go over all pytorch_model inputs and register them in the tensor map: + auto inputs = pytorch_model->inputs(); + // std::cout << "[ --- DEBUG --- ] convert_pytorch_model: number of inputs: " << inputs.size() << '\n'; + for (int i = 0; i < inputs.size(); ++i) { + // std::cout << "Input: " << i << ": " << inputs[i] << "\n"; + PartialShape ps = pytorch_model->get_input_shape(i); + // std::cout << "PartialShape = " << ps << "\n"; + auto parameter = + std::make_shared(ov::element::custom, pytorch_model->get_input_type(i), ps); + parameter->get_output_tensor(0).add_names({std::to_string(pytorch_model->input(i))}); + // std::cout << "Parameter: " << parameter << "\n"; + parameters.push_back(parameter); + auto order = pytorch_model->get_input_transpose_order(i); + if (order.size() > 0 && !std::is_sorted(order.begin(), order.end())) { + OV_FRONTEND_REQUIRE(ps.is_static()); // TODO: make dynamic + auto sh = ps.get_shape(); + Shape new_shape(sh.size()); + for (int i = 0; i < sh.size(); i++) { + new_shape[order[i]] = sh[i]; + } + auto shape_const = opset8::Constant::create(element::i64, {new_shape.size()}, new_shape); + auto reshape = std::make_shared(parameter, shape_const, false); + auto order_const = opset8::Constant::create(element::i32, {order.size()}, order); + auto transpose = std::make_shared(reshape, order_const); + tensor_map[pytorch_model->input(i)] = transpose; + } else { + tensor_map[pytorch_model->input(i)] = parameter; + } + // std::cout << "Level:" << LEVEL << " Added model input: " << tensor_map[pytorch_model->input(i)] << + // std::endl; + } + + auto node_visitor = [&](std::shared_ptr node) { + // std::cerr << "Node convert start" << std::endl; + + // Explore all inputs of node. Node may refer to input value that hasn't been created in the current scope. + // But this value can be found in the outer scope, for this purpose we need to search node in + // external_tensor_map as well + + auto raw_inputs = node->inputs(); + for (size_t i = 0; i < raw_inputs.size(); ++i) { + auto input = node->input(i); + if (tensor_map.find(input) == tensor_map.end()) { + // std::cout << "Level:" << LEVEL << " Trampoline for input index " << i << " with value " << input + // << "\n"; + // input refers value in the outer scope, need to create a new Parameter in the current scope + // TODO: Connect outer scope and inner scope properly -- should be handled at the level of that + // operation that introduced this nest of scopes (e.g. loop or if) + // TODO: Eliminate duplication with the main code for Parameters creation + // TODO: There is no real search for values in outer scope because we don't need to link the usage + // and definition together at this point -- need to do that otherwise graph will fall apart + PartialShape ps = node->get_input_shape(i); + auto parameter = std::make_shared(node->get_input_type(i), ps); + // TODO: Missing get_input_transpose_order handling for not trivial layouts + tensor_map[input] = parameter; + // std::cout << "Parameter created\n"; + // set name of parameter to the index of node in the model + parameter->get_output_tensor(0).add_names({std::to_string(input)}); + parameters.push_back(parameter); + // std::cout << "External tensor: " << input << " node: " << external_tensor_map.at(input) << + // std::endl; + } + } + // std::cerr << "Node convert before translator: " << node->get_op_type() << ", schema: " << + // node->get_schema() << std::endl; + + auto context = NodeContext(node, &tensor_map, ¶meters, external_tensor_map); + auto converted_outputs = convert_node(&context); + // std::cerr << "Node convert before outputs" << std::endl; + + auto mutated_t = context.get_mutated_tensors(); + mutated_tensors.insert(mutated_t.begin(), mutated_t.end()); + + auto fw_outputs = node->outputs(); + // ops with subgraphs has more outputs + FRONT_END_OP_CONVERSION_CHECK(fw_outputs.size() <= converted_outputs.size(), + "Number of ", + node->get_op_type(), + " outputs greater then number of converted outputs."); + + // TODO: Make sure that mapping of fw_outputs to converted_outputs does always work + // FIXME: Now it is not true for at least prim::Constant + for (size_t i = 0; i < fw_outputs.size(); ++i) { + size_t fw_tensor_id = node->output(i); + if (tensor_map.find(fw_tensor_id) != tensor_map.end()) { + // std::cerr << "Duplicated producer for tensor with id = " << fw_tensor_id << " discovered at + // output " + // << "port " << i << " of node " << node->kind().toQualString() << "\n"; + throw std::runtime_error("Duplicated producer for PT value with unique ID: " + + std::to_string(fw_tensor_id)); + } + + // Output shape of converted node should match the original output shape + // std::cerr << "[ DEBUG ] PT output shape = " << get_ov_shape(fw_outputs[i]) << '\n'; + // std::cerr << "[ DEBUG ] OV output shape = " << converted_outputs[i].get_partial_shape() << '\n'; + // OV_FRONTEND_REQUIRE(get_ov_shape(fw_outputs[i]) == converted_outputs[i].get_partial_shape()); + + tensor_map[fw_tensor_id] = converted_outputs[i]; + converted_outputs[i].get_tensor().add_names({std::to_string(fw_tensor_id)}); + // std::cout << "Level:" << LEVEL << " Added node: " << converted_outputs[i] << std::endl; + // std::cout << "Converted node output " << fw_tensor_id << ": " << converted_outputs[i] << std::endl; + } + // std::cout << "Node convert end" << std::endl; + }; + + OV_FRONTEND_REQUIRE(pytorch_model->get_subgraph_size() == 1); + pytorch_model->visit_subgraph(0, node_visitor); + // std::cout << "All nodes convert end" << std::endl; + + ResultVector results; + // std::cerr << "Outputs:" << pytorch_model->num_of_outputs() << "\n"; + for (size_t i = 0; i < pytorch_model->num_of_outputs(); ++i) { + size_t id = pytorch_model->output(i); + // std::cerr << "Output:" << i << ": " << id << "\n"; + // std::cout << "value = " << id << '\n'; + // std::cout << "X\n"; + if (tensor_map.find(id) == tensor_map.end()) { + // Not found in this scope, searching in the outer scope + // TODO: do real search here, skipped for now + + auto parameter = std::make_shared(element::dynamic, PartialShape::dynamic()); + parameter->get_output_tensor(0).add_names({std::to_string(id)}); + parameters.push_back(parameter); + tensor_map[id] = parameter; + // std::cout << "Level:" << LEVEL << "Added new parameter based on external value " << id << "\n"; + } + auto ov_output = tensor_map[id]; + // std::cout << "X\n"; + auto order = pytorch_model->get_output_transpose_order(i); + // std::cout << "X\n"; + if (order.size() > 0 && !std::is_sorted(order.begin(), order.end())) { + throw "Output strides have wrong order."; + } + // TODO: remove when all nodes has ids + ov_output.add_names({std::to_string(id)}); + // std::cout << "X\n"; + // std::cout << ov_output << '\n'; + auto result = std::make_shared(ov_output); + // std::cout << "X\n"; + results.push_back(result); + // std::cerr << "Model result " << result << "\n"; + } + + // Since parameters can be added we need to list all current parameters + std::set param_names; + for (auto param : parameters) { + auto name = param->get_output_tensor(0).get_any_name(); + size_t input_idx = (size_t)std::stoll(name); + param_names.insert(input_idx); + } + for (auto tensor : mutated_tensors) { + if (param_names.count(tensor)) { + OV_FRONTEND_REQUIRE(tensor_map.count(tensor)); + // model input was mutated we need to make a result for it + results.push_back(std::make_shared(tensor_map.at(tensor))); + } + } + // std::cout << "Y\n"; + + /*for (size_t i = 0; i < parameters.size(); ++i) { + auto parameter = parameters[i]; + // std::cerr << "parameter[" << i << "].shape = " + // << parameter->get_output_shape(0) << ", consumers: " << + // parameter->output(0).get_target_inputs().size() << "\n"; + }*/ + // std::cout << "Convert end" << std::endl; + // std::cout << "Number of values collected: " << tensor_map.size() << "\n"; + + // std::cout << "=====Construct model start=====" << std::endl; + /*std::cout << "=====Tensor map start=====" << std::endl; + for (auto node : tensor_map) { + std::cout << node.first << ": " << node.second.get_node_shared_ptr() << std::endl; + }*/ + resulting_model = std::make_shared(results, parameters); + /*std::string m_name = "model_" + std::to_string(LEVEL) + "_" + std::to_string(NUMBER++); + try { + ov::serialize(resulting_model, m_name + ".xml", m_name + ".bin"); + } catch (...) { + std::cout << "Exception happened during model serialization: " + m_name << std::endl; + }*/ + // std::cout << "=====Construct model end=====" << std::endl; + + // Did a conversion in a nested scope to automatically remove any holders of nodes except those in the graph + } + + // std::cout << "=====Convert model:" << LEVEL << " end=====" << std::endl; + LEVEL--; + return resulting_model; +} + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/utils.hpp b/src/frontends/pytorch/src/utils.hpp new file mode 100644 index 00000000000000..125ca93e3bc05c --- /dev/null +++ b/src/frontends/pytorch/src/utils.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include "openvino/frontend/pytorch/node_context.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { + +Output make_optional_bias(Output base_op, + const NodeContext& context, + size_t bias_input_idx, + std::vector unsqueeze_dims = {}); + +std::shared_ptr get_rank_node(ov::Output node); + +Output reshape_kernel_for_group(const NodeContext& context, + Output input, + Output kernel, + int64_t groups); + +std::shared_ptr convert_pytorch_model(std::shared_ptr pytorch_model, + const TensorMap& external_tensor_map = {}); + +OutputVector convert_node(NodeContext* context); + +template +OutputVector inplace_op(NodeContext& context) { + auto translation_res = T(context); + FRONT_END_OP_CONVERSION_CHECK(translation_res.size() == 1, + "inplace_op function must be used on single output translators"); + context.mutate_input(idx, translation_res[0]); + return translation_res; +} + +} // namespace pytorch +} // namespace frontend +} // namespace ov