diff --git a/lib/elixirpb.pb.ex b/lib/elixirpb.pb.ex index 0d64c696..ac7eda28 100644 --- a/lib/elixirpb.pb.ex +++ b/lib/elixirpb.pb.ex @@ -10,9 +10,25 @@ defmodule Elixirpb.FileOptions do field :module_prefix, 1, optional: true, type: :string end +defmodule Elixirpb.MessageOptions do + @moduledoc false + use Protobuf, syntax: :proto2 + + @type t :: %__MODULE__{ + typespec: String.t() + } + defstruct [:typespec] + + field :typespec, 1, optional: true, type: :string +end + defmodule Elixirpb.PbExtension do @moduledoc false use Protobuf, syntax: :proto2 extend Google.Protobuf.FileOptions, :file, 1047, optional: true, type: Elixirpb.FileOptions + + extend Google.Protobuf.MessageOptions, :message, 1047, + optional: true, + type: Elixirpb.MessageOptions end diff --git a/lib/protobuf/builder.ex b/lib/protobuf/builder.ex index e13124a5..74c739d0 100644 --- a/lib/protobuf/builder.ex +++ b/lib/protobuf/builder.ex @@ -71,9 +71,9 @@ defmodule Protobuf.Builder do v = if f_props.embedded? do if f_props.repeated? do - Enum.map(v, fn i -> f_props.type.new(i) end) + Enum.map(v, &protobuf_or_term(&1, f_props.type)) else - f_props.type.new(v) + protobuf_or_term(v, f_props.type) end else v @@ -86,4 +86,9 @@ defmodule Protobuf.Builder do end end) end + + defp protobuf_or_term(value, type), + do: if(encodable?(value), do: value, else: type.new(value)) + + defp encodable?(v), do: Protobuf.Encodable.impl_for(v) != Protobuf.Encodable.Any end diff --git a/lib/protobuf/decodable.ex b/lib/protobuf/decodable.ex new file mode 100644 index 00000000..a4982551 --- /dev/null +++ b/lib/protobuf/decodable.ex @@ -0,0 +1,33 @@ +defprotocol Protobuf.Decodable do + @moduledoc """ + Defines the contract for transformations after decode a message. + + Implementing this protocol is useful to translate protobuf structs to Elixir + terms. + + ## Examples + + defimpl Protobuf.Decodable, for: MyApp.Protobuf.Date do + def to_elixir(%MyApp.Protobuf.Date{year: year, month: month, day: day}) do + {:ok, date} = Date.new(year, month, day) + date + end + end + + # later in a decoded message + proto_message.birthday + ~D[1988-10-29] + """ + @fallback_to_any true + + @doc """ + This function will be called after decode the protobuf message binary. The + returning value will be used in place of current `term` struct. + """ + @spec to_elixir(t) :: any + def to_elixir(term) +end + +defimpl Protobuf.Decodable, for: Any do + def to_elixir(term), do: term +end diff --git a/lib/protobuf/decoder.ex b/lib/protobuf/decoder.ex index 8cd2b9d4..5655a622 100644 --- a/lib/protobuf/decoder.ex +++ b/lib/protobuf/decoder.ex @@ -2,6 +2,9 @@ defmodule Protobuf.Decoder do @moduledoc false import Protobuf.WireTypes import Bitwise, only: [bsl: 2, bsr: 2, band: 2] + + alias Protobuf.Decodable + require Logger @max_bits 64 @@ -14,7 +17,10 @@ defmodule Protobuf.Decoder do kvs = raw_decode_key(data, []) %{repeated_fields: repeated_fields} = msg_props = module.__message_props__() struct = build_struct(kvs, msg_props, module.new()) - reverse_repeated(struct, repeated_fields) + + struct + |> Decodable.to_elixir() + |> reverse_repeated(repeated_fields) end @doc false diff --git a/lib/protobuf/encodable.ex b/lib/protobuf/encodable.ex new file mode 100644 index 00000000..196a787a --- /dev/null +++ b/lib/protobuf/encodable.ex @@ -0,0 +1,34 @@ +defprotocol Protobuf.Encodable do + @moduledoc """ + Defines the contract for Elixir terms transformations before encode a message. + + Implementing this protocol is useful to translate Elixir terms to protobuf + structs, works in combination with `Protobuf.Decodable`. + + ## Examples + + defimpl Protobuf.Encodable, for: Date do + def to_protobuf(%Date{year: year, month: month, day: day}, MyApp.Protobuf.Date) do + MyApp.Protobuf.Date.new(year: year, month: month, day: day) + end + end + + # later, you can use Elixir terms in your fields and those will be + # converted to protobuf structs before binary encoding + %{protobuf_message | birthday: ~D[1988-10-29]} + + """ + @fallback_to_any true + + @doc """ + This function will invoked before encode a term and only if encoding target is + a protobuf message. The returning value will be used in place of current + Elixir `term` struct. + """ + @spec to_protobuf(t, module) :: struct + def to_protobuf(term, target_protobuf_module) +end + +defimpl Protobuf.Encodable, for: Any do + def to_protobuf(term, _target_protobuf_module), do: term +end diff --git a/lib/protobuf/encoder.ex b/lib/protobuf/encoder.ex index dc48df72..0bf03f9c 100644 --- a/lib/protobuf/encoder.ex +++ b/lib/protobuf/encoder.ex @@ -3,7 +3,7 @@ defmodule Protobuf.Encoder do import Protobuf.WireTypes import Bitwise, only: [bsr: 2, band: 2, bsl: 2, bor: 2] - alias Protobuf.{MessageProps, FieldProps} + alias Protobuf.{Encodable, MessageProps, FieldProps} @spec encode(atom, map | struct, keyword) :: iodata def encode(mod, msg, opts) do @@ -111,7 +111,9 @@ defmodule Protobuf.Encoder do ) do repeated = is_repeated || is_map - repeated_or_not(val, repeated, fn v -> + val + |> Encodable.to_protobuf(type) + |> repeated_or_not(repeated, fn v -> v = if is_map, do: struct(prop.type, %{key: elem(v, 0), value: elem(v, 1)}), else: v # so that oneof {:atom, v} can be encoded encoded = encode(type, v, []) diff --git a/lib/protobuf/protoc/cli.ex b/lib/protobuf/protoc/cli.ex index 58a15047..6a7c6fab 100644 --- a/lib/protobuf/protoc/cli.ex +++ b/lib/protobuf/protoc/cli.ex @@ -105,13 +105,13 @@ defmodule Protobuf.Protoc.CLI do new_ctx = append_ns(ctx, name) types - |> update_types(ctx, name) + |> update_types(ctx, desc) |> find_types_in_proto(new_ctx, desc.enum_type) |> find_types_in_proto(new_ctx, desc.nested_type) end - defp find_types_in_proto(types, ctx, %Google.Protobuf.EnumDescriptorProto{name: name}) do - update_types(types, ctx, name) + defp find_types_in_proto(types, ctx, desc) do + update_types(types, ctx, desc) end defp append_ns(%{namespace: ns} = ctx, name) do @@ -119,12 +119,25 @@ defmodule Protobuf.Protoc.CLI do Map.put(ctx, :namespace, new_ns) end - defp update_types(types, %{namespace: ns, package: pkg, module_prefix: prefix}, name) do - type_name = - join_names(prefix || pkg, ns, name) - |> Protobuf.Protoc.Generator.Util.normalize_type_name() + defp update_types(types, %{namespace: ns, package: pkg, module_prefix: prefix}, desc) do + name = desc.name + module_name = gen_module_name(prefix, pkg, ns, name) - Map.put(types, "." <> join_names(pkg, ns, name), %{type_name: type_name}) + typespec = + desc.options + |> get_msg_options() + |> Map.get(:typespec) + + Map.put(types, "." <> join_names(pkg, ns, name), %{ + type_name: module_name, + typespec: typespec + }) + end + + defp gen_module_name(prefix, pkg, ns, name) do + (prefix || pkg) + |> join_names(ns, name) + |> Protobuf.Protoc.Generator.Util.normalize_type_name() end defp join_names(pkg, ns, name) do @@ -134,4 +147,16 @@ defmodule Protobuf.Protoc.CLI do |> Enum.filter(&(&1 && &1 != "")) |> Enum.join(".") end + + defp get_msg_options(nil), do: %{} + + defp get_msg_options(options) do + case Google.Protobuf.MessageOptions.get_extension(options, Elixirpb.PbExtension, :message) do + nil -> + %{} + + opts -> + opts + end + end end diff --git a/lib/protobuf/protoc/generator/message.ex b/lib/protobuf/protoc/generator/message.ex index 33d631a5..5c7a76f7 100644 --- a/lib/protobuf/protoc/generator/message.ex +++ b/lib/protobuf/protoc/generator/message.ex @@ -152,12 +152,21 @@ defmodule Protobuf.Protoc.Generator.Message do "%{#{k_type} => #{v_type}}" end - defp fmt_type(%{label: "repeated", type_enum: type_enum, type: type}) do - "[#{type_to_spec(type_enum, type, true)}]" + defp fmt_type(%{label: "repeated", type_enum: type_enum, type: type, typespec: typespec}) do + "[#{typespec || type_to_spec(type_enum, type, true)}]" end - defp fmt_type(%{type_enum: type_enum, type: type}) do - "#{type_to_spec(type_enum, type)}" + defp fmt_type(%{type_enum: type_enum, type: type, typespec: typespec}) do + cond do + type_enum == :TYPE_MESSAGE and typespec -> + typespec <> " | nil" + + typespec -> + typespec + + true -> + "#{type_to_spec(type_enum, type)}" + end end defp type_to_spec(enum, type, repeated \\ false) @@ -188,12 +197,14 @@ defmodule Protobuf.Protoc.Generator.Message do opts_str = if opts_str == "", do: "", else: ", " <> opts_str type = field_type_name(ctx, f) + typespec = field_typespec(ctx, f) %{ name: f.name, number: f.number, label: label_name(f.label), type: type, + typespec: typespec, type_enum: f.type, opts: opts, opts_str: opts_str, @@ -212,12 +223,17 @@ defmodule Protobuf.Protoc.Generator.Message do type = TypeUtil.from_enum(f.type) if f.type_name && (type == :enum || type == :message) do - Util.type_from_type_name(ctx, f.type_name) + Util.get_metadata_from_type_name(ctx, f.type_name)[:type_name] else ":#{type}" end end + defp field_typespec(_ctx, %{type_name: nil} = _field), do: nil + + defp field_typespec(ctx, %{type_name: type_name} = _field), + do: Util.get_metadata_from_type_name(ctx, type_name)[:typespec] + # Map of protobuf are actually nested(one level) messages defp nested_maps(ctx, desc) do full_name = Util.join_name([ctx.package | ctx.namespace] ++ [desc.name]) diff --git a/lib/protobuf/protoc/generator/util.ex b/lib/protobuf/protoc/generator/util.ex index a9ac87a5..eb7b35e4 100644 --- a/lib/protobuf/protoc/generator/util.ex +++ b/lib/protobuf/protoc/generator/util.ex @@ -31,14 +31,20 @@ defmodule Protobuf.Protoc.Generator.Util do |> Enum.join(", ") end - def type_from_type_name(ctx, type_name) do + def module_from_type_name(ctx, type_name), + do: get_metadata_from_type_name(ctx, type_name)[:module_name] + + def type_from_type_name(ctx, type_name), + do: get_metadata_from_type_name(ctx, type_name)[:type_name] + + def get_metadata_from_type_name(ctx, type_name) do # The doc says there's a situation where type_name begins without a `.`, but I never got that. # Handle that later. metadata = ctx.dep_type_mapping[type_name] || raise "There's something wrong to get #{type_name}'s type, please contact with the lib author." - metadata[:type_name] + metadata end def normalize_type_name(name) do diff --git a/src/elixirpb.proto b/src/elixirpb.proto index 9fa51d26..2e12930a 100644 --- a/src/elixirpb.proto +++ b/src/elixirpb.proto @@ -25,6 +25,35 @@ message FileOptions { optional string module_prefix = 1; } +// Message level options +// +// For example, +// option (elixirpb.message).typespec = "Date.t"; +message MessageOptions { + // Specify a typespec that will used when a message reference this as field. + // For example, let's say you have in your message: + // + // package MyApp.Protobuf; + // + // message Date { + // option (elixirpb.message).typespec = "Date.t"; + // int iso_days = 1; + // } + // + // message User { + // Date birthday = 1; + // } + // + // Then in `MyApp.Protobuf.User`, the type notation will for `birthday` will + // be `Date.t()` and not `MyApp.Protobuf.Date.t()`. This is useulf when + // combined with `Protobuf.Encodable` and `Protobuf.Decodable` mechanism. + optional string typespec = 1; +} + extend google.protobuf.FileOptions { optional FileOptions file = 1047; } + +extend google.protobuf.MessageOptions { + optional MessageOptions message = 1047; +} diff --git a/test/protobuf/decoder_test.exs b/test/protobuf/decoder_test.exs index 19662da1..646e6608 100644 --- a/test/protobuf/decoder_test.exs +++ b/test/protobuf/decoder_test.exs @@ -140,4 +140,8 @@ defmodule Protobuf.DecoderTest do assert Decoder.decode(<<18, 0, 24, 0>>, TestMsg.Oneof) == TestMsg.Oneof.new(first: {:b, ""}, second: {:c, 0}) end + + test "transforms to elixir representation after decode the message" do + assert Decoder.decode(<<8, 132, 171, 44>>, TestMsg.DateFoo) == ~D[1988-10-29] + end end diff --git a/test/protobuf/encoder_test.exs b/test/protobuf/encoder_test.exs index c7f988a7..b882ee0e 100644 --- a/test/protobuf/encoder_test.exs +++ b/test/protobuf/encoder_test.exs @@ -181,4 +181,16 @@ defmodule Protobuf.EncoderTest do msg = TestMsg.Bar2.new(a: 0, b: 1) assert Encoder.encode(msg) == <<8, 0, 16, 1>> end + + test "transforms from Elixir term to protobuf before encode a message" do + msg = %TestMsg.FooWithDate{date: ~D[1988-10-29]} + + assert Encoder.encode(msg) == <<10, 4, 8, 132, 171, 44>> + end + + test "transforms from Elixir term to protobuf before encode a message using new function" do + msg = TestMsg.FooWithDate.new(date: ~D[1988-10-29]) + + assert Encoder.encode(msg) == <<10, 4, 8, 132, 171, 44>> + end end diff --git a/test/protobuf/protoc/generator/message_test.exs b/test/protobuf/protoc/generator/message_test.exs index 605b1780..0f56d9c6 100644 --- a/test/protobuf/protoc/generator/message_test.exs +++ b/test/protobuf/protoc/generator/message_test.exs @@ -212,8 +212,14 @@ defmodule Protobuf.Protoc.Generator.MessageTest do ctx = %Context{ package: "foo_bar.ab_cd", dep_type_mapping: %{ - ".foo_bar.ab_cd.Foo.ProjectsEntry" => %{type_name: "FooBar.AbCd.Foo.ProjectsEntry"}, - ".foo_bar.ab_cd.Bar" => %{type_name: "FooBar.AbCd.Bar"} + ".foo_bar.ab_cd.Foo.ProjectsEntry" => %{ + type_name: "FooBar.AbCd.Foo.ProjectsEntry", + typespec: "FooBar.AbCd.Foo.ProjectsEntry" + }, + ".foo_bar.ab_cd.Bar" => %{ + type_name: "FooBar.AbCd.Bar", + typespec: "FooBar.AbCd.Bar" + } }, module_prefix: "FooBar.AbCd" } @@ -262,7 +268,9 @@ defmodule Protobuf.Protoc.Generator.MessageTest do ctx = %Context{ package: "foo_bar.ab_cd", dep_type_mapping: %{ - ".foo_bar.ab_cd.EnumFoo" => %{type_name: "FooBar.AbCd.EnumFoo"} + ".foo_bar.ab_cd.EnumFoo" => %{ + type_name: "FooBar.AbCd.EnumFoo" + } } } @@ -288,7 +296,12 @@ defmodule Protobuf.Protoc.Generator.MessageTest do test "generate/2 generate right enum type name with different package" do ctx = %Context{ package: "foo_bar.ab_cd", - dep_type_mapping: %{".other_pkg.EnumFoo" => %{type_name: "OtherPkg.EnumFoo"}} + dep_type_mapping: %{ + ".other_pkg.EnumFoo" => %{ + type_name: "OtherPkg.EnumFoo", + typespec: "OtherPkg.EnumFoo" + } + } } desc = @@ -312,7 +325,12 @@ defmodule Protobuf.Protoc.Generator.MessageTest do test "generate/2 generate right message type name with different package" do ctx = %Context{ package: "foo_bar.ab_cd", - dep_type_mapping: %{".other_pkg.MsgFoo" => %{type_name: "OtherPkg.MsgFoo"}} + dep_type_mapping: %{ + ".other_pkg.MsgFoo" => %{ + type_name: "OtherPkg.MsgFoo", + typespec: "CustomType.t" + } + } } desc = @@ -330,7 +348,7 @@ defmodule Protobuf.Protoc.Generator.MessageTest do ) {[], [msg]} = Generator.generate(ctx, desc) - assert msg =~ "a: OtherPkg.MsgFoo.t" + assert msg =~ "a: CustomType.t" assert msg =~ "field :a, 1, optional: true, type: OtherPkg.MsgFoo\n" end diff --git a/test/protobuf/protoc/integration_test.exs b/test/protobuf/protoc/integration_test.exs index f4f70498..98b136b9 100644 --- a/test/protobuf/protoc/integration_test.exs +++ b/test/protobuf/protoc/integration_test.exs @@ -55,4 +55,48 @@ defmodule Protobuf.Protoc.IntegrationTest do test "extensions" do assert "hello" == Protobuf.Protoc.ExtTest.Foo.new(a: "hello").a end + + describe "message extensions" do + setup do + %{protobuf_file: File.read!("test/protobuf/protoc/proto_gen/extension.pb.ex")} + end + + test "uses the option type_name", %{protobuf_file: protobuf_file} do + assert protobuf_file =~ "inserted_at: DateTime.t() | nil\n" + end + + test "keeps the same module", %{protobuf_file: protobuf_file} do + assert protobuf_file =~ "type: Protobuf.Protoc.ExtTest.UnixDateTime\n" + end + end + + describe "encodable protocol" do + test "transform to protobuf before" do + msg = + Protobuf.Protoc.ExtTest.FooWithUnixDateTime.new( + inserted_at: udatetime(~N[2020-04-03 11:28:21.929987]) + ) + + assert Protobuf.encode(msg) == + <<10, 9, 8, 131, 160, 134, 184, 147, 204, 232, 2>> + end + end + + describe "decodable protocol" do + test "transforms to elixir after decoding" do + decoded = + Protobuf.Protoc.ExtTest.FooWithUnixDateTime.decode( + <<10, 9, 8, 131, 160, 134, 184, 147, 204, 232, 2>> + ) + + assert decoded == + Protobuf.Protoc.ExtTest.FooWithUnixDateTime.new( + inserted_at: udatetime(~N[2020-04-03 11:28:21.929987Z]) + ) + end + end + + def udatetime(time) do + DateTime.from_naive!(time, "Etc/UTC") + end end diff --git a/test/protobuf/protoc/proto/extension.proto b/test/protobuf/protoc/proto/extension.proto index 6d3be1f4..095306d1 100644 --- a/test/protobuf/protoc/proto/extension.proto +++ b/test/protobuf/protoc/proto/extension.proto @@ -10,3 +10,13 @@ option (elixirpb.file).module_prefix = "Protobuf.Protoc.ExtTest"; message Foo { optional string a = 1; } + +message UnixDateTime { + option (elixirpb.message).typespec = "DateTime.t"; + + required int64 microseconds = 1; +} + +message FooWithUnixDateTime { + optional UnixDateTime inserted_at = 1; +} diff --git a/test/protobuf/protoc/proto_gen/extension.pb.ex b/test/protobuf/protoc/proto_gen/extension.pb.ex index 8fbddfe9..a46f6772 100644 --- a/test/protobuf/protoc/proto_gen/extension.pb.ex +++ b/test/protobuf/protoc/proto_gen/extension.pb.ex @@ -9,3 +9,27 @@ defmodule Protobuf.Protoc.ExtTest.Foo do field :a, 1, optional: true, type: :string end + +defmodule Protobuf.Protoc.ExtTest.UnixDateTime do + @moduledoc false + use Protobuf, syntax: :proto2 + + @type t :: %__MODULE__{ + microseconds: integer + } + defstruct [:microseconds] + + field :microseconds, 1, required: true, type: :int64 +end + +defmodule Protobuf.Protoc.ExtTest.FooWithUnixDateTime do + @moduledoc false + use Protobuf, syntax: :proto2 + + @type t :: %__MODULE__{ + inserted_at: DateTime.t() | nil + } + defstruct [:inserted_at] + + field :inserted_at, 1, optional: true, type: Protobuf.Protoc.ExtTest.UnixDateTime +end diff --git a/test/support/decodables.ex b/test/support/decodables.ex new file mode 100644 index 00000000..5ac975d0 --- /dev/null +++ b/test/support/decodables.ex @@ -0,0 +1,15 @@ +defimpl Protobuf.Decodable, for: TestMsg.DateFoo do + def to_elixir(%TestMsg.DateFoo{iso_days: iso_days}) do + {year, month, day, _, _, _, _} = + Calendar.ISO.naive_datetime_from_iso_days({iso_days, {0, 86_400_000_000}}) + + {:ok, date} = Date.new(year, month, day) + date + end +end + +defimpl Protobuf.Decodable, for: Protobuf.Protoc.ExtTest.UnixDateTime do + def to_elixir(%Protobuf.Protoc.ExtTest.UnixDateTime{microseconds: microseconds}) do + DateTime.from_unix!(microseconds, :microsecond) + end +end diff --git a/test/support/encodables.ex b/test/support/encodables.ex new file mode 100644 index 00000000..2088b167 --- /dev/null +++ b/test/support/encodables.ex @@ -0,0 +1,16 @@ +defimpl Protobuf.Encodable, for: Date do + def to_protobuf( + %Date{calendar: calendar, year: year, month: month, day: day}, + TestMsg.DateFoo + ) do + {iso_days, _} = calendar.naive_datetime_to_iso_days(year, month, day, 0, 0, 0, {0, 6}) + %TestMsg.DateFoo{iso_days: iso_days} + end +end + +defimpl Protobuf.Encodable, for: DateTime do + def to_protobuf(%DateTime{} = datetime, Protobuf.Protoc.ExtTest.UnixDateTime) do + microseconds = DateTime.to_unix(datetime, :microsecond) + %Protobuf.Protoc.ExtTest.UnixDateTime{microseconds: microseconds} + end +end diff --git a/test/support/test_msg.ex b/test/support/test_msg.ex index 9d60154d..73a63143 100644 --- a/test/support/test_msg.ex +++ b/test/support/test_msg.ex @@ -77,6 +77,22 @@ defmodule TestMsg do field :non_matched, 101, type: :int32, optional: true end + defmodule DateFoo do + use Protobuf, syntax: :proto3 + + defstruct [:iso_days] + + field :iso_days, 1, type: :int32 + end + + defmodule FooWithDate do + use Protobuf, syntax: :proto3 + + defstruct [:date] + + field :date, 1, type: DateFoo + end + defmodule Oneof do use Protobuf