From fd4269b3fe538efb37cc7ab484bed132cb04f3ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Quaresma?= Date: Mon, 10 Jan 2022 15:11:14 +0100 Subject: [PATCH 1/2] wip: adding embbed anonymization --- lib/ecto_anon.ex | 2 +- lib/ecto_anon/anonymizer.ex | 14 ++++++-- mix.exs | 1 + mix.lock | 1 + ...220105150654_create_users_associations.exs | 1 + test/ecto_anon_test.exs | 36 ++++++++++++++----- test/support/comment.ex | 16 +++++++++ 7 files changed, 59 insertions(+), 12 deletions(-) diff --git a/lib/ecto_anon.ex b/lib/ecto_anon.ex index 9ccd356..4b6ce2d 100644 --- a/lib/ecto_anon.ex +++ b/lib/ecto_anon.ex @@ -85,7 +85,7 @@ defmodule EctoAnon do anon_fields = mod.__anon_fields__() |> Enum.map(fn {field, _} -> field end) associations = - mod.__schema__(:associations) + (mod.__schema__(:associations) ++ mod.__schema__(:embeds)) |> Enum.filter(&(&1 in anon_fields and EctoAnon.Anonymizer.is_association?(mod, &1))) struct = repo.preload(struct, associations) diff --git a/lib/ecto_anon/anonymizer.ex b/lib/ecto_anon/anonymizer.ex index 857c1de..e8f3b6c 100644 --- a/lib/ecto_anon/anonymizer.ex +++ b/lib/ecto_anon/anonymizer.ex @@ -6,9 +6,18 @@ defmodule EctoAnon.Anonymizer do end end - defp anonymizable_struct?(%module{}), do: {:__anon_fields__, 0} in module.__info__(:functions) + defp anonymizable_struct?(%module{}) do + {:__anon_fields__, 0} in module.__info__(:functions) + end defp get_anonymized_data(%module{} = struct) do + # embeds = + # module.__schema__(:embeds) + # |> Enum.reduce([], fn embed, acc -> + # %mod{} = Map.get(struct, embed) + # mod.__anon_fields__() ++ acc + # end) + module.__anon_fields__() |> Enum.reject(fn {field, _} -> is_association?(module, field) end) |> Enum.reduce([], fn {field, {func, opts}}, acc -> @@ -30,7 +39,8 @@ defmodule EctoAnon.Anonymizer do association.__struct__ in [ Ecto.Association.Has, Ecto.Association.ManyToMany, - Ecto.Association.HasThrough + Ecto.Association.HasThrough, + Ecto.Embedded ] else _ -> false diff --git a/mix.exs b/mix.exs index 8c88ddb..7149f7b 100644 --- a/mix.exs +++ b/mix.exs @@ -27,6 +27,7 @@ defmodule EctoAnon.MixProject do [ {:ecto, ">= 3.7.1"}, {:ecto_sqlite3, "~> 0.7.1", only: :test}, + {:jason, "~> 1.3", only: :test}, {:mix_test_watch, "~> 1.0", only: [:dev, :test], runtime: false} ] end diff --git a/mix.lock b/mix.lock index 704d5a1..ab22959 100644 --- a/mix.lock +++ b/mix.lock @@ -8,6 +8,7 @@ "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, "exqlite": {:hex, :exqlite, "0.8.4", "6cbfcd15d1307727615c8ce33589a0498e34c1020a3401bba4dd9ae723e99cc6", [:make, :mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "791fee489e0529b24218d49f39e3a9ed64fd2d26a2110a3fab3e26d8c8524b44"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, + "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, "mix_test_watch": {:hex, :mix_test_watch, "1.1.0", "330bb91c8ed271fe408c42d07e0773340a7938d8a0d281d57a14243eae9dc8c3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "52b6b1c476cbb70fd899ca5394506482f12e5f6b0d6acff9df95c7f1e0812ec3"}, "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, } diff --git a/test/db/priv/repo/migrations/20220105150654_create_users_associations.exs b/test/db/priv/repo/migrations/20220105150654_create_users_associations.exs index edd4d3c..9db3c42 100644 --- a/test/db/priv/repo/migrations/20220105150654_create_users_associations.exs +++ b/test/db/priv/repo/migrations/20220105150654_create_users_associations.exs @@ -6,6 +6,7 @@ defmodule EctoAnon.Repo.Migrations.CreateUsersAssociations do create table(:comments) do add(:content, :string) add(:tag, :string) + add(:quote, :map) add(:author_id, references(:users)) anonymized() diff --git a/test/ecto_anon_test.exs b/test/ecto_anon_test.exs index 793dd07..e44f3ed 100644 --- a/test/ecto_anon_test.exs +++ b/test/ecto_anon_test.exs @@ -1,6 +1,6 @@ defmodule EctoAnonTest do use ExUnit.Case, async: true - alias EctoAnon.{Repo, User, Comment} + alias EctoAnon.{Repo, User, Comment, Comment.Quote} defmodule UnknownStruct do defstruct name: "John", age: 27 @@ -48,12 +48,16 @@ defmodule EctoAnonTest do } |> Repo.insert!() - %Comment{ - content: "this is a comment", - tag: "tag", - author_id: user.id - } - |> Repo.insert!() + comment = + %Comment{ + content: "this is a comment", + tag: "tag", + author_id: user.id, + quote: %Quote{ + quote: "this is a quote" + } + } + |> Repo.insert!() {:ok, user: user, mick: mick, emilie: emilie} end @@ -76,8 +80,19 @@ defmodule EctoAnonTest do mick: mick, emilie: emilie } do + comment = + %Comment{ + content: "this is a comment", + tag: "tag", + author_id: user.id, + quote: %Quote{ + quote: "this is a quote" + } + } + |> Repo.insert!() + assert {:ok, updated_user} = - Repo.get(User, user.id) + Repo.get(Comment, comment.id) |> EctoAnon.run(Repo, cascade: true) assert %User{ @@ -89,7 +104,10 @@ defmodule EctoAnonTest do %Comment{ content: "this is a comment", - tag: "tag" + tag: "tag", + quote: %Quote{ + quote: "redacted" + } } = Repo.get_by(Comment, author_id: user.id) %User{ diff --git a/test/support/comment.ex b/test/support/comment.ex index 8cda26f..38b9442 100644 --- a/test/support/comment.ex +++ b/test/support/comment.ex @@ -11,6 +11,22 @@ defmodule EctoAnon.Comment do schema "comments" do field(:content, :string) field(:tag, :string) + + embeds_one(:quote, EctoAnon.Comment.Quote) + belongs_to(:users, User, foreign_key: :author_id) end end + +defmodule EctoAnon.Comment.Quote do + use Ecto.Schema + use EctoAnon.Schema + + import Ecto.Changeset + + embedded_schema do + anon_field(:quote, :string) + end + + def changeset(changeset, attrs), do: cast(changeset, attrs, [:quote]) +end From 5c58c09ff02cad13a70307616db2b0ad30cabe74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Quaresma?= Date: Tue, 31 May 2022 18:44:47 +0200 Subject: [PATCH 2/2] feat: added embeds support --- lib/ecto_anon.ex | 11 +- lib/ecto_anon/anonymizer.ex | 64 ++++-- lib/ecto_anon/query.ex | 13 ++ .../20211213090654_create_users.exs | 2 + test/ecto_anon/anonymizer_test.exs | 14 +- test/ecto_anon/schema_test.exs | 2 + test/ecto_anon_test.exs | 184 ++++++++++++------ test/support/comment.ex | 15 -- test/support/user.ex | 19 ++ 9 files changed, 227 insertions(+), 97 deletions(-) diff --git a/lib/ecto_anon.ex b/lib/ecto_anon.ex index 4b6ce2d..98ce723 100644 --- a/lib/ecto_anon.ex +++ b/lib/ecto_anon.ex @@ -4,7 +4,8 @@ defmodule EctoAnon do """ @doc """ - Updates an Ecto struct with anonymized data based on its anon_fields declared in the struct schema. + Updates an Ecto struct with anonymized data based on its anon_schema declaration. + It also updates any embedded schema declared in anon_schema. defmodule User do use Ecto.Schema @@ -31,16 +32,10 @@ defmodule EctoAnon do ## Options - <<<<<<< HEAD * `:cascade` - When set to `true`, allows ecto_anon to preload and anonymize all associations (and associations of these associations) automatically in cascade. Could be used to anonymize all data related a struct in a single call. Note that this won't traverse `belongs_to` associations to avoid infinite and cyclic anonymizations. - ======= - * `:cascade` - When set to `true`, allows ecto-anon to preload and anonymize - all associations (and associations of these associations) automatically in cascade. - Could be used to anonymize all data related a struct in a single call. - Note that this won't traverse `belongs_to` associations. Default: false * `:log`- When set to `true`, it will set `anonymized` field when EctoAnon.run @@ -85,7 +80,7 @@ defmodule EctoAnon do anon_fields = mod.__anon_fields__() |> Enum.map(fn {field, _} -> field end) associations = - (mod.__schema__(:associations) ++ mod.__schema__(:embeds)) + mod.__schema__(:associations) |> Enum.filter(&(&1 in anon_fields and EctoAnon.Anonymizer.is_association?(mod, &1))) struct = repo.preload(struct, associations) diff --git a/lib/ecto_anon/anonymizer.ex b/lib/ecto_anon/anonymizer.ex index e8f3b6c..bcd7548 100644 --- a/lib/ecto_anon/anonymizer.ex +++ b/lib/ecto_anon/anonymizer.ex @@ -10,40 +10,80 @@ defmodule EctoAnon.Anonymizer do {:__anon_fields__, 0} in module.__info__(:functions) end - defp get_anonymized_data(%module{} = struct) do - # embeds = - # module.__schema__(:embeds) - # |> Enum.reduce([], fn embed, acc -> - # %mod{} = Map.get(struct, embed) - # mod.__anon_fields__() ++ acc - # end) + defp get_anonymized_data(struct) do + fields = anonymize_fields(struct) + embeds = anonymize_embeds(struct) + %{fields: fields, embeds: embeds} + end + + defp anonymize_fields(%module{} = struct) do module.__anon_fields__() - |> Enum.reject(fn {field, _} -> is_association?(module, field) end) + |> Enum.reject(fn {field, _} -> + is_association?(module, field) or is_embed?(module, field) + end) |> Enum.reduce([], fn {field, {func, opts}}, acc -> type = module.__schema__(:type, field) value = case Map.get(struct, field) do - nil -> nil - value -> func.(type, value, opts) + nil -> + nil + + value -> + func.(type, value, opts) end Keyword.put(acc, field, value) end) end + defp anonymize_embeds(%module{} = struct) do + module.__anon_fields__() + |> Enum.filter(fn {field, _} -> is_embed?(module, field) end) + |> Enum.reduce([], fn {field, _}, acc -> + case Map.get(struct, field) do + nil -> acc + value -> Keyword.put(acc, field, anonymize_embed(value)) + end + end) + end + + defp anonymize_embed(field) when is_list(field) do + field + |> Enum.map(&anonymize_embed/1) + end + + defp anonymize_embed(field) do + data = anonymize_fields(field) + + field + |> Ecto.Changeset.change(data) + end + def is_association?(mod, association) do with association <- mod.__schema__(:association, association), false <- is_nil(association) do association.__struct__ in [ Ecto.Association.Has, Ecto.Association.ManyToMany, - Ecto.Association.HasThrough, - Ecto.Embedded + Ecto.Association.HasThrough ] else _ -> false end end + + def is_embed?(mod, field) do + case mod.__schema__(:type, field) do + {_, Ecto.Embedded, %Ecto.Embedded{}} -> + true + + {_, {_, Ecto.Embedded, %Ecto.Embedded{}}} -> + true + + _ -> + false + end + end end diff --git a/lib/ecto_anon/query.ex b/lib/ecto_anon/query.ex index 2c46d8b..e926602 100644 --- a/lib/ecto_anon/query.ex +++ b/lib/ecto_anon/query.ex @@ -1,6 +1,19 @@ defmodule EctoAnon.Query do import Ecto.Query + @spec apply(%{:fields => List.t(), :embeds => List.t()}, Ecto.Repo.t(), struct()) :: + {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} + def apply(%{fields: fields, embeds: embeds}, repo, struct) do + changeset = Ecto.Changeset.change(struct, fields) + + embeds + |> Enum.reduce(changeset, fn {field, data}, acc -> + acc + |> Ecto.Changeset.put_embed(field, data) + end) + |> repo.update() + end + @spec apply(keyword(), Ecto.Repo.t(), struct()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} def apply(data, repo, struct) do diff --git a/test/db/priv/repo/migrations/20211213090654_create_users.exs b/test/db/priv/repo/migrations/20211213090654_create_users.exs index 4ba6583..a883c09 100644 --- a/test/db/priv/repo/migrations/20211213090654_create_users.exs +++ b/test/db/priv/repo/migrations/20211213090654_create_users.exs @@ -7,6 +7,8 @@ defmodule EctoAnon.Repo.Migrations.CreateUsers do add(:email, :string) add(:firstname, :string) add(:lastname, :string) + add(:favorite_quote, :map) + add(:quotes, {:array, :map}) add(:last_sign_in_at, :utc_datetime) anonymized() diff --git a/test/ecto_anon/anonymizer_test.exs b/test/ecto_anon/anonymizer_test.exs index 2fb2566..553c695 100644 --- a/test/ecto_anon/anonymizer_test.exs +++ b/test/ecto_anon/anonymizer_test.exs @@ -6,15 +6,21 @@ defmodule EctoAnon.AnonymizerTest do test "returns the struct with anonymized fields" do user = %User{email: "john.doe@example.com", firstname: "John", lastname: "Doe"} - assert {:ok, [lastname: "redacted", email: "redacted", last_sign_in_at: nil]} = - EctoAnon.Anonymizer.anonymized_data(user) + assert {:ok, + %{ + embeds: [quotes: []], + fields: [lastname: "redacted", email: "redacted", last_sign_in_at: nil] + }} = EctoAnon.Anonymizer.anonymized_data(user) end test "returns anonymized fields only for non-nil fields" do user = %User{email: "john.doe@example.com", lastname: nil} - assert {:ok, [lastname: nil, email: "redacted", last_sign_in_at: nil]} = - EctoAnon.Anonymizer.anonymized_data(user) + assert {:ok, + %{ + embeds: [quotes: []], + fields: [lastname: nil, email: "redacted", last_sign_in_at: nil] + }} = EctoAnon.Anonymizer.anonymized_data(user) end end end diff --git a/test/ecto_anon/schema_test.exs b/test/ecto_anon/schema_test.exs index 02d531e..e95bc4b 100644 --- a/test/ecto_anon/schema_test.exs +++ b/test/ecto_anon/schema_test.exs @@ -41,6 +41,8 @@ defmodule EctoAnon.SchemaTest do test "returns a list of {field, function} tuples" do assert [ last_sign_in_at: {&EctoAnon.Functions.AnonymizedDate.run/3, [:only_year]}, + quotes: {&EctoAnon.Functions.Default.run/3, []}, + favorite_quote: {&EctoAnon.Functions.Default.run/3, []}, followers: {&EctoAnon.Functions.Default.run/3, []}, email: {&EctoAnon.Functions.Default.run/3, []}, lastname: {&EctoAnon.Functions.Default.run/3, []} diff --git a/test/ecto_anon_test.exs b/test/ecto_anon_test.exs index e44f3ed..459c584 100644 --- a/test/ecto_anon_test.exs +++ b/test/ecto_anon_test.exs @@ -1,6 +1,6 @@ defmodule EctoAnonTest do use ExUnit.Case, async: true - alias EctoAnon.{Repo, User, Comment, Comment.Quote} + alias EctoAnon.{Repo, User, Comment, User.Quote} defmodule UnknownStruct do defstruct name: "John", age: 27 @@ -14,7 +14,21 @@ defmodule EctoAnonTest do firstname: "Mick", lastname: "Rogers", last_sign_in_at: ~U[2021-01-03 00:00:00Z], - followers: [] + followers: [], + favorite_quote: %Quote{ + quote: "this is a quote", + author: "author" + }, + quotes: [ + %Quote{ + quote: "this is a quote", + author: "author" + }, + %Quote{ + quote: "this is a quote", + author: "author" + } + ] } |> Repo.insert!() @@ -24,7 +38,9 @@ defmodule EctoAnonTest do firstname: "Fred", lastname: "Duncan", last_sign_in_at: ~U[2023-03-20 00:00:00Z], - followers: [] + followers: [], + favorite_quote: nil, + quotes: [] } |> Repo.insert!() @@ -34,7 +50,21 @@ defmodule EctoAnonTest do firstname: "Emilie", lastname: "Duncan", last_sign_in_at: ~U[2018-09-04 00:00:00Z], - followers: [fred] + followers: [fred], + favorite_quote: %Quote{ + quote: "this is a quote", + author: "author" + }, + quotes: [ + %Quote{ + quote: "this is a quote", + author: "author" + }, + %Quote{ + quote: "this is a quote", + author: "author" + } + ] } |> Repo.insert!() @@ -44,20 +74,30 @@ defmodule EctoAnonTest do firstname: "John", lastname: "Doe", last_sign_in_at: ~U[2022-05-04 00:00:00Z], - followers: [mick, emilie] + followers: [mick, emilie], + favorite_quote: %Quote{ + quote: "this is a quote", + author: "author" + }, + quotes: [ + %Quote{ + quote: "this is a quote", + author: "author" + }, + %Quote{ + quote: "this is a quote", + author: "author" + } + ] } |> Repo.insert!() - comment = - %Comment{ - content: "this is a comment", - tag: "tag", - author_id: user.id, - quote: %Quote{ - quote: "this is a quote" - } - } - |> Repo.insert!() + %Comment{ + content: "this is a comment", + tag: "tag", + author_id: user.id + } + |> Repo.insert!() {:ok, user: user, mick: mick, emilie: emilie} end @@ -80,58 +120,86 @@ defmodule EctoAnonTest do mick: mick, emilie: emilie } do - comment = - %Comment{ - content: "this is a comment", - tag: "tag", - author_id: user.id, - quote: %Quote{ - quote: "this is a quote" - } - } - |> Repo.insert!() - - assert {:ok, updated_user} = - Repo.get(Comment, comment.id) - |> EctoAnon.run(Repo, cascade: true) + assert {:ok, updated_user} = EctoAnon.run(user, Repo, cascade: true) assert %User{ email: "redacted", firstname: "John", last_sign_in_at: ~U[2022-01-01 00:00:00Z], - lastname: "redacted" + lastname: "redacted", + favorite_quote: %Quote{ + quote: "redacted", + author: "redacted" + }, + quotes: [ + %Quote{ + quote: "redacted", + author: "redacted" + }, + %Quote{ + quote: "redacted", + author: "redacted" + } + ] } = updated_user - %Comment{ - content: "this is a comment", - tag: "tag", - quote: %Quote{ - quote: "redacted" - } - } = Repo.get_by(Comment, author_id: user.id) + assert %Comment{ + content: "this is a comment", + tag: "tag" + } = Repo.get_by(Comment, author_id: user.id) - %User{ - email: "redacted", - firstname: "Mick", - last_sign_in_at: ~U[2021-01-01 00:00:00Z], - lastname: "redacted", - followers: [] - } = Repo.get(User, mick.id) |> Repo.preload(:followers) + assert %User{ + email: "redacted", + firstname: "Mick", + last_sign_in_at: ~U[2021-01-01 00:00:00Z], + lastname: "redacted", + followers: [], + favorite_quote: %Quote{ + quote: "redacted", + author: "redacted" + }, + quotes: [ + %Quote{ + quote: "redacted", + author: "redacted" + }, + %Quote{ + quote: "redacted", + author: "redacted" + } + ] + } = Repo.get(User, mick.id) |> Repo.preload(:followers) - %User{ - email: "redacted", - firstname: "Emilie", - last_sign_in_at: ~U[2018-01-01 00:00:00Z], - lastname: "redacted", - followers: [ - %User{ - email: "redacted", - firstname: "Fred", - last_sign_in_at: ~U[2023-01-01 00:00:00Z], - lastname: "redacted" - } - ] - } = Repo.get(User, emilie.id) |> Repo.preload(:followers) + assert %User{ + email: "redacted", + firstname: "Emilie", + last_sign_in_at: ~U[2018-01-01 00:00:00Z], + lastname: "redacted", + favorite_quote: %Quote{ + quote: "redacted", + author: "redacted" + }, + followers: [ + %User{ + email: "redacted", + firstname: "Fred", + last_sign_in_at: ~U[2023-01-01 00:00:00Z], + lastname: "redacted", + favorite_quote: nil, + quotes: [] + } + ], + quotes: [ + %Quote{ + quote: "redacted", + author: "redacted" + }, + %Quote{ + quote: "redacted", + author: "redacted" + } + ] + } = Repo.get(User, emilie.id) |> Repo.preload(:followers) end end end diff --git a/test/support/comment.ex b/test/support/comment.ex index 38b9442..fb6b1c4 100644 --- a/test/support/comment.ex +++ b/test/support/comment.ex @@ -12,21 +12,6 @@ defmodule EctoAnon.Comment do field(:content, :string) field(:tag, :string) - embeds_one(:quote, EctoAnon.Comment.Quote) - belongs_to(:users, User, foreign_key: :author_id) end end - -defmodule EctoAnon.Comment.Quote do - use Ecto.Schema - use EctoAnon.Schema - - import Ecto.Changeset - - embedded_schema do - anon_field(:quote, :string) - end - - def changeset(changeset, attrs), do: cast(changeset, attrs, [:quote]) -end diff --git a/test/support/user.ex b/test/support/user.ex index 4578545..fb3c99a 100644 --- a/test/support/user.ex +++ b/test/support/user.ex @@ -8,6 +8,8 @@ defmodule EctoAnon.User do :lastname, :email, :followers, + :favorite_quote, + :quotes, last_sign_in_at: [:anonymized_date, options: [:only_year]] ]) @@ -18,6 +20,8 @@ defmodule EctoAnon.User do field(:last_sign_in_at, :utc_datetime) has_many(:comments, Comment, foreign_key: :author_id, references: :id) + embeds_one(:favorite_quote, EctoAnon.User.Quote) + embeds_many(:quotes, EctoAnon.User.Quote) many_to_many( :followers, @@ -30,6 +34,21 @@ defmodule EctoAnon.User do end end +defmodule EctoAnon.User.Quote do + use Ecto.Schema + use EctoAnon.Schema + + anon_schema([ + :quote, + :author + ]) + + embedded_schema do + field(:quote, :string) + field(:author, :string) + end +end + defmodule EctoAnon.User.Follower do use Ecto.Schema