diff --git a/lib/adbc_column.ex b/lib/adbc_column.ex index 737582d5..55abb520 100644 --- a/lib/adbc_column.ex +++ b/lib/adbc_column.ex @@ -1240,111 +1240,72 @@ defmodule Adbc.Column do ...> sizes: [3, 0, 4, 0, 2] ...> } ...> } - %Adbc.Column{ - name: nil, - type: :list_view, - nullable: true, - metadata: nil, - data: %{ - offsets: [4, 7, 0, 0, 3], - sizes: [3, 0, 4, 0, 2], - validity: [true, false, true, true, true], - values: %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [0, -127, 127, 50, 12, -7, 25] - } - } - } iex> Adbc.Column.to_list(list_view) - %Adbc.Column{ - name: nil, - type: :list, - nullable: true, - metadata: nil, - data: [ - %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [12, -7, 25] - }, - nil, - %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [0, -127, 127, 50] - }, - %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [] - }, - %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: ~c"2\f" - } - ] - } + [[12, -7, 25], nil, [0, -127, 127, 50], [], ~c"2\f"] """ - @spec to_list(%Adbc.Column{data: map()}) :: %Adbc.Column{} - def to_list( - column = %Adbc.Column{ - type: type, - data: %{ - validity: validity, - offsets: offsets, - sizes: sizes, - values: values = %Adbc.Column{} - } + @spec to_list(%Adbc.Column{}) :: [term()] + def to_list(%Adbc.Column{ + type: type, + data: %{ + validity: validity, + offsets: offsets, + sizes: sizes, + values: values = %Adbc.Column{} } - ) + }) when type in @list_view_types and is_list(validity) and is_list(offsets) and is_list(sizes) do - values = + list = if values.type in @list_view_types do - Adbc.Column.to_list(values) + to_list(values) else - values + values.data end - new_data = - Enum.map(Enum.zip([offsets, sizes, validity]), fn {offset, size, valid} -> - if valid do - %{ - values - | data: Enum.map(Enum.slice(values.data, offset, size), &Adbc.Column.to_list/1) - } - else - nil - end - end) - - %{column | data: new_data, type: :list} + Enum.zip_with([offsets, sizes, validity], fn [offset, size, valid] -> + if valid do + Enum.slice(list, offset, size) + else + nil + end + end) end - def to_list( - column = %Adbc.Column{ - type: :run_end_encoded, - data: %{ - run_ends: run_ends = %Adbc.Column{type: run_end_type}, - values: values - }, - length: length, - offset: offset - } - ) + # defp list_to_map(nil), do: nil + + # defp list_to_map(%Adbc.Column{name: name, type: type, data: data}) do + # case type do + # :list -> + # list = Enum.map(data, &list_to_map/1) + + # if name == "item" do + # list + # else + # {name, list} + # end + + # :struct -> + # Enum.map(data, &list_to_map/1) + + # _ -> + # if name == "item" do + # data + # else + # {name, data} + # end + # end + # end + + def to_list(%Adbc.Column{ + type: :run_end_encoded, + data: %{ + run_ends: run_ends = %Adbc.Column{type: run_end_type, data: data}, + values: values + }, + length: length, + offset: offset + }) when is_integer(offset) and offset >= 0 and is_integer(length) and length >= 1 do - values = Adbc.Column.to_list(values) + values = to_list(values) max_allowed_length = case run_end_type do @@ -1367,13 +1328,13 @@ defmodule Adbc.Column do "Run end data exceeds maximum allowed length: #{length} + #{offset} > #{max_allowed_length}" end - run_end_len = Enum.count(run_ends.data) + run_end_len = Enum.count(data) {run_end_start_index, values_start_index, encoded} = - case Enum.drop_while(run_ends.data, &(&1 < offset)) do + case Enum.drop_while(data, &(&1 < offset)) do [] -> raise Adbc.Error, - "Last run end is #{hd(Enum.reverse(run_ends.data))} but it should >= #{offset + length} (offset: #{offset}, length: #{length})" + "Last run end is #{hd(Enum.reverse(data))} but it should >= #{offset + length} (offset: #{offset}, length: #{length})" encoded = [run_end_start_index | _] -> values_start_index = run_end_len - Enum.count(encoded) @@ -1385,7 +1346,7 @@ defmodule Adbc.Column do end end - if offset + length > hd(Enum.reverse(run_ends.data)) do + if offset + length > hd(Enum.reverse(data)) do raise Adbc.Error, "Last run end is #{hd(Enum.reverse(run_ends.data))} but it should >= #{offset + length} (offset: #{offset}, length: #{length})" end @@ -1400,41 +1361,30 @@ defmodule Adbc.Column do run_end end - if is_map(values.data) do - {run_end, value_index + 1, List.duplicate(to_list(values), real_end - index) ++ acc} + if is_map(values) do + {run_end, value_index + 1, List.duplicate(values, real_end - index) ++ acc} else {run_end, value_index + 1, - List.duplicate(Enum.at(values.data, value_index), real_end - index) ++ acc} + List.duplicate(Enum.at(values, value_index), real_end - index) ++ acc} end end) - %Adbc.Column{ - name: column.name, - type: values.type, - nullable: column.nullable, - data: Enum.reverse(decoded), - metadata: nil - } + Enum.reverse(decoded) end - def to_list(column = %Adbc.Column{data: %{key: key, value: value}, type: :dictionary}) do + def to_list(%Adbc.Column{data: %{key: key, value: value}, type: :dictionary}) do value = to_list(value) - column_data = - Enum.map(key.data, fn - index when is_integer(index) -> - Enum.at(value.data, index) - - nil -> - nil - end) + Enum.map(key.data, fn + index when is_integer(index) -> + Enum.at(value, index) - %{column | data: column_data, type: value.type} + nil -> + nil + end) end - def to_list(column = %Adbc.Column{data: data}) when is_list(data) do - %{column | data: Enum.map(data, &Adbc.Column.to_list/1)} - end + def to_list(%Adbc.Column{data: data, type: :list}), do: Enum.map(data, &to_list/1) - def to_list(v), do: v + def to_list(%Adbc.Column{data: data}), do: data end diff --git a/lib/adbc_result.ex b/lib/adbc_result.ex index 6451fbc9..4271c8fa 100644 --- a/lib/adbc_result.ex +++ b/lib/adbc_result.ex @@ -4,7 +4,8 @@ defmodule Adbc.Result do It has two fields: - * `:data` - a list of `Adbc.Column` + * `:data` - a list of `Adbc.Column`. The `Adbc.Column` may + not yet have been materialized * `:num_rows` - the number of rows returned, if returned by the database @@ -29,44 +30,22 @@ defmodule Adbc.Result do Returns a map of columns as a result. """ def to_map(result = %Adbc.Result{}) do - Map.new(to_list(materialize(result)).data, fn %Adbc.Column{name: name, type: type, data: data} -> - case type do - :list -> {name, Enum.map(data, &list_to_map/1)} - _ -> {name, data} - end + Map.new(result.data, fn %Adbc.Column{name: name} = column -> + {name, column |> Adbc.Column.materialize() |> Adbc.Column.to_list()} end) end +end - @doc """ - Convert any list view in the result set to normal lists. - """ - @spec to_list(%Adbc.Result{}) :: %Adbc.Result{} - def to_list(result = %Adbc.Result{data: data}) when is_list(data) do - %{result | data: Enum.map(data, &Adbc.Column.to_list/1)} - end - - defp list_to_map(nil), do: nil - - defp list_to_map(%Adbc.Column{name: name, type: type, data: data}) do - case type do - :list -> - list = Enum.map(data, &list_to_map/1) - - if name == "item" do - list - else - {name, list} - end - - :struct -> - Enum.map(data, &list_to_map/1) - - _ -> - if name == "item" do - data - else - {name, data} - end - end +defimpl Table.Reader, for: Adbc.Result do + def init(result) do + data = + Enum.map(result.data, fn column -> + column + |> Adbc.Column.materialize() + |> Adbc.Column.to_list() + end) + + names = Enum.map(result.data, & &1.name) + {:columns, %{columns: names, count: result.num_rows}, data} end end diff --git a/mix.exs b/mix.exs index b8053ad5..962f99c7 100644 --- a/mix.exs +++ b/mix.exs @@ -79,6 +79,7 @@ defmodule Adbc.MixProject do # runtime {:decimal, "~> 2.1"}, + {:table, "~> 0.1.2"}, # docs {:ex_doc, "~> 0.29", only: :docs, runtime: false} diff --git a/mix.lock b/mix.lock index c091a5df..2480a707 100644 --- a/mix.lock +++ b/mix.lock @@ -9,4 +9,5 @@ "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "table": {:hex, :table, "0.1.2", "87ad1125f5b70c5dea0307aa633194083eb5182ec537efc94e96af08937e14a8", [:mix], [], "hexpm", "7e99bc7efef806315c7e65640724bf165c3061cdc5d854060f74468367065029"}, } diff --git a/test/adbc_column_test.exs b/test/adbc_column_test.exs index d911c2ee..a56fc317 100644 --- a/test/adbc_column_test.exs +++ b/test/adbc_column_test.exs @@ -294,46 +294,13 @@ defmodule Adbc.Column.Test do } } - assert %Adbc.Column{ - name: "my_array", - type: :list, - nullable: true, - metadata: nil, - data: [ - %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [12, -7, 25] - }, - _inner2 = nil, - inner3 = %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [0, -127, 127, 50] - }, - inner4 = %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [] - }, - inner5 = %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: ~c"2\f" - } - ] - } = Adbc.Column.to_list(list_view) - - assert %{"my_array" => [[12, -7, 25], nil, [0, -127, 127, 50], [], ~c"2\f"]} == - Adbc.Result.to_map(%Adbc.Result{data: [list_view]}) + assert Adbc.Column.to_list(list_view) == [ + [12, -7, 25], + nil, + [0, -127, 127, 50], + [], + ~c"2\f" + ] nested_list_view = %Adbc.Column{ name: nil, @@ -348,91 +315,13 @@ defmodule Adbc.Column.Test do } } - assert %Adbc.Column{ - data: [ - # offsets=2, sizes=2 - # => [inner3, inner4] - %Adbc.Column{ - metadata: nil, - name: "my_array", - nullable: true, - type: :list, - data: [ - ^inner3 = %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [0, -127, 127, 50] - }, - ^inner4 = %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [] - } - ] - }, - # offsets=5, sizes=0 - # => nil - nil, - # offsets=0, sizes=1 - # => inner1 - %Adbc.Column{ - metadata: nil, - name: "my_array", - nullable: true, - type: :list, - data: [ - %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [12, -7, 25] - } - ] - }, - # offsets=0, sizes=0 - # => [] - %Adbc.Column{ - data: [], - metadata: nil, - name: "my_array", - nullable: true, - type: :list - }, - # offsets=3, sizes=2 - # => [inner4, inner5] - %Adbc.Column{ - data: [ - ^inner4 = %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: [] - }, - ^inner5 = %Adbc.Column{ - name: "item", - type: :s32, - nullable: false, - metadata: nil, - data: ~c"2\f" - } - ], - metadata: nil, - name: "my_array", - nullable: true, - type: :list - } - ], - metadata: nil, - name: nil, - nullable: true, - type: :list - } = Adbc.Column.to_list(nested_list_view) + assert Adbc.Column.to_list(nested_list_view) == [ + [[0, -127, 127, 50], []], + nil, + [[12, -7, 25]], + [], + [[], ~c"2\f"] + ] end end @@ -476,25 +365,15 @@ defmodule Adbc.Column.Test do } } - assert %Adbc.Column{ - data: [1.0, 1.0, 1.0, 1.0, nil, nil, 2.0], - metadata: nil, - name: "sample_run_end_encoded_array", - nullable: false, - type: :f32 - } == Adbc.Column.to_list(run_end_array) + assert Adbc.Column.to_list(run_end_array) == + [1.0, 1.0, 1.0, 1.0, nil, nil, 2.0] # change logical length = 6 and offset = 1 # [1.0, 1.0, 1.0, 1.0, null, null, 2.0] # ^ ^ # |- offset = 1 |- length = 6 - assert %Adbc.Column{ - data: [1.0, 1.0, 1.0, nil, nil, 2.0], - metadata: nil, - name: "sample_run_end_encoded_array", - nullable: false, - type: :f32 - } == Adbc.Column.to_list(%{run_end_array | offset: 1, length: 6}) + assert Adbc.Column.to_list(%{run_end_array | offset: 1, length: 6}) == + [1.0, 1.0, 1.0, nil, nil, 2.0] # change logical length = 7 and offset = 1 # [1.0, 1.0, 1.0, 1.0, null, null, 2.0] @@ -554,13 +433,7 @@ defmodule Adbc.Column.Test do } } - assert %Adbc.Column{ - name: "inner_run_end_encoded_array", - type: :s32, - nullable: true, - metadata: nil, - data: [1, 1, 2] - } == Adbc.Column.to_list(inner_run_end_array) + assert Adbc.Column.to_list(inner_run_end_array) == [1, 1, 2] run_end_array = %Adbc.Column{ name: "sample_run_end_encoded_array", @@ -581,13 +454,7 @@ defmodule Adbc.Column.Test do } } - assert %Adbc.Column{ - data: [1, 2, 2], - metadata: nil, - name: "sample_run_end_encoded_array", - nullable: false, - type: :s32 - } == Adbc.Column.to_list(run_end_array) + assert Adbc.Column.to_list(run_end_array) == [1, 2, 2] end end @@ -608,14 +475,7 @@ defmodule Adbc.Column.Test do value = Adbc.Column.string(["foo", "bar", "baz"], name: "value", nullable: false) dict = Adbc.Column.dictionary(key, value) - assert %Adbc.Column{ - name: nil, - type: :string, - nullable: false, - metadata: nil, - data: ["foo", "bar", "foo", "bar", nil, "baz"] - } = - Adbc.Column.to_list(dict) + assert Adbc.Column.to_list(dict) == ["foo", "bar", "foo", "bar", nil, "baz"] end end end diff --git a/test/adbc_result_test.exs b/test/adbc_result_test.exs index 0be3f634..db174eb6 100644 --- a/test/adbc_result_test.exs +++ b/test/adbc_result_test.exs @@ -1,10 +1,19 @@ defmodule Adbc.Result.Test do - use ExUnit.Case - + use ExUnit.Case, async: true alias Adbc.Result - test "to_map with list views" do - result = %Adbc.Result{ + # Just some imaginary data and context so it's easier to understand this test + # measurements: [1, 2, 3, 4, 5, 6] + # data points: [ + # [1], # 2024-05-31 12:00:00 - 2024-05-31 12:15:00 + # [2, 3], # 2024-05-31 12:15:00 - 2024-05-31 12:30:00 + # [3, 4], # 2024-05-31 12:30:00 - 2024-05-31 12:45:00 + # [4], # 2024-05-31 12:45:00 - 2024-05-31 13:00:00 + # [5, 6], # 2024-05-31 13:00:00 - 2024-05-31 13:15:00 + # [6] # 2024-05-31 13:15:00 - 2024-05-31 13:30:00 + # ] + defp result do + %Adbc.Result{ data: [ %Adbc.Column{ name: "start_time", @@ -61,34 +70,28 @@ defmodule Adbc.Result.Test do } ] } + end - # Just some imaginary data and context so it's easier to understand this test - # measurements: [1, 2, 3, 4, 5, 6] - # data points: [ - # [1], # 2024-05-31 12:00:00 - 2024-05-31 12:15:00 - # [2, 3], # 2024-05-31 12:15:00 - 2024-05-31 12:30:00 - # [3, 4], # 2024-05-31 12:30:00 - 2024-05-31 12:45:00 - # [4], # 2024-05-31 12:45:00 - 2024-05-31 13:00:00 - # [5, 6], # 2024-05-31 13:00:00 - 2024-05-31 13:15:00 - # [6] # 2024-05-31 13:15:00 - 2024-05-31 13:30:00 - # ] + test "implements table reader" do + assert result() |> Table.to_rows() |> Enum.to_list() == [ + %{ + "end_time" => ~N[2024-05-31 13:00:00], + "start_time" => ~N[2024-05-31 12:00:00], + "time_series" => [[1], [2, 3], [3, 4], [4]] + }, + %{ + "end_time" => ~N[2024-05-31 13:30:00], + "start_time" => ~N[2024-05-31 12:30:00], + "time_series" => [[3, 4], [4], [5, 6], [6]] + } + ] + end + + test "to_map with list views" do assert %{ "start_time" => [~N[2024-05-31 12:00:00], ~N[2024-05-31 12:30:00]], "end_time" => [~N[2024-05-31 13:00:00], ~N[2024-05-31 13:30:00]], - "time_series" => [ - [ - {"variable_sliding_window", [1]}, - {"variable_sliding_window", [2, 3]}, - {"variable_sliding_window", [3, 4]}, - {"variable_sliding_window", [4]} - ], - [ - {"variable_sliding_window", [3, 4]}, - {"variable_sliding_window", [4]}, - {"variable_sliding_window", [5, 6]}, - {"variable_sliding_window", [6]} - ] - ] - } == Result.to_map(result) + "time_series" => [[[1], [2, 3], [3, 4], [4]], [[3, 4], [4], [5, 6], [6]]] + } == Result.to_map(result()) end end