optimise postgres queries speed

lalabuy948 · Sep 23, 2024 · f59bef2 · f59bef2
1 parent 4fab8bb
commit f59bef2
Show file tree

Hide file tree

Showing 12 changed files with 77 additions and 44 deletions.
diff --git a/examples/duck_only/mix.lock b/examples/duck_only/mix.lock
@@ -8,6 +8,7 @@
   "duckdbex": {:hex, :duckdbex, "0.3.3", "ca15a98ac7007d72afac72fbbeea065555bbcf60971af73f43c36746a633b385", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "b411e1250c27c52e1119ac9d4a6640c6670f099a60bea1d10f4e25e305cd5b2f"},
   "ecto": {:hex, :ecto, "3.12.3", "1a9111560731f6c3606924c81c870a68a34c819f6d4f03822f370ea31a582208", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9efd91506ae722f95e48dc49e70d0cb632ede3b7a23896252a60a14ac6d59165"},
   "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"},
+  "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},
   "esbuild": {:hex, :esbuild, "0.8.1", "0cbf919f0eccb136d2eeef0df49c4acf55336de864e63594adcea3814f3edf41", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "25fc876a67c13cb0a776e7b5d7974851556baeda2085296c14ab48555ea7560f"},
   "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
   "expo": {:hex, :expo, "1.1.0", "f7b9ed7fb5745ebe1eeedf3d6f29226c5dd52897ac67c0f8af62a07e661e5c75", [:mix], [], "hexpm", "fbadf93f4700fb44c331362177bdca9eeb8097e8b0ef525c9cc501cb9917c960"},

diff --git a/examples/duck_postgres/mix.exs b/examples/duck_postgres/mix.exs
@@ -19,7 +19,7 @@ defmodule DuckPostgres.MixProject do
   def application do
     [
       mod: {DuckPostgres.Application, []},
-      extra_applications: [:logger, :runtime_tools]
+      extra_applications: [:logger, :runtime_tools, :wx, :observer]
     ]
   end
 

diff --git a/lib/phoenix_analytics/queries/analytics/stats/per_period.ex b/lib/phoenix_analytics/queries/analytics/stats/per_period.ex
@@ -112,7 +112,7 @@ defmodule PhoenixAnalytics.Queries.Analytics.Stats.PerPeriod do
                   )) AS period
           )
           SELECT * FROM (
-            SELECT date_trunc('#{interval}', ds.period)::VARCHAR AS date, count(*) as hits,
+            SELECT date_trunc('#{interval}', ds.period)::VARCHAR AS date, count(request_id) as hits,
 
             FROM date_series ds
             LEFT JOIN #{@table} ON date_trunc('day', inserted_at) = ds.period AND method = 'GET'
@@ -138,7 +138,7 @@ defmodule PhoenixAnalytics.Queries.Analytics.Stats.PerPeriod do
                 )) AS period
         )
         SELECT * FROM (
-          SELECT date_trunc('#{interval}', ds.period)::VARCHAR AS date, count(*) as hits,
+          SELECT date_trunc('#{interval}', ds.period)::VARCHAR AS date, count(request_id) as hits,
 
           FROM date_series ds
           LEFT JOIN #{@table} ON date_trunc('day', inserted_at) = ds.period

diff --git a/lib/phoenix_analytics/repo.ex b/lib/phoenix_analytics/repo.ex
@@ -48,15 +48,15 @@ defmodule PhoenixAnalytics.Repo do
   end
 
   @doc """
-  Retrieves the current `read` DuckDB connection from the GenServer state.
+  Creates and returns a new DuckDB read connection.
 
-  This function sends a synchronous call to the GenServer to get the current
-  database connection stored in its state.
+  This function opens a new connection to the DuckDB database for read operations,
+  allowing for increased read throughput by not relying on the GenServer state.
 
   ## Returns
 
-    * `{:ok, connection}` - If the connection is successfully retrieved.
-    * `{:error, reason}` - If there's an error retrieving the connection.
+    * `{:ok, connection}` - If a new connection is successfully created.
+    * `{:error, reason}` - If there's an error creating the connection.
 
   ## Examples
 
@@ -65,7 +65,11 @@ defmodule PhoenixAnalytics.Repo do
 
   """
   def get_read_connection do
-    GenServer.call(__MODULE__, :get_read_connection)
+    with {:ok, db} <- Duckdbex.open(@db_path),
+         {:ok, connection} <- Duckdbex.connection(db),
+         {:ok, _} <- Bridge.attach_postgres(db, connection) do
+      {:ok, connection}
+    end
   end
 
   # --- server callbacks ---
@@ -90,9 +94,9 @@ defmodule PhoenixAnalytics.Repo do
   """
   def init(_state) do
     with {:ok, db} <- Duckdbex.open(@db_path),
-         {:ok, conn} = Duckdbex.connection(db),
-         {:ok, read_conn} = Duckdbex.connection(db),
-         {:ok, _} = Bridge.attach_postgres(db, conn) do
+         {:ok, conn} <- Duckdbex.connection(db),
+         {:ok, read_conn} <- Duckdbex.connection(db),
+         {:ok, _} <- Bridge.attach_postgres(db, conn) do
       {:ok, %{connection: conn, read_connection: read_conn}}
     else
       {:error, reason} ->
@@ -106,11 +110,6 @@ defmodule PhoenixAnalytics.Repo do
     {:reply, {:ok, state.connection}, state}
   end
 
-  @doc false
-  def handle_call(:get_read_connection, _from, state) do
-    {:reply, {:ok, state.read_connection}, state}
-  end
-
   @doc """
   Executes an unsafe query on the DuckDB connection.
 
@@ -258,7 +257,6 @@ defmodule PhoenixAnalytics.Repo do
       {:ok, conection} ->
         {:ok, stmt_ref} = Duckdbex.prepare_statement(conection, query)
         {:ok, result_ref} = Duckdbex.execute_statement(stmt_ref, params)
-
         Duckdbex.fetch_all(result_ref)
 
       {:error, reason} ->

diff --git a/lib/phoenix_analytics/services/bridge.ex b/lib/phoenix_analytics/services/bridge.ex
@@ -27,15 +27,18 @@ defmodule PhoenixAnalytics.Services.Bridge do
         {:error, "duckdb: failed to load postgres extension"}
       end
 
-      Duckdbex.query(conn, "SET pg_experimental_filter_pushdown=TRUE;")
-      Duckdbex.query(conn, "SET pg_pages_per_task = 9876543;")
-      Duckdbex.query(conn, "SET pg_use_ctid_scan=false;")
-
       postgres_conn = Application.fetch_env!(:phoenix_analytics, :postgres_conn)
 
       case Duckdbex.query(conn, "ATTACH '#{postgres_conn}' AS postgres_db (TYPE POSTGRES);") do
-        {:ok, _} -> {:ok, "duckdb: postgres database connected"}
-        {:error, error} -> {:error, "duckdb: postgres connection failed: #{inspect(error)}"}
+        {:ok, _} ->
+          {:ok, _ref} = Duckdbex.query(conn, "SET pg_experimental_filter_pushdown=TRUE;")
+          {:ok, _ref} = Duckdbex.query(conn, "SET pg_pages_per_task = 9876543;")
+          {:ok, _ref} = Duckdbex.query(conn, "SET pg_use_ctid_scan=false;")
+
+          {:ok, "duckdb: postgres database connected"}
+
+        {:error, error} ->
+          {:error, "duckdb: postgres connection failed: #{inspect(error)}"}
       end
     end
 

diff --git a/lib/phoenix_analytics/web/live/components/charts/device_chart.ex b/lib/phoenix_analytics/web/live/components/charts/device_chart.ex
@@ -12,19 +12,25 @@ defmodule PhoenixAnalytics.Web.Live.Components.DeviceChart do
   def render(assigns) do
     ~H"""
     <div>
-      <.react name="DeviceChart" dateRange={@date_range} chartData={@chart_data} socket={@socket} />
+      <.react
+        name="DeviceChart"
+        dateRange={@date_range}
+        chartData={@chart_data.result || []}
+        socket={@socket}
+      />
     </div>
     """
   end
 
   @impl true
   def update(assigns, socket) do
     date_range = assigns.date_range
-    chart_data = chart_data(date_range)
 
     {:ok,
      assign(socket, assigns)
-     |> assign(:chart_data, chart_data)}
+     |> assign_async(:chart_data, fn ->
+       {:ok, %{chart_data: chart_data(date_range)}}
+     end)}
   end
 
   defp chart_data(%{from: from, to: to} = _date_range) do

diff --git a/lib/phoenix_analytics/web/live/components/charts/popular_chart.ex b/lib/phoenix_analytics/web/live/components/charts/popular_chart.ex
@@ -15,7 +15,7 @@ defmodule PhoenixAnalytics.Web.Live.Components.PopularChart do
       <.react
         name="PopularChart"
         dateRange={@date_range}
-        chartData={@chart_data}
+        chartData={@chart_data.result || []}
         chartTitle={@chart_title}
         socket={@socket}
       />
@@ -30,7 +30,9 @@ defmodule PhoenixAnalytics.Web.Live.Components.PopularChart do
 
     {:ok,
      assign(socket, assigns)
-     |> assign(:chart_data, chart_data(data_source, date_range))}
+     |> assign_async(:chart_data, fn ->
+       {:ok, %{chart_data: chart_data(data_source, date_range)}}
+     end)}
   end
 
   defp chart_data(source, %{from: from, to: to} = _date_range) do

diff --git a/lib/phoenix_analytics/web/live/components/charts/requests_chart.ex b/lib/phoenix_analytics/web/live/components/charts/requests_chart.ex
@@ -12,7 +12,12 @@ defmodule PhoenixAnalytics.Web.Live.Components.RequestsChart do
   def render(assigns) do
     ~H"""
     <div>
-      <.react name="RequestsChart" dateRange={@date_range} chartData={@chart_data} socket={@socket} />
+      <.react
+        name="RequestsChart"
+        dateRange={@date_range}
+        chartData={@chart_data.result || []}
+        socket={@socket}
+      />
     </div>
     """
   end
@@ -24,7 +29,9 @@ defmodule PhoenixAnalytics.Web.Live.Components.RequestsChart do
 
     {:ok,
      assign(socket, assigns)
-     |> assign(:chart_data, chart_data(date_range, interval))}
+     |> assign_async(:chart_data, fn ->
+       {:ok, %{chart_data: chart_data(date_range, interval)}}
+     end)}
   end
 
   def chart_data(%{from: from, to: to} = _date_range, interval) do

diff --git a/lib/phoenix_analytics/web/live/components/charts/res_chart.ex b/lib/phoenix_analytics/web/live/components/charts/res_chart.ex
@@ -15,7 +15,7 @@ defmodule PhoenixAnalytics.Web.Live.Components.ResChart do
       <.react
         name="ResChart"
         dateRange={@date_range}
-        chartData={@chart_data}
+        chartData={@chart_data.result || []}
         chartTitle={@chart_title}
         socket={@socket}
       />
@@ -30,7 +30,9 @@ defmodule PhoenixAnalytics.Web.Live.Components.ResChart do
 
     {:ok,
      assign(socket, assigns)
-     |> assign(:chart_data, chart_data(data_source, date_range))}
+     |> assign_async(:chart_data, fn ->
+       {:ok, %{chart_data: chart_data(data_source, date_range)}}
+     end)}
   end
 
   defp chart_data(source, %{from: from, to: to} = _date_range) do

diff --git a/lib/phoenix_analytics/web/live/components/charts/status_chart.ex b/lib/phoenix_analytics/web/live/components/charts/status_chart.ex
@@ -12,7 +12,12 @@ defmodule PhoenixAnalytics.Web.Live.Components.StatusChart do
   def render(assigns) do
     ~H"""
     <div>
-      <.react name="StatusChart" dateRange={@date_range} chartData={@chart_data} socket={@socket} />
+      <.react
+        name="StatusChart"
+        dateRange={@date_range}
+        chartData={@chart_data.result || []}
+        socket={@socket}
+      />
     </div>
     """
   end
@@ -24,7 +29,7 @@ defmodule PhoenixAnalytics.Web.Live.Components.StatusChart do
 
     {:ok,
      assign(socket, assigns)
-     |> assign(:chart_data, chart_data(date_range, interval))}
+     |> assign_async(:chart_data, fn -> {:ok, %{chart_data: chart_data(date_range, interval)}} end)}
   end
 
   defp chart_data(%{from: from, to: to} = _date_range, interval) do

diff --git a/lib/phoenix_analytics/web/live/components/charts/visits_chart.ex b/lib/phoenix_analytics/web/live/components/charts/visits_chart.ex
@@ -12,7 +12,12 @@ defmodule PhoenixAnalytics.Web.Live.Components.VisitsChart do
   def render(assigns) do
     ~H"""
     <div>
-      <.react name="VisitsChart" dateRange={@date_range} chartData={@chart_data} socket={@socket} />
+      <.react
+        name="VisitsChart"
+        dateRange={@date_range}
+        chartData={@chart_data.result || []}
+        socket={@socket}
+      />
     </div>
     """
   end
@@ -24,7 +29,7 @@ defmodule PhoenixAnalytics.Web.Live.Components.VisitsChart do
 
     {:ok,
      assign(socket, assigns)
-     |> assign(:chart_data, chart_data(date_range, interval))}
+     |> assign_async(:chart_data, fn -> {:ok, %{chart_data: chart_data(date_range, interval)}} end)}
   end
 
   defp chart_data(%{from: from, to: to} = _date_range, interval) do

diff --git a/lib/phoenix_analytics/web/live/components/stats/single_stat.ex b/lib/phoenix_analytics/web/live/components/stats/single_stat.ex
@@ -13,23 +13,23 @@ defmodule PhoenixAnalytics.Web.Live.Components.SingleStat do
     <div>
       <.react
         name="SingleStat"
-        statData={@stat_data}
         statUnit={@stat_unit}
         statTitle={@stat_title}
         dateRange={@date_range}
-        chartData={@chart_data}
+        statData={@stat_data.result || 0}
+        chartData={@chart_data.result || []}
         socket={@socket}
       />
     </div>
     """
   end
 
   def update(assigns, socket) do
-    stat_data = stat_data(assigns.source, assigns.date_range)
-    chart_data = chart_data(assigns.source, assigns.date_range)
+    source = assigns.source
+    date_range = assigns.date_range
 
     stat_title =
-      case assigns.source do
+      case source do
         :unique_visitors -> "Unique visitors"
         :total_pageviews -> "Total Pageviews"
         :total_requests -> "Total Requests"
@@ -41,8 +41,12 @@ defmodule PhoenixAnalytics.Web.Live.Components.SingleStat do
     {:ok,
      assign(socket, assigns)
      |> assign(:stat_title, stat_title)
-     |> assign(:stat_data, stat_data)
-     |> assign(:chart_data, chart_data)}
+     |> assign_async(:stat_data, fn ->
+       {:ok, %{stat_data: stat_data(source, date_range)}}
+     end)
+     |> assign_async(:chart_data, fn ->
+       {:ok, %{chart_data: chart_data(source, date_range)}}
+     end)}
   end
 
   defp stat_data(source, %{from: from, to: to} = _date_range) do