Plaid portfolio sync algorithm and calculation improvements (#1526)

* Start tests rework

* Cash balance on schema

* Add reverse syncer

* Reverse balance sync with holdings

* Reverse holdings sync

* Reverse holdings sync should work with only trade entries

* Consolidate brokerage cash

* Add forward sync option

* Update new balance info after syncs

* Intraday balance calculator and sync fixes

* Show only balance for trade entries

* Tests passing

* Update Gemfile.lock

* Cleanup, performance improvements

* Remove account reloads for reliable sync outputs

* Simplify valuation view logic

* Special handling for Plaid cash holding

Gemfile

@@ -50,7 +50,6 @@ gem "csv"
 gem "redcarpet"
 gem "stripe"
 gem "intercom-rails"
-gem "holidays"
 gem "plaid"
 
 group :development, :test do

Gemfile.lock

@@ -185,7 +185,6 @@ GEM
       thor (>= 1.0.0)
     hashdiff (1.1.1)
     highline (3.0.1)
-    holidays (8.8.0)
     hotwire-livereload (1.4.1)
       actioncable (>= 6.0.0)
       listen (>= 3.0.0)
@@ -493,7 +492,6 @@ DEPENDENCIES
   faraday-multipart
   faraday-retry
   good_job
-  holidays
   hotwire-livereload
   hotwire_combobox
   i18n-tasks

app/controllers/account/holdings_controller.rb

@@ -5,8 +5,6 @@ class Account::HoldingsController < ApplicationController
 
   def index
     @account = Current.family.accounts.find(params[:account_id])
-    @holdings = Current.family.holdings.current
-    @holdings = @holdings.where(account: @account) if @account
   end
 
   def show

app/controllers/accounts_controller.rb

@@ -4,8 +4,8 @@ class AccountsController < ApplicationController
   before_action :set_account, only: %i[sync]
 
   def index
-    @manual_accounts = Current.family.accounts.manual.alphabetically
-    @plaid_items = Current.family.plaid_items.ordered
+    @manual_accounts = Current.family.accounts.where(scheduled_for_deletion: false).manual.alphabetically
+    @plaid_items = Current.family.plaid_items.where(scheduled_for_deletion: false).ordered
   end
 
   def summary
@@ -14,7 +14,7 @@ def summary
     @net_worth_series = snapshot[:net_worth_series]
     @asset_series = snapshot[:asset_series]
     @liability_series = snapshot[:liability_series]
-    @accounts = Current.family.accounts
+    @accounts = Current.family.accounts.active
     @account_groups = @accounts.by_group(period: @period, currency: Current.family.currency)
   end
 

app/controllers/concerns/localize.rb

@@ -3,11 +3,17 @@ module Localize
 
   included do
     around_action :switch_locale
+    around_action :switch_timezone
   end
 
   private
     def switch_locale(&action)
       locale = Current.family.try(:locale) || I18n.default_locale
       I18n.with_locale(locale, &action)
     end
+
+    def switch_timezone(&action)
+      timezone = Current.family.try(:timezone) || Time.zone
+      Time.use_zone(timezone, &action)
+    end
 end

app/controllers/users_controller.rb

@@ -41,7 +41,7 @@ def should_purge_profile_image?
     def user_params
       params.require(:user).permit(
         :first_name, :last_name, :profile_image, :redirect_to, :delete_profile_image, :onboarded_at,
-        family_attributes: [ :name, :currency, :country, :locale, :date_format, :id ]
+        family_attributes: [ :name, :currency, :country, :locale, :date_format, :timezone, :id ]
       )
     end
 

app/helpers/account/entries_helper.rb

@@ -13,17 +13,12 @@ def transfer_entries(entries)
   end
 
   def entries_by_date(entries, selectable: true, totals: false)
-    entries.group_by(&:date).map do |date, grouped_entries|
-      # Valuations always go first, then sort by created_at desc
-      sorted_entries = grouped_entries.sort_by do |entry|
-        [ entry.account_valuation? ? 0 : 1, -entry.created_at.to_i ]
-      end
-
+    entries.reverse_chronological.group_by(&:date).map do |date, grouped_entries|
       content = capture do
-        yield sorted_entries
+        yield grouped_entries
       end
 
-      render partial: "account/entries/entry_group", locals: { date:, entries: sorted_entries, content:, selectable:, totals: }
+      render partial: "account/entries/entry_group", locals: { date:, entries: grouped_entries, content:, selectable:, totals: }
     end.join.html_safe
   end
 

app/helpers/account/cashes_helper.rb → app/helpers/account/holdings_helper.rb

@@ -1,13 +1,13 @@
-module Account::CashesHelper
-  def brokerage_cash(account)
+module Account::HoldingsHelper
+  def brokerage_cash_holding(account)
     currency = Money::Currency.new(account.currency)
 
     account.holdings.build \
       date: Date.current,
-      qty: account.balance,
+      qty: account.cash_balance,
       price: 1,
-      amount: account.balance,
-      currency: account.currency,
+      amount: account.cash_balance,
+      currency: currency.iso_code,
       security: Security.new(ticker: currency.iso_code, name: currency.name)
   end
 end

app/helpers/languages_helper.rb

@@ -363,4 +363,8 @@ def language_options
       end
       .sort_by { |label, locale| label }
   end
+
+  def timezone_options
+    ActiveSupport::TimeZone.all.map { |tz| [ tz.name + " (#{tz.tzinfo.identifier})", tz.tzinfo.identifier ] }
+  end
 end

app/models/account.rb

@@ -16,7 +16,7 @@ class Account < ApplicationRecord
   has_many :balances, dependent: :destroy
   has_many :issues, as: :issuable, dependent: :destroy
 
-  monetize :balance
+  monetize :balance, :cash_balance
 
   enum :classification, { asset: "asset", liability: "liability" }, validate: { allow_nil: true }
 
@@ -32,8 +32,6 @@ class Account < ApplicationRecord
 
   accepts_nested_attributes_for :accountable, update_only: true
 
-  delegate :value, :series, to: :accountable
-
   class << self
     def by_group(period: Period.all, currency: Money.default_currency.iso_code)
       grouped_accounts = { assets: ValueGroup.new("Assets", currency), liabilities: ValueGroup.new("Liabilities", currency) }
@@ -59,7 +57,7 @@ def by_group(period: Period.all, currency: Money.default_currency.iso_code)
 
     def create_and_sync(attributes)
       attributes[:accountable_attributes] ||= {} # Ensure accountable is created, even if empty
-      account = new(attributes)
+      account = new(attributes.merge(cash_balance: attributes[:balance]))
 
       transaction do
         # Create 2 valuations for new accounts to establish a value history for users to see
@@ -94,15 +92,27 @@ def destroy_later
   def sync_data(start_date: nil)
     update!(last_synced_at: Time.current)
 
-    resolve_stale_issues
-    Balance::Syncer.new(self, start_date: start_date).run
-    Holding::Syncer.new(self, start_date: start_date).run
+    Syncer.new(self, start_date: start_date).run
   end
 
   def post_sync
+    broadcast_remove_to(family, target: "syncing-notice")
+    resolve_stale_issues
     accountable.post_sync
   end
 
+  def series(period: Period.last_30_days, currency: nil)
+    balance_series = balances.in_period(period).where(currency: currency || self.currency)
+
+    if balance_series.empty? && period.date_range.end == Date.current
+      TimeSeries.new([ { date: Date.current, value: balance_money.exchange_to(currency || self.currency) } ])
+    else
+      TimeSeries.from_collection(balance_series, :balance_money, favorable_direction: asset? ? "up" : "down")
+    end
+  rescue Money::ConversionError
+    TimeSeries.new([])
+  end
+
   def original_balance
     balance_amount = balances.chronological.first&.balance || balance
     Money.new(balance_amount, currency)