From 7c0dfc7c82e9637f014edd468f3526a829fbc078 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Wed, 28 Sep 2022 16:45:32 +0000 Subject: [PATCH] feat: amortize streams --- .../daily_cashflow_averages.json | 374 +++++++++++++++++- yearn/entities.py | 306 ++++++++++++-- .../treasury/accountant/expenses/__init__.py | 1 - yearn/treasury/accountant/expenses/people.py | 13 - yearn/treasury/accountant/ignore/__init__.py | 1 + yearn/treasury/accountant/ignore/general.py | 3 + yearn/treasury/streams.py | 104 +++++ 7 files changed, 752 insertions(+), 50 deletions(-) create mode 100644 yearn/treasury/streams.py diff --git a/grafana/provisioning/dashboards/treasury-dashboards/daily_cashflow_averages.json b/grafana/provisioning/dashboards/treasury-dashboards/daily_cashflow_averages.json index 03831ce30..3afd3e1e2 100644 --- a/grafana/provisioning/dashboards/treasury-dashboards/daily_cashflow_averages.json +++ b/grafana/provisioning/dashboards/treasury-dashboards/daily_cashflow_averages.json @@ -25,6 +25,358 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "Considers all txs except YFI-based comp", + "type": "row" + }, + { + "datasource": "POSTGRES", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": "POSTGRES", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select *\nfrom (\n SELECT\n date as time,\n sum(daily_total),\n 'Cash Out' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Cost of Revenue','Operating Expenses','Other Operating Expense') and\n (a.token != 'YFI' or a.token is null)\n GROUP BY date\n union\n SELECT\n date as time,\n sum(daily_total),\n 'Cash In' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Protocol Revenue','Other Income') and\n (a.token != 'YFI' or a.token is null)\n GROUP BY date\n) final\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Daily Avg. for Period (Streams are expensed daily)", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Net", + "binary": { + "left": "Cash In", + "operator": "-", + "reducer": "sum", + "right": "Cash Out" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "stat" + }, + { + "datasource": "POSTGRES", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": "POSTGRES", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with groups as (\n SELECT distinct txgroup_id\n from treasury_time_averages\n where $__timeFilter(date) and daily_total != 0\n)\n\nselect *\nfrom (\n SELECT\n date as time,\n sum(daily_total),\n b.top_level_account as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account = 'Other Income' and\n a.txgroup_id in (select * from groups) and\n (a.token != 'YFI' or a.token is null)\n GROUP BY date, b.top_level_account\n union\n SELECT\n date as time,\n sum(daily_total),\n b.subaccount1 as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account = 'Protocol Revenue' and\n b.subaccount1 != 'Fees' and\n a.txgroup_id in (select * from groups) and\n (a.token != 'YFI' or a.token is null)\n GROUP BY date, b.subaccount1\n union\n SELECT\n date as time,\n sum(daily_total),\n b.subaccount2 as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account = 'Protocol Revenue' and\n b.subaccount1 = 'Fees' and\n b.subaccount2 is not null and\n a.txgroup_id in (select * from groups) and\n (a.token != 'YFI' or a.token is null)\n GROUP BY date, b.subaccount2\n) final\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Average Daily Revenue for Period", + "transformations": [], + "type": "stat" + }, + { + "datasource": "POSTGRES", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": "POSTGRES", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with groups as (\n SELECT distinct txgroup_id\n from treasury_time_averages\n where $__timeFilter(date) and daily_total != 0\n)\n\nSELECT\n date as time,\n sum(daily_total),\n b.subaccount1 as metric\nFROM treasury_time_averages a\nleft join txgroup_parentage b on a.txgroup_id = b.txgroup_id\nWHERE\n $__timeFilter(date) and\n b.top_level_account = 'Cost of Revenue' and\n a.txgroup_id in (select * from groups) and\n (a.token != 'YFI' or a.token is null)\nGROUP BY date, b.subaccount1\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Average Daily Cost of Revenue for Period", + "transformations": [], + "type": "stat" + }, + { + "datasource": "POSTGRES", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": "POSTGRES", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with groups as (\n SELECT distinct txgroup_id\n from treasury_time_averages\n where $__timeFilter(date) and daily_total != 0\n)\n\nselect *\nfrom (\n SELECT\n date as time,\n sum(daily_total),\n b.top_level_account as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Other Operating Expense') and\n a.txgroup_id in (select * from groups) and\n (a.token != 'YFI' or a.token is null)\n GROUP BY date, b.top_level_account\n union\n SELECT\n date as time,\n sum(daily_total),\n b.subaccount1 as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Operating Expenses') and\n b.subaccount1 is not null and\n a.txgroup_id in (select * from groups) and\n (a.token != 'YFI' or a.token is null)\n GROUP BY date, b.subaccount1\n) final\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Average Daily Spend for Period", + "transformations": [], + "type": "stat" + }, { "datasource": "POSTGRES", "description": "", @@ -83,9 +435,9 @@ "h": 9, "w": 24, "x": 0, - "y": 0 + "y": 22 }, - "id": 4, + "id": 14, "options": { "legend": { "calcs": [], @@ -103,7 +455,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select *\nfrom (\n SELECT\n date as time,\n sum(average_14d),\n 'Cash Out' as metric\n FROM treasury_time_averages a\n left join txgroups b on a.parent_txgroup_id = b.txgroup_id\n left join txgroups c on b.parent_txgroup = c.txgroup_id\n left join txgroups d on c.parent_txgroup = d.txgroup_id\n left join txgroups e on d.parent_txgroup = e.txgroup_id\n WHERE\n $__timeFilter(date) and\n coalesce(e.name, d.name, c.name, b.name, a.parent_txgroup) in ('Cost of Revenue','Operating Expenses','Other Operating Expense')\n group by date\n union\n SELECT\n date as time,\n sum(average_14d),\n 'Cash In' as metric\n FROM treasury_time_averages a\n left join txgroups b on a.parent_txgroup_id = b.txgroup_id\n left join txgroups c on b.parent_txgroup = c.txgroup_id\n left join txgroups d on c.parent_txgroup = d.txgroup_id\n left join txgroups e on d.parent_txgroup = e.txgroup_id\n WHERE\n $__timeFilter(date) and\n coalesce(e.name, d.name, c.name, b.name, a.parent_txgroup) in ('Protocol Revenue','Other Income')\n group by date\n) final\norder by time", + "rawSql": "select *\nfrom (\n SELECT\n date as time,\n sum(average_14d),\n 'Cash Out' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Cost of Revenue','Operating Expenses','Other Operating Expense') and\n (a.token != 'YFI' or a.token is null)\n group by date\n union\n SELECT\n date as time,\n sum(average_14d),\n 'Cash In' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Protocol Revenue','Other Income') and\n (a.token != 'YFI' or a.token is null)\n group by date\n) final\norder by time", "refId": "A", "select": [ [ @@ -203,9 +555,9 @@ "h": 9, "w": 24, "x": 0, - "y": 9 + "y": 31 }, - "id": 2, + "id": 15, "options": { "legend": { "calcs": [], @@ -223,7 +575,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select *\nfrom (\n SELECT\n date as time,\n sum(average_30d),\n 'Cash Out' as metric\n FROM treasury_time_averages a\n left join txgroups b on a.parent_txgroup_id = b.txgroup_id\n left join txgroups c on b.parent_txgroup = c.txgroup_id\n left join txgroups d on c.parent_txgroup = d.txgroup_id\n left join txgroups e on d.parent_txgroup = e.txgroup_id\n WHERE\n $__timeFilter(date) and\n coalesce(e.name, d.name, c.name, b.name, a.parent_txgroup) in ('Cost of Revenue','Operating Expenses','Other Operating Expense')\n group by date\n union\n SELECT\n date as time,\n sum(average_30d),\n 'Cash In' as metric\n FROM treasury_time_averages a\n left join txgroups b on a.parent_txgroup_id = b.txgroup_id\n left join txgroups c on b.parent_txgroup = c.txgroup_id\n left join txgroups d on c.parent_txgroup = d.txgroup_id\n left join txgroups e on d.parent_txgroup = e.txgroup_id\n WHERE\n $__timeFilter(date) and\n coalesce(e.name, d.name, c.name, b.name, a.parent_txgroup) in ('Protocol Revenue','Other Income')\n group by date\n) final\norder by time", + "rawSql": "select *\nfrom (\n SELECT\n date as time,\n sum(average_30d),\n 'Cash Out' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Cost of Revenue','Operating Expenses','Other Operating Expense') and\n (a.token != 'YFI' or a.token is null)\n group by date\n union\n SELECT\n date as time,\n sum(average_30d),\n 'Cash In' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Protocol Revenue','Other Income') and\n (a.token != 'YFI' or a.token is null)\n group by date\n) final\norder by time", "refId": "A", "select": [ [ @@ -323,9 +675,9 @@ "h": 9, "w": 24, "x": 0, - "y": 18 + "y": 40 }, - "id": 3, + "id": 16, "options": { "legend": { "calcs": [], @@ -343,7 +695,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select *\nfrom (\n SELECT\n date as time,\n sum(average_90d),\n 'Cash Out' as metric\n FROM treasury_time_averages a\n left join txgroups b on a.parent_txgroup_id = b.txgroup_id\n left join txgroups c on b.parent_txgroup = c.txgroup_id\n left join txgroups d on c.parent_txgroup = d.txgroup_id\n left join txgroups e on d.parent_txgroup = e.txgroup_id\n WHERE\n $__timeFilter(date) and\n coalesce(e.name, d.name, c.name, b.name, a.parent_txgroup) in ('Cost of Revenue','Operating Expenses','Other Operating Expense')\n group by date\n union\n SELECT\n date as time,\n sum(average_90d),\n 'Cash In' as metric\n FROM treasury_time_averages a\n left join txgroups b on a.parent_txgroup_id = b.txgroup_id\n left join txgroups c on b.parent_txgroup = c.txgroup_id\n left join txgroups d on c.parent_txgroup = d.txgroup_id\n left join txgroups e on d.parent_txgroup = e.txgroup_id\n WHERE\n $__timeFilter(date) and\n coalesce(e.name, d.name, c.name, b.name, a.parent_txgroup) in ('Protocol Revenue','Other Income')\n group by date\n) final\norder by time", + "rawSql": "select *\nfrom (\n SELECT\n date as time,\n sum(average_90d),\n 'Cash Out' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Cost of Revenue','Operating Expenses','Other Operating Expense') and\n (a.token != 'YFI' or a.token is null)\n group by date\n union\n SELECT\n date as time,\n sum(average_90d),\n 'Cash In' as metric\n FROM treasury_time_averages a\n left join txgroup_parentage b on a.txgroup_id = b.txgroup_id\n WHERE\n $__timeFilter(date) and\n b.top_level_account in ('Protocol Revenue','Other Income') and\n (a.token != 'YFI' or a.token is null)\n group by date\n) final\norder by time", "refId": "A", "select": [ [ @@ -395,13 +747,13 @@ "list": [] }, "time": { - "from": "now-2y", + "from": "now-30d", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Daily Cashflow Averages", "uid": "Usi2_nMVk", - "version": 2, + "version": 3, "weekStart": "" } \ No newline at end of file diff --git a/yearn/entities.py b/yearn/entities.py index 4a071c418..e7fcdba94 100644 --- a/yearn/entities.py +++ b/yearn/entities.py @@ -1,12 +1,16 @@ import os -from datetime import datetime +import typing +from datetime import date, datetime, timedelta from decimal import Decimal from functools import cached_property, lru_cache -from brownie import chain +from brownie import Contract, chain from brownie.network.transaction import TransactionReceipt from pony.orm import * +from yearn.treasury.constants import BUYER +from yearn.utils import closest_block_after_timestamp, contract + db = Database() @@ -65,6 +69,9 @@ class Address(db.Entity): user_tx_to = Set("UserTx", reverse="to_address") treasury_tx_from = Set("TreasuryTx", reverse="from_address") treasury_tx_to = Set("TreasuryTx", reverse="to_address") + streams_from = Set("Stream", reverse="from_address") + streams_to = Set("Stream", reverse="to_address") + streams = Set("Stream", reverse="contract") class Token(db.Entity): @@ -80,6 +87,7 @@ class Token(db.Entity): treasury_tx = Set('TreasuryTx', reverse="token") partner_harvest_event = Set('PartnerHarvestEvent', reverse="vault") address = Required(Address, column="address_id") + streams = Set('Stream', reverse="token") @property def scale(self) -> int: @@ -119,6 +127,7 @@ class TxGroup(db.Entity): treasury_tx = Set('TreasuryTx', reverse="txgroup") parent_txgroup = Optional("TxGroup", reverse="child_txgroups") child_txgroups = Set("TxGroup", reverse="parent_txgroup") + streams = Set("Stream", reverse="txgroup") @property def top_txgroup(self): @@ -190,6 +199,159 @@ def _symbol(self): return self.token.symbol +v3_multisig = "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7" + +class Stream(db.Entity): + _table_ = 'streams' + stream_id = PrimaryKey(str) + + contract = Required(Address, reverse="streams") + start_block = Required(int) + end_block = Optional(int) + token = Required(Token, reverse='streams', index=True) + from_address = Required(Address, reverse='streams_from') + to_address = Required(Address, reverse='streams_to') + amount_per_second = Required(Decimal, 38, 1) + status = Required(str, default="Active") + txgroup = Optional(TxGroup, reverse="streams") + + streamed_funds = Set("StreamedFunds") + + # NOTE Streams always use 20 decimals + scale = int(1e20) + + @property + def is_alive(self) -> bool: + if self.end_block is None: + assert self.status in ["Active", "Paused"] + return self.status == "Active" + assert self.status == "Stopped" + return False + + @property + def amount_per_minute(self) -> int: + return self.amount_per_second * 60 + + @property + def amount_per_hour(self) -> int: + return self.amount_per_minute * 60 + + @property + def amount_per_day(self) -> int: + return self.amount_per_hour * 24 + + def stop_stream(self, block: int) -> None: + self.end_block = block + self.status = "Stopped" + + def pause(self) -> None: + self.status = "Paused" + + @classmethod + def get_or_create_entity(cls, log) -> "Stream": + if len(log.values()) == 4: # StreamCreated + from_address, to_address, amount_per_second, stream_id = log.values() + elif len(log.values()) == 7: # StreamModified + from_address, _, _, _, to_address, amount_per_second, stream_id = log.values() + else: + raise NotImplementedError("This is not an appropriate event log.") + + stream_id = stream_id.hex() + + try: + return Stream[stream_id] + except ObjectNotFound: + from yearn.outputs.postgres.utils import (cache_address, + cache_token, + cache_txgroup) + + txgroup = { + BUYER: "Top-up Buyer Contract", + v3_multisig: "V3 Development", + }.get(to_address, "Other Grants") + + txgroup = cache_txgroup(txgroup) + stream_contract = cache_address(log.address) + token = cache_token(contract(log.address).token()) + from_address = cache_address(from_address) + to_address = cache_address(to_address) + + entity = Stream( + stream_id = stream_id, + contract = stream_contract, + start_block = log.block_number, + token = token, + from_address = from_address, + to_address = to_address, + amount_per_second = amount_per_second, + txgroup = txgroup, + ) + commit() + return entity + + @property + def stream_contract(self) -> Contract: + return contract(self.contract.address) + + def start_timestamp(self, block: typing.Optional[int] = None) -> int: + return int(self.stream_contract.streamToStart('0x' + self.stream_id, block_identifier=block)) + + def amount_withdrawable(self, block: int): + return self.stream_contract.withdrawable.call( + self.from_address.address, + self.to_address.address, + int(self.amount_per_second), + block_identifier = block, + ) + + def print(self): + symbol = self.token.symbol + print(f'{symbol} per second: {self.amount_per_second / self.scale}') + print(f'{symbol} per day: {self.amount_per_day / self.scale}') + +one_day = 60 * 60 * 24 + +class StreamedFunds(db.Entity): + _table_ = "streamed_funds" + + date = Required(date) + stream = Required(Stream, reverse="streamed_funds") + PrimaryKey(stream, date) + + amount = Required(Decimal, 38, 18) + price = Required(Decimal, 38, 18) + value_usd = Required(Decimal, 38, 18) + seconds_active = Required(int) + + @classmethod + def get_or_create_entity(cls, stream: Stream, date: datetime) -> "StreamedFunds": + if entity := StreamedFunds.get(date=date, stream=stream): + return entity + + check_at = date + timedelta(days=1) - timedelta(seconds=1) + block = closest_block_after_timestamp(check_at.timestamp()) + start_timestamp = stream.start_timestamp(block) + if start_timestamp == 0: + print("Stream cancelled. Must handle.") + return + + seconds_active = int(check_at.timestamp()) - start_timestamp + seconds_active_today = seconds_active if seconds_active < one_day else one_day + amount_streamed_today = stream.amount_per_second * seconds_active_today / stream.scale + + # We only need this for this function so we import in this function to save time where this function isn't needed. + from yearn.prices import magic + + price = Decimal(magic.get_price(stream.token.address.address, block)) + + return StreamedFunds( + date = date, + stream = stream, + amount = amount_streamed_today, + price = price, + value_usd = amount_streamed_today * price, + seconds_active = seconds_active_today, + ) # Caching for partners.py class PartnerHarvestEvent(db.Entity): @@ -223,21 +385,88 @@ class PartnerHarvestEvent(db.Entity): db.generate_mapping(create_tables=True) +@db_session +def create_txgroup_parentage_view() -> None: + try: + db.execute( + """ + CREATE VIEW txgroup_parentage as + SELECT a.txgroup_id, + COALESCE(d.name,c.name, b.name, a.name) top_level_account, + CASE WHEN d.name is not null THEN c.name when c.name is not null THEN b.name when b.name IS not NULL THEN a.name else null end subaccount1, + CASE when d.name is not null THEN b.name when c.name IS not NULL THEN a.name else null end subaccount2, + CASE when d.name IS not NULL THEN a.name else null end subaccount3 + FROM txgroups a + LEFT JOIN txgroups b ON a.parent_txgroup = b.txgroup_id + LEFT JOIN txgroups c ON b.parent_txgroup = c.txgroup_id + LEFT JOIN txgroups d ON c.parent_txgroup = d.txgroup_id + """ + ) + except ProgrammingError as e: + if str(e).strip() != 'relation "txgroup_parentage" already exists': + raise + +@db_session +def create_stream_ledger_view() -> None: + """ + Prepares daily stream data with the same structure as general_ledger view so we can combine them with a union. + """ + + try: + db.execute( + """ + create view stream_ledger as + SELECT 'Mainnet' as chain_name, + cast(DATE AS timestamp) as timestamp, + NULL as block, + NULL as hash, + NULL as log_index, + symbol as token, + d.address AS "from", + d.nickname as from_nickname, + e.address as "to", + e.nickname as to_nickname, + amount, + price, + value_usd, + txgroup.name as txgroup, + parent.name as parent_txgroup, + txgroup.txgroup_id + FROM streamed_funds a + LEFT JOIN streams b ON a.stream = b.stream_id + LEFT JOIN tokens c ON b.token = c.token_id + LEFT JOIN addresses d ON b.from_address = d.address_id + LEFT JOIN addresses e ON b.to_address = e.address_id + LEFT JOIN txgroups txgroup ON b.txgroup = txgroup.txgroup_id + LEFT JOIN txgroups parent ON txgroup.parent_txgroup = parent.txgroup_id + """ + ) + except ProgrammingError as e: + if str(e).strip() != 'relation "stream_ledger" already exists': + raise + + @db_session def create_general_ledger_view() -> None: try: db.execute( """ create VIEW general_ledger as - SELECT b.chain_name, TO_TIMESTAMP(a.timestamp) AS timestamp, a.block, a.hash, a.log_index, c.symbol AS token, d.address AS "from", d.nickname as from_nickname, e.address AS "to", e.nickname as to_nickname, a.amount, a.price, a.value_usd, f.name AS txgroup, g.name AS parent_txgroup - FROM treasury_txs a - LEFT JOIN chains b ON a.chain = b.chain_dbid - LEFT JOIN tokens c ON a.token_id = c.token_id - LEFT JOIN addresses d ON a."from" = d.address_id - LEFT JOIN addresses e ON a."to" = e.address_id - LEFT JOIN txgroups f ON a.txgroup_id = f.txgroup_id - LEFT JOIN txgroups g ON f.parent_txgroup = g.txgroup_id - ORDER BY TO_TIMESTAMP(a.timestamp) + select * + from ( + SELECT b.chain_name, TO_TIMESTAMP(a.timestamp) AS timestamp, a.block, a.hash, a.log_index, c.symbol AS token, d.address AS "from", d.nickname as from_nickname, e.address AS "to", e.nickname as to_nickname, a.amount, a.price, a.value_usd, f.name AS txgroup, g.name AS parent_txgroup, f.txgroup_id + FROM treasury_txs a + LEFT JOIN chains b ON a.chain = b.chain_dbid + LEFT JOIN tokens c ON a.token_id = c.token_id + LEFT JOIN addresses d ON a."from" = d.address_id + LEFT JOIN addresses e ON a."to" = e.address_id + LEFT JOIN txgroups f ON a.txgroup_id = f.txgroup_id + LEFT JOIN txgroups g ON f.parent_txgroup = g.txgroup_id + UNION + SELECT chain_name, TIMESTAMP, cast(block AS integer) block, hash, CAST(log_index AS integer) as log_index, token, "from", from_nickname, "to", to_nickname, amount, price, value_usd, txgroup, parent_txgroup, txgroup_id + FROM stream_ledger + ) a + ORDER BY timestamp """ ) except ProgrammingError as e: @@ -267,25 +496,28 @@ def create_treasury_time_averages_view() -> None: """ CREATE VIEW treasury_time_averages AS WITH base AS ( - SELECT gs as DATE, a.NAME AS txgroup, b.name as parent_txgroup, b.txgroup_id AS parent_txgroup_id - FROM txgroups a - LEFT JOIN txgroups b ON a.parent_txgroup = b.txgroup_id - LEFT JOIN generate_series('2020-07-21', '2022-09-03', interval '1 day') gs ON 1=1 + SELECT gs as DATE, txgroup_id, token + FROM ( + SELECT DISTINCT grp.txgroup_id, gl.token + FROM txgroups grp + LEFT JOIN general_ledger gl ON grp.txgroup_id = gl.txgroup_id + ) a + LEFT JOIN generate_series('2020-07-21', CURRENT_DATE, interval '1 day') gs ON 1=1 ), summed AS ( SELECT DATE, - coalesce(sum(value_usd), 0) daily_total, - a.txgroup, - a.parent_txgroup, - a.parent_txgroup_id + a.txgroup_id, + a.token, + coalesce(sum(value_usd), 0) daily_total FROM base a - left join general_ledger b ON date = CAST(TIMESTAMP AS DATE) and a.txgroup = b.txgroup AND a.parent_txgroup = b.parent_txgroup - GROUP BY date, a.txgroup, a.parent_txgroup, a.parent_txgroup_id + left join general_ledger b ON date = CAST(TIMESTAMP AS DATE) and a.txgroup_id = b.txgroup_id AND a.token = b.token + GROUP BY date, a.txgroup_id, a.token ) + SELECT *, - sum(daily_total) OVER (partition BY txgroup, parent_txgroup ORDER BY date ROWS 6 PRECEDING) / 7 average_7d, - sum(daily_total) OVER (partition BY txgroup, parent_txgroup ORDER BY date ROWS 13 PRECEDING) / 14 average_14d, - sum(daily_total) OVER (partition BY txgroup, parent_txgroup ORDER BY date ROWS 29 PRECEDING) / 30 average_30d, - sum(daily_total) OVER (partition BY txgroup, parent_txgroup ORDER BY date ROWS 89 PRECEDING) / 90 average_90d + sum(daily_total) OVER (partition BY txgroup_id, token ORDER BY date ROWS 6 PRECEDING) / 7 average_7d, + sum(daily_total) OVER (partition BY txgroup_id, token ORDER BY date ROWS 13 PRECEDING) / 14 average_14d, + sum(daily_total) OVER (partition BY txgroup_id, token ORDER BY date ROWS 29 PRECEDING) / 30 average_30d, + sum(daily_total) OVER (partition BY txgroup_id, token ORDER BY date ROWS 89 PRECEDING) / 90 average_90d FROM summed ORDER BY DATE """ @@ -294,7 +526,31 @@ def create_treasury_time_averages_view() -> None: if str(e).strip() != 'relation "treasury_time_averages" already exists': raise +@db_session +def create_txgroup_parentage_view() -> None: + try: + db.execute( + """ + CREATE VIEW txgroup_parentage as + SELECT a.txgroup_id, + COALESCE(d.name,c.name, b.name, a.name) top_level_account, + CASE WHEN d.name is not null THEN c.name when c.name is not null THEN b.name when b.name IS not NULL THEN a.name else null end subaccount1, + CASE when d.name is not null THEN b.name when c.name IS not NULL THEN a.name else null end subaccount2, + CASE when d.name IS not NULL THEN a.name else null end subaccount3 + FROM txgroups a + LEFT JOIN txgroups b ON a.parent_txgroup = b.txgroup_id + LEFT JOIN txgroups c ON b.parent_txgroup = c.txgroup_id + LEFT JOIN txgroups d ON c.parent_txgroup = d.txgroup_id + """ + ) + except ProgrammingError as e: + if str(e).strip() != 'relation "txgroup_parentage" already exists': + raise + + def create_views() -> None: + create_txgroup_parentage_view() + create_stream_ledger_view() create_general_ledger_view() create_unsorted_txs_view() create_treasury_time_averages_view() diff --git a/yearn/treasury/accountant/expenses/__init__.py b/yearn/treasury/accountant/expenses/__init__.py index 6439ccccd..0750044bd 100644 --- a/yearn/treasury/accountant/expenses/__init__.py +++ b/yearn/treasury/accountant/expenses/__init__.py @@ -11,7 +11,6 @@ if chain.id == Network.Mainnet: team = expenses_txgroup.create_child("Team Payments", people.is_team_payment) - team.create_child("Replenish Streams", people.is_stream_replenishment) expenses_txgroup.create_child("Coordinape", people.is_coordinape) expenses_txgroup.create_child("The 0.03%", people.is_0_03_percent) diff --git a/yearn/treasury/accountant/expenses/people.py b/yearn/treasury/accountant/expenses/people.py index 2b84d6f3f..fd58ed87b 100644 --- a/yearn/treasury/accountant/expenses/people.py +++ b/yearn/treasury/accountant/expenses/people.py @@ -35,14 +35,6 @@ def is_team_payment(tx: TreasuryTx) -> bool: if tx.hash == '0x91656eb3246a560498e808cbfb00af0518ec12f50effda42fc1496d0b0fb080a' and tx.log_index == 421: return False - llamapay_hashes = [ - "0xd268946b6937df798e965a98b3f9348f7fc8519a2df9ba124210e4ce6c3fecaf", - "0x9d8fc48fe2f552a424fa2e4fa35f2ddbe73eb9f1eae33bb3b7b27466b8dbb62f", - "0x7979a77ab8a30bc6cd12e1df92e5ba0478a8907caf6e100317b7968668d0d4a2", - "0x91656eb3246a560498e808cbfb00af0518ec12f50effda42fc1496d0b0fb080a", - "0x16c193b8891af35ec811ebe8416f25addc0c4ffe39e074b5820577f1d8be72ec", - ] - zerox_splits_hashes = [ "0x38dbe0e943d978483439d4dec9e29f71fc4690e790261661ef3b2d179dee26b9" ] @@ -51,8 +43,6 @@ def is_team_payment(tx: TreasuryTx) -> bool: return True elif tx in HashMatcher(disperse_app_hashes) and tx._from_nickname == 'Disperse.app': return True - elif tx in HashMatcher(llamapay_hashes) and tx._to_nickname == "Contract: LlamaPay": - return True elif tx in HashMatcher(zerox_splits_hashes) and tx._to_nickname == "Non-Verified Contract: 0x426eF4C1d57b4772BcBE23979f4bbb236c135e1C": return True return False @@ -110,9 +100,6 @@ def is_other_grant(tx: TreasuryTx) -> bool: return tx in HashMatcher(disperse_hashes) return tx in HashMatcher(hashes) -def is_stream_replenishment(tx: TreasuryTx) -> bool: - pass - def is_0_03_percent(tx: TreasuryTx) -> bool: return tx in HashMatcher([["0xe56521d79b0b87425e90c08ed3da5b4fa3329a40fe31597798806db07f68494e", Filter('_from_nickname', 'Disperse.app')]]) diff --git a/yearn/treasury/accountant/ignore/__init__.py b/yearn/treasury/accountant/ignore/__init__.py index 5b90ec86f..7cf0ece3a 100644 --- a/yearn/treasury/accountant/ignore/__init__.py +++ b/yearn/treasury/accountant/ignore/__init__.py @@ -149,6 +149,7 @@ def is_keep_crv(tx: TreasuryTx) -> bool: ignore_txgroup.create_child("Transfer to yGov (Deprecated)", ygov.is_sent_to_ygov) ignore_txgroup.create_child("Maker CDP Deposit", maker.is_yfi_cdp_deposit) ignore_txgroup.create_child("Maker CDP Withdrawal", maker.is_yfi_cdp_withdrawal) + ignore_txgroup.create_child("Replenish Streams", general.is_stream_replenishment) elif chain.id == Network.Fantom: ignore_txgroup.create_child("OTCTrader", general.is_otc_trader) diff --git a/yearn/treasury/accountant/ignore/general.py b/yearn/treasury/accountant/ignore/general.py index da639377f..9a707f054 100644 --- a/yearn/treasury/accountant/ignore/general.py +++ b/yearn/treasury/accountant/ignore/general.py @@ -122,6 +122,9 @@ def is_weth(tx: TreasuryTx) -> bool: if tx.from_address.address == constants.weth and tx.to_address and tx.to_address.address in treasury.addresses and tx.token.address.address == "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE": return True +def is_stream_replenishment(tx: TreasuryTx) -> bool: + return tx._to_nickname == "Contract: LlamaPay" + def is_scam_airdrop(tx: TreasuryTx) -> bool: hashes = { Network.Mainnet: [ diff --git a/yearn/treasury/streams.py b/yearn/treasury/streams.py new file mode 100644 index 000000000..97e0fb440 --- /dev/null +++ b/yearn/treasury/streams.py @@ -0,0 +1,104 @@ + +from typing import List, Optional + +from brownie import chain +from pony.orm import db_session, select +from yearn.constants import YCHAD_MULTISIG, YFI +from yearn.entities import Stream, Token, TxGroup +from yearn.events import decode_logs, get_logs_asap +from yearn.outputs.postgres.utils import cache_token +from yearn.treasury.constants import BUYER +from yearn.utils import contract + +dai = "0x6B175474E89094C44Da98b954EedeAC495271d0F" + +streams_dai = contract('0x60c7B0c5B3a4Dc8C690b074727a17fF7aA287Ff2') +streams_yfi = contract('0xf3764eC89B1ad20A31ed633b1466363FAc1741c4') + +class YearnStreams: + def __init__(self): + assert chain.id == 1 + self.stream_contracts = [streams_dai, streams_yfi] + self.skipped_events = ["PayerDeposit", "Withdraw"] + self.handled_events = ["StreamCreated", "StreamModified", "StreamPaused", "StreamCancelled"] + self.get_streams() + + def __getitem__(self, key: str): + if isinstance(key, bytes): + key = key.hex() + return Stream[key] + + def streams_for_recipient(self, recipient: str, at_block: Optional[int] = None) -> List[Stream]: + if at_block is None: + return list(select(s for s in Stream if s.to_address.address == recipient)) + return list(select(s for s in Stream if s.to_address.address == recipient and (s.end_block is None or at_block <= s.end_block))) + + def streams_for_token(self, token: Token, include_inactive: bool = False) -> List[Stream]: + if not isinstance(token, Token): + token = cache_token(token) + streams = list(select(s for s in Stream if s.token == token)) + if include_inactive is False: + streams = [s for s in streams if s.is_alive] + return streams + + def buyback_streams(self, include_inactive: bool = False) -> List[Stream]: + streams = self.streams_for_recipient(BUYER) + if include_inactive is False: + streams = [s for s in streams if s.is_alive] + return streams + + @db_session + def dai_streams(self, include_inactive: bool = False) -> List[Stream]: + return self.streams_for_token(dai, include_inactive=include_inactive) + + @db_session + def yfi_streams(self, include_inactive: bool = False) -> List[Stream]: + return self.streams_for_token(YFI, include_inactive=include_inactive) + + @db_session + def streams(self, include_inactive: bool = False): + if include_inactive is True: + return list(select(s for s in Stream)) + return list(select(s for s in Stream if s.is_alive)) + + @db_session + def get_streams(self): + for stream_contract in self.stream_contracts: + logs = decode_logs(get_logs_asap([stream_contract.address])) + + for k in logs.keys(): + if k not in self.handled_events and k not in self.skipped_events: + raise NotImplementedError(f"We need to build handling for {k} events.") + + for log in logs['StreamCreated']: + from_address, *_ = log.values() + if from_address != YCHAD_MULTISIG: + continue + Stream.get_or_create_entity(log) + + for log in logs['StreamModified']: + from_address, _, _, old_stream_id, *_ = log.values() + if from_address != YCHAD_MULTISIG: + continue + self[old_stream_id].stop_stream(log.block_number) + Stream.get_or_create_entity(log) + + if 'StreamPaused' in logs: + for log in logs['StreamPaused']: + from_address, *_, stream_id = log.values() + if from_address != YCHAD_MULTISIG: + continue + self[stream_id].pause(log.block_number) + + if 'StreamCancelled' in logs: + for log in logs['StreamCancelled']: + from_address, *_, stream_id = log.values() + if from_address != YCHAD_MULTISIG: + continue + self[stream_id].stop_stream(log.block_number) + + team_payments_txgroup = TxGroup.get(name = "Team Payments") + for stream in self.yfi_streams(include_inactive=True): + for stream in self.streams_for_recipient(stream.to_address.address): + if stream.txgroup is None: + stream.txgroup = team_payments_txgroup