"
+ exit 1
+fi
+DATA_DIR="$1"
RUNNING_COUNT=0
+# Suppress telemetry reporting for tests
+export ALGOTEST=1
function update_running_count() {
- PIDS=($(pgrep -u $(whoami) -x algod)) || true
+ PIDS=($(pgrep -u "$(whoami)" -x algod)) || true
RUNNING_COUNT=${#PIDS[@]}
}
function verify_at_least_one_running() {
# Starting up can take some time, so wait at least 2 seconds
- for TRIES in 1 2 3 4 5; do
+ for _ in 1 2 3 4 5; do
update_running_count
- if [ ${RUNNING_COUNT} -ge 1 ]; then
+ if [ "$RUNNING_COUNT" -ge 1 ]; then
return 0
fi
sleep .4
@@ -28,34 +35,32 @@ function verify_at_least_one_running() {
}
function verify_none_running() {
- local datadir=$1
-
# Shutting down can take some time, so wait at least 5 seconds
- for TRIES in 1 2 3 4 5; do
+ for _ in 1 2 3 4 5; do
update_running_count
- if [ ${RUNNING_COUNT} -eq 0 ]; then
+ if [ "$RUNNING_COUNT" -eq 0 ]; then
return 0
fi
sleep 1.4
done
echo "algod not expected to be running but it is"
- if [ -n "$datadir" ]; then
+ if [ -n "$DATA_DIR" ]; then
echo "last 20 lines of node.log:"
- tail -20 "$datadir/node.log"
+ tail -20 "$DATA_DIR/node.log"
echo "================================"
echo "stdout and stdin:"
- cat "$datadir/algod-out.log"
+ cat "$DATA_DIR/algod-out.log"
echo "================================"
- cat "$datadir/algod-err.log"
+ cat "$DATA_DIR/algod-err.log"
fi
exit 1
}
function verify_one_running() {
# Starting up can take some time, so retry up to 2 seconds
- for TRIES in 1 2 3 4 5; do
+ for _ in 1 2 3 4 5; do
update_running_count
- if [ ${RUNNING_COUNT} -eq 1 ]; then
+ if [ "$RUNNING_COUNT" -eq 1 ]; then
return 0
fi
sleep .4
@@ -70,33 +75,33 @@ verify_none_running
#----------------------
# Test that we can start & stop a generic node with no overrides
echo Verifying a generic node will start using goal
-goal node start -d ${DATADIR}
+goal node start -d "$DATA_DIR"
verify_at_least_one_running
echo Verifying we can stop it using goal
-goal node stop -d ${DATADIR}
-verify_none_running ${DATADIR}
+goal node stop -d "$DATA_DIR"
+verify_none_running
#----------------------
# Test that we can start a generic node straight with no overrides
echo Verifying a generic node will start directly
-algod -d ${DATADIR} &
+algod -d "$DATA_DIR" &
verify_at_least_one_running
-pkill -u $(whoami) -x algod || true
-verify_none_running ${DATADIR}
+pkill -u "$(whoami)" -x algod || true
+verify_none_running
#----------------------
-# Test that we can start a generic node against the datadir
-# but that we cannot start a second one against same datadir
-echo Verifying that the datadir algod lock works correctly
-algod -d ${DATADIR} &
+# Test that we can start a generic node against the data dir
+# but that we cannot start a second one against same data dir
+echo Verifying that the data dir algod lock works correctly
+algod -d "$DATA_DIR" &
verify_at_least_one_running
-algod -d ${DATADIR} &
+algod -d "$DATA_DIR" &
verify_at_least_one_running # one should still be running
verify_one_running # in fact, exactly one should still be running
# clean up
-pkill -u $(whoami) -x algod || true
-verify_none_running ${DATADIR}
+pkill -u "$(whoami)" -x algod || true
+verify_none_running
echo "----------------------------------------------------------------------"
echo " DONE: e2e_basic_start_stop"
diff --git a/test/scripts/e2e_subs/e2e-app-simple.sh b/test/scripts/e2e_subs/e2e-app-simple.sh
index e1f1458ce4..660487621d 100755
--- a/test/scripts/e2e_subs/e2e-app-simple.sh
+++ b/test/scripts/e2e_subs/e2e-app-simple.sh
@@ -122,3 +122,27 @@ ${gcmd} app optin --app-id $APPID --from $ACCOUNT
# Succeed in clearing state for the app
${gcmd} app clear --app-id $APPID --from $ACCOUNT
+
+
+# Empty program:
+printf ' ' > "${TEMPDIR}/empty_clear.teal"
+
+# Fail to compile an empty program
+RES=$(${gcmd} clerk compile "${TEMPDIR}/empty_clear.teal" 2>&1 | tr -d '\n' || true)
+EXPERROR='Cannot assemble empty program text'
+if [[ $RES != *"${EXPERROR}"* ]]; then
+ echo RES="$RES"
+ echo EXPERROR="$EXPERROR"
+ date '+clerk-compile-test FAIL wrong error for compiling empty program %Y%m%d_%H%M%S'
+ false
+fi
+
+# Fail to create an app because the clear program is empty
+RES=$(${gcmd} app create --creator "${ACCOUNT}" --approval-prog "${TEMPDIR}/simple.teal" --clear-prog "${TEMPDIR}/empty_clear.teal" --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0 2>&1 | tr -d '\n' || true)
+EXPERROR='Cannot assemble empty program text'
+if [[ $RES != *"${EXPERROR}"* ]]; then
+ echo RES="$RES"
+ echo EXPERROR="$EXPERROR"
+ date '+app-create-test FAIL wrong error for creating app with empty clear program %Y%m%d_%H%M%S'
+ false
+fi
diff --git a/test/scripts/e2e_tx_latency.py b/test/scripts/e2e_tx_latency.py
new file mode 100644
index 0000000000..b181002d90
--- /dev/null
+++ b/test/scripts/e2e_tx_latency.py
@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+#
+# Measure total percieved tx latency.
+# Submit transactions to algod, watch blocks for committed transaction.
+
+import argparse
+import atexit
+import base64
+import datetime
+import glob
+import json
+import logging
+import os
+import queue
+import re
+import shutil
+import statistics
+import subprocess
+import sys
+import tempfile
+import time
+import threading
+
+# pip install py-algorand-sdk
+import algosdk
+from algosdk.encoding import msgpack
+import algosdk.v2client
+import algosdk.v2client.algod
+
+logger = logging.getLogger(__name__)
+
+def openkmd(algodata):
+ kmdnetpath = sorted(glob.glob(os.path.join(algodata,'kmd-*','kmd.net')))[-1]
+ kmdnet = open(kmdnetpath, 'rt').read().strip()
+ kmdtokenpath = sorted(glob.glob(os.path.join(algodata,'kmd-*','kmd.token')))[-1]
+ kmdtoken = open(kmdtokenpath, 'rt').read().strip()
+ kmd = algosdk.kmd.KMDClient(kmdtoken, 'http://' + kmdnet)
+ return kmd
+
+def openalgod(algodata):
+ algodnetpath = os.path.join(algodata,'algod.net')
+ algodnet = open(algodnetpath, 'rt').read().strip()
+ algodtokenpath = os.path.join(algodata,'algod.token')
+ algodtoken = open(algodtokenpath, 'rt').read().strip()
+ algod = algosdk.v2client.algod.AlgodClient(algodtoken, 'http://' + algodnet)
+ return algod
+
+def addr_token_from_algod(algorand_data):
+ with open(os.path.join(algorand_data, 'algod.net')) as fin:
+ addr = fin.read().strip()
+ with open(os.path.join(algorand_data, 'algod.token')) as fin:
+ token = fin.read().strip()
+ if not addr.startswith('http'):
+ addr = 'http://' + addr
+ return addr, token
+
+def bstr(x):
+ if isinstance(x, bytes):
+ try:
+ return x.decode()
+ except:
+ pass
+ return x
+
+def obnice(ob):
+ if isinstance(ob, dict):
+ return {bstr(k):obnice(v) for k,v in ob.items()}
+ if isinstance(ob, list):
+ return [obnice(x) for x in ob]
+ return ob
+
+class TxLatencyTest:
+ def __init__(self, args, algorand_data=None, prev_round=None, out=None):
+ self.algorand_data = algorand_data
+ self.token = args.token
+ self.addr = args.addr
+ self.headers = header_list_to_dict(args.headers)
+ self.prev_round = prev_round
+ self.out = out or sys.stdout
+ self.lock = threading.Lock()
+ self.terminated = None
+ self._kmd = None
+ self._algod = None
+ self.privatekey = None
+ self.pubw = None
+ self.maxpubaddr = None
+ self.errors = []
+ self.statuses = []
+ self.jsonfile = None
+ self.sentq = queue.Queue()
+ self.txidq = queue.Queue()
+ self.period = 1/args.tps
+ self.sendcount = args.sendcount
+ self.go = True
+ self.roundTimes = {}
+ self.txTimes = []
+ return
+
+ def connect(self):
+ with self.lock:
+ self._connect()
+ return self._algod, self._kmd
+
+ def _connect(self):
+ if self._algod and self._kmd:
+ return
+
+ # should run from inside self.lock
+ algodata = self.algorand_data
+
+ logger.debug('pre kmd')
+ subprocess.run(['goal', 'kmd', 'start', '-t', '3600', '-d', algodata], timeout=5, check=True)
+ logger.debug('post kmd')
+ self._kmd = openkmd(algodata)
+ self._algod = self._algod_connect() #openalgod(algodata)
+
+ def algod(self):
+ with self.lock:
+ if self._algod is None:
+ self._algod = self._algod_connect()
+ return self._algod
+
+ def _algod_connect(self):
+ if self.algorand_data:
+ addr, token = addr_token_from_algod(self.algorand_data)
+ logger.debug('algod from %r, (%s %s)', self.algorand_data, addr, token)
+ else:
+ token = self.token
+ addr = self.addr
+ logger.debug('algod from args (%s %s)', self.addr, self.token)
+ self._algod = algosdk.v2client.algod.AlgodClient(token, addr, headers=self.headers)
+ return self._algod
+
+ def get_pub_wallet(self):
+ with self.lock:
+ self._connect()
+ if not (self.pubw and self.maxpubaddr):
+ # find private test node public wallet and its richest account
+ wallets = self._kmd.list_wallets()
+ pubwid = None
+ for xw in wallets:
+ if xw['name'] == 'unencrypted-default-wallet':
+ pubwid = xw['id']
+ pubw = self._kmd.init_wallet_handle(pubwid, '')
+ pubaddrs = self._kmd.list_keys(pubw)
+ maxamount = 0
+ maxpubaddr = None
+ for pa in pubaddrs:
+ pai = self._algod.account_info(pa)
+ if pai['amount'] > maxamount:
+ maxamount = pai['amount']
+ maxpubaddr = pai['address']
+ self.pubw = pubw
+ self.maxpubaddr = maxpubaddr
+ return self.pubw, self.maxpubaddr
+
+ def send_thread(self):
+ #opriv, opub = algosdk.account.generate_account()
+ algod = self.algod()
+ nextsend = time.time()
+ params = algod.suggested_params()
+ paramsMtime = nextsend
+ count = 0
+
+ while True:
+ txn = algosdk.transaction.PaymentTxn(self.maxpubaddr, 1000, params.first, params.last, params.gh, self.maxpubaddr, 1, gen=params.gen, flat_fee=True, note='{}_'.format(count).encode() + os.getrandom(8))
+ ptxid = txn.get_txid()
+ if self.privatekey:
+ stxn = txn.sign(self.privatekey)
+ else:
+ stxn = self._kmd.sign_transaction(self.pubw, '', txn)
+ txid = algod.send_transaction(stxn)
+ if ptxid != txid:
+ logger.error('python txid %s, API txid %s', ptxid, txid)
+ logger.debug('%r', txn.dictify())
+ sendt = time.time()
+ logger.debug('sent %s %f', txid, sendt)
+ self.sentq.put((txid, sendt))
+
+ if self.sendcount is not None:
+ self.sendcount -= 1
+ if self.sendcount <= 0:
+ # signal to consumer end of sending
+ self.sentq.put((None, None))
+ return
+
+ if sendt - paramsMtime > 5:
+ params = algod.suggested_params()
+ paramsMtime = sendt
+
+ while nextsend < sendt:
+ nextsend += self.period
+ time.sleep(nextsend - sendt)
+
+ def measure_thread(self):
+ lastround = self.prev_round
+ algod = self.algod()
+ while self.go:
+ b = self.nextblock(lastround)
+ if b is None:
+ print("got None nextblock. exiting")
+ return
+ b = msgpack.loads(b, strict_map_key=False, raw=True)
+ b = obnice(b)
+ nowround = b['block'].get('rnd', 0)
+ logger.debug('r%d', nowround)
+ if (lastround is not None) and (nowround != lastround + 1):
+ logger.info('round jump %d to %d', lastround, nowround)
+ self._block_handler(b)
+ lastround = nowround
+
+ def nextblock(self, lastround=None, retries=30):
+ trycount = 0
+ while (trycount < retries) and self.go:
+ trycount += 1
+ try:
+ return self._nextblock_inner(lastround)
+ except Exception as e:
+ if trycount >= retries:
+ logger.error('too many errors in nextblock retries')
+ raise
+ else:
+ logger.warning('error in nextblock(%r) (retrying): %s', lastround, e)
+ self._algod = None # retry with a new connection
+ time.sleep(1.2)
+ return None
+
+ def _nextblock_inner(self, lastround):
+ self.block_time = None
+ algod = self.algod()
+ if lastround is None:
+ status = algod.status()
+ lastround = status['last-round']
+ logger.debug('nextblock status last-round %s', lastround)
+ else:
+ try:
+ blk = algod.block_info(lastround + 1, response_format='msgpack')
+ if blk:
+ return blk
+ logger.warning('null block %d, lastround=%r', lastround+1, lastround)
+ except Exception as e:
+ pass
+ #logger.debug('could not get block %d: %s', lastround + 1, e, exc_info=True)
+ status = algod.status_after_block(lastround)
+ block_time = time.time() # the block has happened, don't count block data transit time
+ nbr = status['last-round']
+ retries = 30
+ while (nbr > lastround + 1) and self.go:
+ # if more than one block elapsed, we don't have a good time for either block
+ block_time = None
+ # try lastround+1 one last time
+ try:
+ blk = algod.block_info(lastround + 1, response_format='msgpack')
+ if blk:
+ return blk
+ logger.warning('null block %d, lastround=%r, status.last-round=%d', lastround+1, lastround, nbr)
+ time.sleep(1.1)
+ retries -= 1
+ if retries <= 0:
+ raise Exception("too many null block for %d", lastround+1)
+ except:
+ break
+ blk = algod.block_info(nbr, response_format='msgpack')
+ if blk:
+ self.block_time = block_time
+ return blk
+ raise Exception('got None for blk {}'.format(nbr))
+
+ def _block_handler(self, b):
+ block_time = self.block_time or time.time()
+ nowround = b['block'].get('rnd', 0)
+ self.roundTimes[nowround] = block_time
+ # throw away txns, count is kept in round differential ['block']['tc']
+ stxibs = b['block'].get('txns', [])
+ txids = []
+ for stxib in stxibs:
+ txn = stxib['txn']
+ #logger.debug('stxib.txn %r', txn)
+ hgi = stxib.pop('hgi', False)
+ if hgi:
+ txn['gh'] = b['block']['gh']
+ txn['gen'] = bstr(b['block']['gen'])
+ txnd = txn
+ txn = algosdk.transaction.Transaction.undictify(txn)
+ txid = txn.get_txid()
+ logger.debug('rx txn %r, txid %s', txnd, txid)
+ txids.append(txid)
+ self.txidq.put((txids, block_time))
+
+ def join_thread(self):
+ lastSendt = None
+ sentq = self.sentq
+ sentByTxid = {}
+ while self.go:
+ if sentq is not None:
+ try:
+ txid, sendt = self.sentq.get(block=True, timeout=0.2)
+ if sendt is None:
+ sentq = None
+ else:
+ lastSendt = sendt
+ sentByTxid[txid] = sendt
+ except queue.Empty:
+ pass
+ elif (lastSendt is None) or (time.time() > (lastSendt + 8)):
+ self.go = False
+ return
+ try:
+ txids, rxt = self.txidq.get(block=False)
+ for txid in txids:
+ sendt = sentByTxid.get(txid)
+ if sendt is not None:
+ dt = rxt - sendt
+ self.txTimes.append(dt)
+ logger.debug('rx %s %f', txid, dt)
+ self.out.write('{}\n'.format(dt))
+ else:
+ logger.debug('unk blk txid %r', txid)
+ except queue.Empty:
+ pass
+
+def header_list_to_dict(hlist):
+ if not hlist:
+ return None
+ p = re.compile(r':\s+')
+ out = {}
+ for x in hlist:
+ a, b = p.split(x, 1)
+ out[a] = b
+ return out
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('-d', '--algod', default=None, help='algod data dir')
+ ap.add_argument('-a', '--addr', default=None, help='algod host:port address')
+ ap.add_argument('-t', '--token', default=None, help='algod API access token')
+ ap.add_argument('--header', dest='headers', nargs='*', help='"Name: value" HTTP header (repeatable)')
+ ap.add_argument('--tps', type=float, default=5, help='TPS to send at')
+ ap.add_argument('--sendcount', type=int, default=50, help='number of test txns to send at 5 TPS')
+ ap.add_argument('--verbose', default=False, action='store_true')
+ args = ap.parse_args()
+
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+
+ algorand_data = args.algod or os.getenv('ALGORAND_DATA')
+ if not algorand_data and not ((args.token or args.headers) and args.addr):
+ sys.stderr.write('must specify algod data dir by $ALGORAND_DATA or -d/--algod; OR --a/--addr and -t/--token\n')
+ sys.exit(1)
+
+ out = sys.stdout
+
+ bot = TxLatencyTest(
+ args,
+ algorand_data,
+ prev_round=None,
+ out=out,
+ )
+
+ pubw, maxpubaddr = bot.get_pub_wallet()
+ logger.debug('get pub wallet -> %s %s', pubw, maxpubaddr)
+
+ sender = threading.Thread(target=bot.send_thread)
+ sender.start()
+ measure = threading.Thread(target=bot.measure_thread)
+ measure.start()
+ merge = threading.Thread(target=bot.join_thread)
+ merge.start()
+ sender.join()
+ measure.join()
+ merge.join()
+
+ txTimes = bot.txTimes
+ rounds = sorted(bot.roundTimes.keys())
+ prevRound = None
+ prevRoundTime = None
+ roundDts = []
+ for rnd in rounds:
+ rndTime = bot.roundTimes[rnd]
+ if (prevRound is not None) and (prevRound + 1 == rnd):
+ dt = rndTime - prevRoundTime
+ roundDts.append(dt)
+ prevRound = rnd
+ prevRoundTime = rndTime
+ roundTimeMean = statistics.mean(roundDts)
+ roundTimeMin = min(roundDts)
+ roundTimeMax = max(roundDts)
+ txMean = statistics.mean(txTimes)
+ tmin = min(txTimes)
+ tmax = max(txTimes)
+ out.write('# {} txns measured, {} rounds seen\n'.format(len(txTimes), len(rounds)))
+ out.write('# rnd (min={}, mean={}, max={})\n'.format(roundTimeMin, roundTimeMean, roundTimeMax))
+ out.write('# tx (min={}, mean={}, max={})\n'.format(tmin, txMean, tmax))
+ out.write('# tx (min={}, mean={}, max={}) (/(mean rnd))\n'.format(tmin/roundTimeMean, txMean/roundTimeMean, tmax/roundTimeMean))
+ return 0
+
+if __name__ == '__main__':
+ main()
diff --git a/test/scripts/latency_plot.py b/test/scripts/latency_plot.py
new file mode 100644
index 0000000000..aab94d4391
--- /dev/null
+++ b/test/scripts/latency_plot.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+#
+# process pingpong TotalLatencyOut output file into a graph
+#
+# requires:
+# pip install matplotlib
+
+import argparse
+import gzip
+import math
+import os
+import statistics
+import sys
+import tarfile
+
+def mmstdm(data):
+ dmin = min(data)
+ dmax = max(data)
+ dmean = statistics.mean(data)
+ dstd = statistics.pstdev(data)
+ return f'[{dmin:.3f}/{dmean:.3f} ({dstd:.3f})/{dmax:.3f}]'
+
+class LatencyAnalyzer:
+ def __init__(self):
+ self.data = []
+ def read(self, path):
+ if path.endswith('.gz'):
+ with gzip.open(path, 'rt') as fin:
+ self.rlines(fin)
+ else:
+ with open(path, 'rt') as fin:
+ self.rlines(fin)
+ def rlines(self, linesource):
+ for line in linesource:
+ rec = int(line.strip())/1000000000.0
+ self.data.append(rec)
+ def plot(self, outname):
+ from matplotlib import pyplot as plt
+ plt.plot(self.data)
+ plt.savefig(outname + '.png', format='png')
+ plt.savefig(outname + '.svg', format='svg')
+ def report_data(self):
+ self.data.sort()
+ some = int(math.log(len(self.data))*4)
+ lowest = self.data[:some]
+ highest = self.data[-some:]
+ return some, lowest, highest
+ def report(self):
+ lines = []
+ some, lowest, highest = self.report_data()
+ lines.append(f'{len(self.data)} points: {mmstdm(self.data)}')
+ lines.append('min {:.3f}s'.format(min(self.data)))
+ lines.append('max {:.3f}s'.format(max(self.data)))
+ lines.append(f'lowest-{some}: {mmstdm(lowest)}')
+ lines.append(f'highest-{some}: {mmstdm(highest)}')
+ return '\n'.join(lines)
+ def report_html(self):
+ lines = []
+ some, lowest, highest = self.report_data()
+ lines.append('Latency
')
+ lines.append('[min/mean (std)/max]
')
+ lines.append(f'{len(self.data)} points: {mmstdm(self.data)}
')
+ lines.append(f'min {min(self.data):.3f}s
')
+ lines.append(f'max {max(self.data):.3f}s
')
+ lines.append(f'lowest-{some}: {mmstdm(lowest)}
')
+ lines.append(f'highest-{some}: {mmstdm(highest)}
')
+ return ''.join(lines)
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('latency_log', nargs='*')
+ ap.add_argument('-p', '--plot', help='plot base name for .png .svg')
+ ap.add_argument('-a', '--aout', help='report text output path (append)')
+ ap.add_argument('-H', '--ahtml', help='report html output path (append)')
+ ap.add_argument('--tardir')
+ args = ap.parse_args()
+
+ la = LatencyAnalyzer()
+ for path in args.latency_log:
+ la.read(path)
+ if args.tardir:
+ for dirpath, dirnames, filenames in os.walk(args.tardir):
+ for fname in filenames:
+ if fname.endswith('.tar.bz2'):
+ tarname = os.path.join(dirpath, fname)
+ tf = tarfile.open(tarname, 'r:bz2')
+ for tinfo in tf:
+ if not tinfo.isfile():
+ continue
+ if 'latency' in tinfo.name:
+ rawf = tf.extractfile(tinfo)
+ if tinfo.name.endswith('.gz'):
+ fin = gzip.open(rawf, 'rt')
+ else:
+ fin = rawf
+ rawf = None
+ try:
+ la.rlines(fin)
+ except Exception as e:
+ sys.stderr.write(f'{tarname}/{tinfo.name}: {e}\n')
+ fin.close()
+ if rawf is not None:
+ rawf.close()
+
+ if args.plot:
+ la.plot(args.plot)
+ if args.ahtml:
+ with open(args.ahtml, 'at') as fout:
+ fout.write(la.report_html())
+ if args.aout:
+ with open(args.aout, 'at') as fout:
+ fout.write(la.report())
+ if (not args.aout) and (not args.ahtml):
+ print(la.report())
+
+if __name__ == '__main__':
+ main()
diff --git a/test/testdata/configs/config-v26.json b/test/testdata/configs/config-v26.json
index 31d693f90a..6589d03266 100644
--- a/test/testdata/configs/config-v26.json
+++ b/test/testdata/configs/config-v26.json
@@ -38,6 +38,7 @@
"EnableBlockServiceFallbackToArchiver": true,
"EnableCatchupFromArchiveServers": false,
"EnableDeveloperAPI": false,
+ "EnableExperimentalAPI": false,
"EnableGossipBlockService": true,
"EnableIncomingMessageFilter": false,
"EnableLedgerService": false,
@@ -93,9 +94,9 @@
"RunHosted": false,
"SuggestedFeeBlockHistory": 3,
"SuggestedFeeSlidingWindowSize": 50,
+ "TelemetryToLog": true,
"TLSCertFile": "",
"TLSKeyFile": "",
- "TelemetryToLog": true,
"TransactionSyncDataExchangeRate": 0,
"TransactionSyncSignificantMessageThreshold": 0,
"TxIncomingFilteringFlags": 1,
diff --git a/test/testdata/configs/config-v27.json b/test/testdata/configs/config-v27.json
new file mode 100644
index 0000000000..137578e0fa
--- /dev/null
+++ b/test/testdata/configs/config-v27.json
@@ -0,0 +1,116 @@
+{
+ "Version": 27,
+ "AccountUpdatesStatsInterval": 5000000000,
+ "AccountsRebuildSynchronousMode": 1,
+ "AgreementIncomingBundlesQueueLength": 15,
+ "AgreementIncomingProposalsQueueLength": 50,
+ "AgreementIncomingVotesQueueLength": 20000,
+ "AnnounceParticipationKey": true,
+ "Archival": false,
+ "BaseLoggerDebugLevel": 4,
+ "BlockServiceCustomFallbackEndpoints": "",
+ "BroadcastConnectionsLimit": -1,
+ "CadaverDirectory": "",
+ "CadaverSizeTarget": 0,
+ "CatchpointFileHistoryLength": 365,
+ "CatchpointInterval": 10000,
+ "CatchpointTracking": 0,
+ "CatchupBlockDownloadRetryAttempts": 1000,
+ "CatchupBlockValidateMode": 0,
+ "CatchupFailurePeerRefreshRate": 10,
+ "CatchupGossipBlockFetchTimeoutSec": 4,
+ "CatchupHTTPBlockFetchTimeoutSec": 4,
+ "CatchupLedgerDownloadRetryAttempts": 50,
+ "CatchupParallelBlocks": 16,
+ "ConnectionsRateLimitingCount": 60,
+ "ConnectionsRateLimitingWindowSeconds": 1,
+ "DNSBootstrapID": ".algorand.network",
+ "DNSSecurityFlags": 1,
+ "DeadlockDetection": 0,
+ "DeadlockDetectionThreshold": 30,
+ "DisableLocalhostConnectionRateLimit": true,
+ "DisableNetworking": false,
+ "DisableOutgoingConnectionThrottling": false,
+ "EnableAccountUpdatesStats": false,
+ "EnableAgreementReporting": false,
+ "EnableAgreementTimeMetrics": false,
+ "EnableAssembleStats": false,
+ "EnableBlockService": false,
+ "EnableBlockServiceFallbackToArchiver": true,
+ "EnableCatchupFromArchiveServers": false,
+ "EnableDeveloperAPI": false,
+ "EnableExperimentalAPI": false,
+ "EnableGossipBlockService": true,
+ "EnableIncomingMessageFilter": false,
+ "EnableLedgerService": false,
+ "EnableMetricReporting": false,
+ "EnableOutgoingNetworkMessageFiltering": true,
+ "EnablePingHandler": true,
+ "EnableProcessBlockStats": false,
+ "EnableProfiler": false,
+ "EnableRequestLogger": false,
+ "EnableRuntimeMetrics": false,
+ "EnableTopAccountsReporting": false,
+ "EnableTxBacklogRateLimiting": false,
+ "EnableUsageLog": false,
+ "EnableVerbosedTransactionSyncLogging": false,
+ "EndpointAddress": "127.0.0.1:0",
+ "FallbackDNSResolverAddress": "",
+ "ForceFetchTransactions": false,
+ "ForceRelayMessages": false,
+ "GossipFanout": 4,
+ "HeartbeatUpdateInterval": 600,
+ "IncomingConnectionsLimit": 2400,
+ "IncomingMessageFilterBucketCount": 5,
+ "IncomingMessageFilterBucketSize": 512,
+ "IsIndexerActive": false,
+ "LedgerSynchronousMode": 2,
+ "LogArchiveMaxAge": "",
+ "LogArchiveName": "node.archive.log",
+ "LogSizeLimit": 1073741824,
+ "MaxAPIBoxPerApplication": 100000,
+ "MaxAPIResourcesPerAccount": 100000,
+ "MaxAcctLookback": 4,
+ "MaxCatchpointDownloadDuration": 7200000000000,
+ "MaxConnectionsPerIP": 15,
+ "MinCatchpointFileDownloadBytesPerSecond": 20480,
+ "NetAddress": "",
+ "NetworkMessageTraceServer": "",
+ "NetworkProtocolVersion": "",
+ "NodeExporterListenAddress": ":9100",
+ "NodeExporterPath": "./node_exporter",
+ "OptimizeAccountsDatabaseOnStartup": false,
+ "OutgoingMessageFilterBucketCount": 3,
+ "OutgoingMessageFilterBucketSize": 128,
+ "ParticipationKeysRefreshInterval": 60000000000,
+ "PeerConnectionsUpdateInterval": 3600,
+ "PeerPingPeriodSeconds": 0,
+ "PriorityPeers": {},
+ "ProposalAssemblyTime": 500000000,
+ "PublicAddress": "",
+ "ReconnectTime": 60000000000,
+ "ReservedFDs": 256,
+ "RestConnectionsHardLimit": 2048,
+ "RestConnectionsSoftLimit": 1024,
+ "RestReadTimeoutSeconds": 15,
+ "RestWriteTimeoutSeconds": 120,
+ "RunHosted": false,
+ "SuggestedFeeBlockHistory": 3,
+ "SuggestedFeeSlidingWindowSize": 50,
+ "TLSCertFile": "",
+ "TLSKeyFile": "",
+ "TelemetryToLog": true,
+ "TransactionSyncDataExchangeRate": 0,
+ "TransactionSyncSignificantMessageThreshold": 0,
+ "TxBacklogReservedCapacityPerPeer": 20,
+ "TxBacklogServiceRateWindowSeconds": 10,
+ "TxBacklogSize": 26000,
+ "TxIncomingFilteringFlags": 1,
+ "TxPoolExponentialIncreaseFactor": 2,
+ "TxPoolSize": 75000,
+ "TxSyncIntervalSeconds": 60,
+ "TxSyncServeResponseSize": 1000000,
+ "TxSyncTimeoutSeconds": 30,
+ "UseXForwardedForAddressField": "",
+ "VerifiedTranscationsCacheSize": 150000
+}
diff --git a/tools/boxkey/convertBoxKey.go b/tools/boxkey/convertBoxKey.go
new file mode 100644
index 0000000000..0d77d2f84b
--- /dev/null
+++ b/tools/boxkey/convertBoxKey.go
@@ -0,0 +1,48 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package main
+
+import (
+ "encoding/base64"
+ "encoding/hex"
+ "flag"
+ "fmt"
+
+ "github.com/algorand/avm-abi/apps"
+)
+
+func main() {
+ var name string
+ var appIdx uint64
+ flag.Uint64Var(&appIdx, "a", 0, "base64/algorand address to convert to the other")
+ flag.StringVar(&name, "n", "", "base64 box name")
+ flag.Parse()
+
+ if appIdx == 0 && name == "" {
+ fmt.Println("provide input with '-a' and '-k' flags.")
+ return
+ }
+
+ nameBytes, err := base64.StdEncoding.DecodeString(name)
+ if err != nil {
+ fmt.Println("invalid key value")
+ return
+ }
+ key := apps.MakeBoxKey(appIdx, string(nameBytes))
+ fmt.Println(base64.StdEncoding.EncodeToString([]byte(key)))
+ fmt.Println(hex.EncodeToString([]byte(key)))
+}
diff --git a/tools/debug/algodump/main.go b/tools/debug/algodump/main.go
index f61944ff1a..be80768fc5 100644
--- a/tools/debug/algodump/main.go
+++ b/tools/debug/algodump/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/debug/carpenter/main.go b/tools/debug/carpenter/main.go
index d43f2267b0..625211694a 100644
--- a/tools/debug/carpenter/main.go
+++ b/tools/debug/carpenter/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/debug/coroner/main.go b/tools/debug/coroner/main.go
index 669e560de7..1ea04537d4 100644
--- a/tools/debug/coroner/main.go
+++ b/tools/debug/coroner/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/debug/determaccount/main.go b/tools/debug/determaccount/main.go
index 84dfbf73fd..2c3c16d65e 100644
--- a/tools/debug/determaccount/main.go
+++ b/tools/debug/determaccount/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/debug/dumpblocks/main.go b/tools/debug/dumpblocks/main.go
index 09698bdb05..461e7949cf 100644
--- a/tools/debug/dumpblocks/main.go
+++ b/tools/debug/dumpblocks/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/debug/genconsensusconfig/main.go b/tools/debug/genconsensusconfig/main.go
index 74f6eda2f6..edef949d3d 100644
--- a/tools/debug/genconsensusconfig/main.go
+++ b/tools/debug/genconsensusconfig/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/debug/logfilter/main.go b/tools/debug/logfilter/main.go
index 1c7545a48f..11878ea470 100644
--- a/tools/debug/logfilter/main.go
+++ b/tools/debug/logfilter/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/debug/logfilter/main_test.go b/tools/debug/logfilter/main_test.go
index 45ab8605fc..6b6902d62d 100644
--- a/tools/debug/logfilter/main_test.go
+++ b/tools/debug/logfilter/main_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/misc/convertAddress.go b/tools/misc/convertAddress.go
index c8c4e99970..801c26106b 100644
--- a/tools/misc/convertAddress.go
+++ b/tools/misc/convertAddress.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/bootstrap.go b/tools/network/bootstrap.go
index cd15ef2ab8..b3bd79b23c 100644
--- a/tools/network/bootstrap.go
+++ b/tools/network/bootstrap.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/cloudflare/cloudflare.go b/tools/network/cloudflare/cloudflare.go
index 714fb9635b..84f62f6533 100644
--- a/tools/network/cloudflare/cloudflare.go
+++ b/tools/network/cloudflare/cloudflare.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/cloudflare/createRecord.go b/tools/network/cloudflare/createRecord.go
index c68747f5bd..a41dcbccc3 100644
--- a/tools/network/cloudflare/createRecord.go
+++ b/tools/network/cloudflare/createRecord.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/cloudflare/deleteRecord.go b/tools/network/cloudflare/deleteRecord.go
index f0bf90ce5c..864a223dfc 100644
--- a/tools/network/cloudflare/deleteRecord.go
+++ b/tools/network/cloudflare/deleteRecord.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/cloudflare/helpers.go b/tools/network/cloudflare/helpers.go
index e4b995f67e..e330d93309 100644
--- a/tools/network/cloudflare/helpers.go
+++ b/tools/network/cloudflare/helpers.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/cloudflare/listRecords.go b/tools/network/cloudflare/listRecords.go
index 1617b61188..34efc71d53 100644
--- a/tools/network/cloudflare/listRecords.go
+++ b/tools/network/cloudflare/listRecords.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/cloudflare/updateRecord.go b/tools/network/cloudflare/updateRecord.go
index c4be7c362e..7add2d05be 100644
--- a/tools/network/cloudflare/updateRecord.go
+++ b/tools/network/cloudflare/updateRecord.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/cloudflare/zones.go b/tools/network/cloudflare/zones.go
index f5aa4b9ac2..c9a54a6dfd 100644
--- a/tools/network/cloudflare/zones.go
+++ b/tools/network/cloudflare/zones.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/anchor.go b/tools/network/dnssec/anchor.go
index 8f7066904a..18d259aca3 100644
--- a/tools/network/dnssec/anchor.go
+++ b/tools/network/dnssec/anchor.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/anchor_test.go b/tools/network/dnssec/anchor_test.go
index cea0d2adbc..dbd7396e0f 100644
--- a/tools/network/dnssec/anchor_test.go
+++ b/tools/network/dnssec/anchor_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/client.go b/tools/network/dnssec/client.go
index 5ebfc84782..79cd79a9cf 100644
--- a/tools/network/dnssec/client.go
+++ b/tools/network/dnssec/client.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/client_test.go b/tools/network/dnssec/client_test.go
index 3bcc8f79d5..7c2eede7db 100644
--- a/tools/network/dnssec/client_test.go
+++ b/tools/network/dnssec/client_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/config.go b/tools/network/dnssec/config.go
index a9c38bff9a..a5c9bbbfe4 100644
--- a/tools/network/dnssec/config.go
+++ b/tools/network/dnssec/config.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/config_test.go b/tools/network/dnssec/config_test.go
index 6115c7eef8..8ad1ae511b 100644
--- a/tools/network/dnssec/config_test.go
+++ b/tools/network/dnssec/config_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/config_unix.go b/tools/network/dnssec/config_unix.go
index 4a8c574de1..74b3cb0f9d 100644
--- a/tools/network/dnssec/config_unix.go
+++ b/tools/network/dnssec/config_unix.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/config_unix_test.go b/tools/network/dnssec/config_unix_test.go
index c757b5369f..07a6a584bf 100644
--- a/tools/network/dnssec/config_unix_test.go
+++ b/tools/network/dnssec/config_unix_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/config_windows.go b/tools/network/dnssec/config_windows.go
index 41d6950070..8d291be673 100644
--- a/tools/network/dnssec/config_windows.go
+++ b/tools/network/dnssec/config_windows.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/dialer.go b/tools/network/dnssec/dialer.go
index 8bb9f869b4..08144ae5bc 100644
--- a/tools/network/dnssec/dialer.go
+++ b/tools/network/dnssec/dialer.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/dnssec_test.go b/tools/network/dnssec/dnssec_test.go
index 4964990487..86c7239182 100644
--- a/tools/network/dnssec/dnssec_test.go
+++ b/tools/network/dnssec/dnssec_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/relay-check/main.go b/tools/network/dnssec/relay-check/main.go
index 4b2dcad295..f82211e096 100644
--- a/tools/network/dnssec/relay-check/main.go
+++ b/tools/network/dnssec/relay-check/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/resolver.go b/tools/network/dnssec/resolver.go
index 5aabcfd285..2b0b873c31 100644
--- a/tools/network/dnssec/resolver.go
+++ b/tools/network/dnssec/resolver.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/sort.go b/tools/network/dnssec/sort.go
index 0bdf17f88f..2c8de2653a 100644
--- a/tools/network/dnssec/sort.go
+++ b/tools/network/dnssec/sort.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/sort_test.go b/tools/network/dnssec/sort_test.go
index 3440e23030..ab1be21fbc 100644
--- a/tools/network/dnssec/sort_test.go
+++ b/tools/network/dnssec/sort_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/testHarness.go b/tools/network/dnssec/testHarness.go
index 5f5310a8e3..0f4c7f4161 100644
--- a/tools/network/dnssec/testHarness.go
+++ b/tools/network/dnssec/testHarness.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/trustchain.go b/tools/network/dnssec/trustchain.go
index 3cde9bea70..15f69aea1d 100644
--- a/tools/network/dnssec/trustchain.go
+++ b/tools/network/dnssec/trustchain.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/trustedchain_test.go b/tools/network/dnssec/trustedchain_test.go
index 2f671c007f..6310e22ab4 100644
--- a/tools/network/dnssec/trustedchain_test.go
+++ b/tools/network/dnssec/trustedchain_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/trustedzone.go b/tools/network/dnssec/trustedzone.go
index b3f20dc447..a71385503e 100644
--- a/tools/network/dnssec/trustedzone.go
+++ b/tools/network/dnssec/trustedzone.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/trustedzone_test.go b/tools/network/dnssec/trustedzone_test.go
index 6d3bbaa043..877a687415 100644
--- a/tools/network/dnssec/trustedzone_test.go
+++ b/tools/network/dnssec/trustedzone_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/util.go b/tools/network/dnssec/util.go
index 1622f490f1..48f5c9448c 100644
--- a/tools/network/dnssec/util.go
+++ b/tools/network/dnssec/util.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/dnssec/util_test.go b/tools/network/dnssec/util_test.go
index d467d52c80..2a77ad1c09 100644
--- a/tools/network/dnssec/util_test.go
+++ b/tools/network/dnssec/util_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/externalIP.go b/tools/network/externalIP.go
index a726d03fc0..121c43df32 100644
--- a/tools/network/externalIP.go
+++ b/tools/network/externalIP.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/resolveController.go b/tools/network/resolveController.go
index fd12d0dbf6..a36914f21b 100644
--- a/tools/network/resolveController.go
+++ b/tools/network/resolveController.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/resolveController_test.go b/tools/network/resolveController_test.go
index 8a843c9021..db8a80573a 100644
--- a/tools/network/resolveController_test.go
+++ b/tools/network/resolveController_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/resolver.go b/tools/network/resolver.go
index 467bb1e12f..b823b17b20 100644
--- a/tools/network/resolver.go
+++ b/tools/network/resolver.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/resolver_test.go b/tools/network/resolver_test.go
index a00f07ea34..6efe1bea45 100644
--- a/tools/network/resolver_test.go
+++ b/tools/network/resolver_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -22,14 +22,16 @@ import (
"testing"
"time"
- "github.com/algorand/go-algorand/test/partitiontest"
"github.com/stretchr/testify/require"
+
+ "github.com/algorand/go-algorand/test/partitiontest"
)
-func TestResolver(t *testing.T) {
+func TestResolverWithDefaultDNSResolution(t *testing.T) {
partitiontest.PartitionTest(t)
+ t.Parallel()
- // start with a resolver that has no specific DNS address defined.
+ // configure a resolver that has no specific DNS address defined.
// we want to make sure that it will go to the default DNS server ( 8.8.8.8 )
resolver := Resolver{}
cname, addrs, err := resolver.LookupSRV(context.Background(), "telemetry", "tls", "devnet.algodev.network")
@@ -37,18 +39,34 @@ func TestResolver(t *testing.T) {
require.Equal(t, "_telemetry._tls.devnet.algodev.network.", cname)
require.True(t, len(addrs) == 1)
require.Equal(t, defaultDNSAddress, resolver.EffectiveResolverDNS())
+}
- // specify a specific resolver to work with ( cloudflare DNS server is 1.1.1.1 )
- cloudFlareIPAddr, _ := net.ResolveIPAddr("ip", "1.1.1.1")
+func TestResolverWithCloudflareDNSResolution(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ t.Parallel()
+
+ resolver := Resolver{}
+
+ // The test previously specified Cloudflare's primary DNS server (1.1.1.1).
+ // However, CircleCI began blocking requests to 1.1.1.1. In order to
+ // preserve the test's spirit, it now uses Cloudflare's secondary DNS
+ // server (1.0.0.1).
+ cloudflareIPAddr, _ := net.ResolveIPAddr("ip", "1.0.0.1")
resolver = Resolver{
- dnsAddress: *cloudFlareIPAddr,
+ dnsAddress: *cloudflareIPAddr,
}
- cname, addrs, err = resolver.LookupSRV(context.Background(), "telemetry", "tls", "devnet.algodev.network")
+ cname, addrs, err := resolver.LookupSRV(context.Background(), "telemetry", "tls", "devnet.algodev.network")
require.NoError(t, err)
require.Equal(t, "_telemetry._tls.devnet.algodev.network.", cname)
require.True(t, len(addrs) == 1)
- require.Equal(t, "1.1.1.1", resolver.EffectiveResolverDNS())
+ require.Equal(t, "1.0.0.1", resolver.EffectiveResolverDNS())
+}
+func TestResolverWithInvalidDNSResolution(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ t.Parallel()
+
+ resolver := Resolver{}
// specify an invalid dns resolver ip address and examine the fail case.
dummyIPAddr, _ := net.ResolveIPAddr("ip", "255.255.128.1")
resolver = Resolver{
@@ -56,7 +74,7 @@ func TestResolver(t *testing.T) {
}
timingOutContext, timingOutContextFunc := context.WithTimeout(context.Background(), time.Duration(100)*time.Millisecond)
defer timingOutContextFunc()
- cname, addrs, err = resolver.LookupSRV(timingOutContext, "telemetry", "tls", "devnet.algodev.network")
+ cname, addrs, err := resolver.LookupSRV(timingOutContext, "telemetry", "tls", "devnet.algodev.network")
require.Error(t, err)
require.Equal(t, "", cname)
require.True(t, len(addrs) == 0)
diff --git a/tools/network/telemetryURIUpdateService.go b/tools/network/telemetryURIUpdateService.go
index ab0cc8110b..becae7c743 100644
--- a/tools/network/telemetryURIUpdateService.go
+++ b/tools/network/telemetryURIUpdateService.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/network/telemetryURIUpdateService_test.go b/tools/network/telemetryURIUpdateService_test.go
index afe8bf242d..1014d28df3 100644
--- a/tools/network/telemetryURIUpdateService_test.go
+++ b/tools/network/telemetryURIUpdateService_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/teal/algotmpl/extract.go b/tools/teal/algotmpl/extract.go
index b8c045366c..f542d6bce8 100644
--- a/tools/teal/algotmpl/extract.go
+++ b/tools/teal/algotmpl/extract.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/teal/algotmpl/main.go b/tools/teal/algotmpl/main.go
index 7568dda92f..824ab7ee9b 100644
--- a/tools/teal/algotmpl/main.go
+++ b/tools/teal/algotmpl/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/teal/dkey/dsign/main.go b/tools/teal/dkey/dsign/main.go
index 6f380d8a0b..5a4e2adfcd 100644
--- a/tools/teal/dkey/dsign/main.go
+++ b/tools/teal/dkey/dsign/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/tools/teal/tealcut/main.go b/tools/teal/tealcut/main.go
index 7615cdc979..0740929db5 100644
--- a/tools/teal/tealcut/main.go
+++ b/tools/teal/tealcut/main.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/cmdUtils.go b/util/cmdUtils.go
index 29641e6284..1382be178e 100644
--- a/util/cmdUtils.go
+++ b/util/cmdUtils.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/codecs/json.go b/util/codecs/json.go
index 2d2c21134e..e283ef0624 100644
--- a/util/codecs/json.go
+++ b/util/codecs/json.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/codecs/json_test.go b/util/codecs/json_test.go
index de35acf3cf..1f56531971 100644
--- a/util/codecs/json_test.go
+++ b/util/codecs/json_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/condvar/timedwait.go b/util/condvar/timedwait.go
index a4605d0998..e14f2b33b7 100644
--- a/util/condvar/timedwait.go
+++ b/util/condvar/timedwait.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/condvar/timedwait_test.go b/util/condvar/timedwait_test.go
index 20b8f70a4d..fa7deef219 100644
--- a/util/condvar/timedwait_test.go
+++ b/util/condvar/timedwait_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/dbpair.go b/util/db/dbpair.go
index 95be092633..0fa382063e 100644
--- a/util/db/dbpair.go
+++ b/util/db/dbpair.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/dbutil.go b/util/db/dbutil.go
index 34a32320a1..7e62dfee61 100644
--- a/util/db/dbutil.go
+++ b/util/db/dbutil.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/dbutil_test.go b/util/db/dbutil_test.go
index 412f04a6c2..c35beccb23 100644
--- a/util/db/dbutil_test.go
+++ b/util/db/dbutil_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/fullfsync_darwin.go b/util/db/fullfsync_darwin.go
index 8a529305ae..3261e7864a 100644
--- a/util/db/fullfsync_darwin.go
+++ b/util/db/fullfsync_darwin.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/initialize.go b/util/db/initialize.go
index 36bf83770d..fc11bc1aa4 100644
--- a/util/db/initialize.go
+++ b/util/db/initialize.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/initialize_test.go b/util/db/initialize_test.go
index d7b879d6ae..c0c8774b11 100644
--- a/util/db/initialize_test.go
+++ b/util/db/initialize_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/interfaces.go b/util/db/interfaces.go
index 5d4abed852..938edc1031 100644
--- a/util/db/interfaces.go
+++ b/util/db/interfaces.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/perf_test.go b/util/db/perf_test.go
index 1e0a8bccd6..5740ce2d47 100644
--- a/util/db/perf_test.go
+++ b/util/db/perf_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/versioning.go b/util/db/versioning.go
index 28f6b1dbd1..6104fe6b9c 100644
--- a/util/db/versioning.go
+++ b/util/db/versioning.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/db/versioning_test.go b/util/db/versioning_test.go
index fd2fecc23a..74f048e30c 100644
--- a/util/db/versioning_test.go
+++ b/util/db/versioning_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/execpool/backlog.go b/util/execpool/backlog.go
index 966aafe97f..9e95ebe653 100644
--- a/util/execpool/backlog.go
+++ b/util/execpool/backlog.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -43,6 +43,7 @@ type backlogItemTask struct {
type BacklogPool interface {
ExecutionPool
EnqueueBacklog(enqueueCtx context.Context, t ExecFunc, arg interface{}, out chan interface{}) error
+ BufferSize() (length, capacity int)
}
// MakeBacklog creates a backlog
@@ -94,6 +95,11 @@ func (b *backlog) Enqueue(enqueueCtx context.Context, t ExecFunc, arg interface{
}
}
+// BufferSize returns the length and the capacity of the buffer
+func (b *backlog) BufferSize() (length, capacity int) {
+ return len(b.buffer), cap(b.buffer)
+}
+
// Enqueue enqueues a single task into the backlog
func (b *backlog) EnqueueBacklog(enqueueCtx context.Context, t ExecFunc, arg interface{}, out chan interface{}) error {
select {
diff --git a/util/execpool/pool.go b/util/execpool/pool.go
index 1b37dd95fc..50acbf9a5a 100644
--- a/util/execpool/pool.go
+++ b/util/execpool/pool.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/io.go b/util/io.go
index 43068f39c9..1ab09b3b68 100644
--- a/util/io.go
+++ b/util/io.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/io_test.go b/util/io_test.go
index 353616c64e..5919051135 100644
--- a/util/io_test.go
+++ b/util/io_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/counter.go b/util/metrics/counter.go
index 5c195b5f8a..bb8355cacc 100644
--- a/util/metrics/counter.go
+++ b/util/metrics/counter.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -66,13 +66,12 @@ func (counter *Counter) Inc(labels map[string]string) {
if len(labels) == 0 {
counter.fastAddUint64(1)
} else {
- counter.Add(1.0, labels)
+ counter.addLabels(1.0, labels)
}
}
-// Add increases counter by x
-// For adding an integer, see AddUint64(x)
-func (counter *Counter) Add(x float64, labels map[string]string) {
+// addLabels increases counter by x
+func (counter *Counter) addLabels(x uint64, labels map[string]string) {
counter.Lock()
defer counter.Unlock()
@@ -95,13 +94,12 @@ func (counter *Counter) Add(x float64, labels map[string]string) {
}
// AddUint64 increases counter by x
-// If labels is nil this is much faster than Add()
-// Calls through to Add() if labels is not nil.
+// If labels is nil this is much faster than if labels is not nil.
func (counter *Counter) AddUint64(x uint64, labels map[string]string) {
if len(labels) == 0 {
counter.fastAddUint64(x)
} else {
- counter.Add(float64(x), labels)
+ counter.addLabels(x, labels)
}
}
@@ -122,7 +120,7 @@ func (counter *Counter) fastAddUint64(x uint64) {
// is the first Add. Create a dummy
// counterValue for the no-labels value.
// Dummy counterValue simplifies display in WriteMetric.
- counter.Add(0, nil)
+ counter.addLabels(0, nil)
}
}
@@ -191,9 +189,9 @@ func (counter *Counter) WriteMetric(buf *strings.Builder, parentLabels string) {
buf.WriteString("} ")
value := l.counter
if len(l.labels) == 0 {
- value += float64(atomic.LoadUint64(&counter.intValue))
+ value += atomic.LoadUint64(&counter.intValue)
}
- buf.WriteString(strconv.FormatFloat(value, 'f', -1, 32))
+ buf.WriteString(strconv.FormatUint(value, 10))
buf.WriteString("\n")
}
}
@@ -210,12 +208,12 @@ func (counter *Counter) AddMetric(values map[string]float64) {
for _, l := range counter.values {
sum := l.counter
if len(l.labels) == 0 {
- sum += float64(atomic.LoadUint64(&counter.intValue))
+ sum += atomic.LoadUint64(&counter.intValue)
}
var suffix string
if len(l.formattedLabels) > 0 {
suffix = ":" + l.formattedLabels
}
- values[sanitizeTelemetryName(counter.name+suffix)] = sum
+ values[sanitizeTelemetryName(counter.name+suffix)] = float64(sum)
}
}
diff --git a/util/metrics/counterCommon.go b/util/metrics/counterCommon.go
index 26ef7f9245..2a810ace69 100644
--- a/util/metrics/counterCommon.go
+++ b/util/metrics/counterCommon.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -35,7 +35,7 @@ type Counter struct {
}
type counterValues struct {
- counter float64
+ counter uint64
labels map[string]string
formattedLabels string
}
diff --git a/util/metrics/counter_test.go b/util/metrics/counter_test.go
index 72e1d3b1ad..fe7d553e4c 100644
--- a/util/metrics/counter_test.go
+++ b/util/metrics/counter_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -144,8 +144,8 @@ func TestMetricCounterMixed(t *testing.T) {
counter := MakeCounter(MetricName{Name: "metric_test_name1", Description: "this is the metric test for counter object"})
- counter.Add(5.25, nil)
- counter.Add(8.25, map[string]string{})
+ counter.AddUint64(5, nil)
+ counter.AddUint64(8, map[string]string{})
for i := 0; i < 20; i++ {
counter.Inc(nil)
// wait half-a cycle
@@ -169,7 +169,7 @@ func TestMetricCounterMixed(t *testing.T) {
for k, v := range test.metrics {
// we have increased each one of the labels exactly 4 times. See that the counter was counting correctly.
// ( counters starts at zero )
- require.Equal(t, "35.5", v, fmt.Sprintf("The metric '%s' reached value '%s'", k, v))
+ require.Equal(t, "35", v, fmt.Sprintf("The metric '%s' reached value '%s'", k, v))
}
}
@@ -188,13 +188,13 @@ testname{host="myhost"} 0
`
require.Equal(t, expected, sbOut.String())
- c.Add(2.3, nil)
+ c.AddUint64(2, nil)
// ensure non-zero counters are logged
sbOut = strings.Builder{}
c.WriteMetric(&sbOut, `host="myhost"`)
expected = `# HELP testname testhelp
# TYPE testname counter
-testname{host="myhost"} 2.3
+testname{host="myhost"} 2
`
require.Equal(t, expected, sbOut.String())
}
diff --git a/util/metrics/gauge.go b/util/metrics/gauge.go
index f37896775a..ce203d47c0 100644
--- a/util/metrics/gauge.go
+++ b/util/metrics/gauge.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -19,16 +19,14 @@ package metrics
import (
"strconv"
"strings"
-
- "github.com/algorand/go-deadlock"
+ "sync/atomic"
)
// Gauge represent a single gauge variable.
type Gauge struct {
- deadlock.Mutex
+ value uint64
name string
description string
- value float64
}
// MakeGauge create a new gauge with the provided name and description.
@@ -60,24 +58,17 @@ func (gauge *Gauge) Deregister(reg *Registry) {
}
// Add increases gauge by x
-func (gauge *Gauge) Add(x float64) {
- gauge.Lock()
- defer gauge.Unlock()
- gauge.value += x
+func (gauge *Gauge) Add(x uint64) {
+ atomic.AddUint64(&gauge.value, x)
}
// Set sets gauge to x
-func (gauge *Gauge) Set(x float64) {
- gauge.Lock()
- defer gauge.Unlock()
- gauge.value = x
+func (gauge *Gauge) Set(x uint64) {
+ atomic.StoreUint64(&gauge.value, x)
}
// WriteMetric writes the metric into the output stream
func (gauge *Gauge) WriteMetric(buf *strings.Builder, parentLabels string) {
- gauge.Lock()
- defer gauge.Unlock()
-
buf.WriteString("# HELP ")
buf.WriteString(gauge.name)
buf.WriteString(" ")
@@ -91,14 +82,14 @@ func (gauge *Gauge) WriteMetric(buf *strings.Builder, parentLabels string) {
buf.WriteString(parentLabels)
}
buf.WriteString("} ")
- buf.WriteString(strconv.FormatFloat(gauge.value, 'f', -1, 32))
+ value := atomic.LoadUint64(&gauge.value)
+ buf.WriteString(strconv.FormatUint(value, 10))
buf.WriteString("\n")
}
// AddMetric adds the metric into the map
func (gauge *Gauge) AddMetric(values map[string]float64) {
- gauge.Lock()
- defer gauge.Unlock()
+ value := atomic.LoadUint64(&gauge.value)
- values[sanitizeTelemetryName(gauge.name)] = gauge.value
+ values[sanitizeTelemetryName(gauge.name)] = float64(value)
}
diff --git a/util/metrics/gauge_test.go b/util/metrics/gauge_test.go
index 41a9edb927..afa0bbd593 100644
--- a/util/metrics/gauge_test.go
+++ b/util/metrics/gauge_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -52,8 +52,8 @@ func TestMetricGauge(t *testing.T) {
gauges[i] = MakeGauge(MetricName{Name: fmt.Sprintf("gauge_%d", i), Description: "this is the metric test for gauge object"})
}
for i := 0; i < 9; i++ {
- gauges[i%3].Set(float64(i * 100))
- gauges[i%3].Add(float64(i))
+ gauges[i%3].Set(uint64(i * 100))
+ gauges[i%3].Add(uint64(i))
// wait half-a cycle
time.Sleep(test.sampleRate / 2)
}
diff --git a/util/metrics/metrics.go b/util/metrics/metrics.go
index 06a0431ecb..11973e85b7 100644
--- a/util/metrics/metrics.go
+++ b/util/metrics/metrics.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -95,6 +95,8 @@ var (
TransactionMessagesAlreadyCommitted = MetricName{Name: "algod_transaction_messages_err_or_committed", Description: "Number of duplicate or error transaction messages after TX handler backlog"}
// TransactionMessagesTxGroupInvalidFee "Number of transaction messages with invalid txgroup fee"
TransactionMessagesTxGroupInvalidFee = MetricName{Name: "algod_transaction_messages_txgroup_invalid_fee", Description: "Number of transaction messages with invalid txgroup fee"}
+ // TransactionMessagesTxnDroppedCongestionManagement "Number of transaction messages dropped because the tx backlog is under congestion management"
+ TransactionMessagesTxnDroppedCongestionManagement = MetricName{Name: "algod_transaction_messages_txn_dropped_congestion_ctrl", Description: "Number of transaction messages dropped because the tx backlog is under congestion management"}
// TransactionMessagesTxnNotWellFormed "Number of transaction messages not well formed"
TransactionMessagesTxnNotWellFormed = MetricName{Name: "algod_transaction_messages_txn_notwell_formed", Description: "Number of transaction messages not well formed"}
// TransactionMessagesTxnSigNotWellFormed "Number of transaction messages with bad formed signature"
diff --git a/util/metrics/metrics_test.go b/util/metrics/metrics_test.go
index 2e2828b1fb..84cd4292fb 100644
--- a/util/metrics/metrics_test.go
+++ b/util/metrics/metrics_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/registry.go b/util/metrics/registry.go
index 53ada420a3..2f727aaabd 100644
--- a/util/metrics/registry.go
+++ b/util/metrics/registry.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/registryCommon.go b/util/metrics/registryCommon.go
index 8a0f53464f..848ddc8369 100644
--- a/util/metrics/registryCommon.go
+++ b/util/metrics/registryCommon.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/registry_test.go b/util/metrics/registry_test.go
index 2256993f20..17328f1cba 100644
--- a/util/metrics/registry_test.go
+++ b/util/metrics/registry_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -29,17 +29,17 @@ func TestWriteAdd(t *testing.T) {
// Test AddMetrics and WriteMetrics with a counter
counter := MakeCounter(MetricName{Name: "gauge-name", Description: "gauge description"})
- counter.Add(12.34, nil)
+ counter.AddUint64(12, nil)
labelCounter := MakeCounter(MetricName{Name: "label-counter", Description: "counter with labels"})
- labelCounter.Add(5, map[string]string{"label": "a label value"})
+ labelCounter.AddUint64(5, map[string]string{"label": "a label value"})
results := make(map[string]float64)
DefaultRegistry().AddMetrics(results)
require.Equal(t, 2, len(results), "results", results)
require.Contains(t, results, "gauge-name")
- require.InDelta(t, 12.34, results["gauge-name"], 0.01)
+ require.InDelta(t, 12, results["gauge-name"], 0.01)
require.Contains(t, results, "label-counter_label__a_label_value_")
require.InDelta(t, 5, results["label-counter_label__a_label_value_"], 0.01)
@@ -50,7 +50,7 @@ func TestWriteAdd(t *testing.T) {
DefaultRegistry().AddMetrics(results)
require.Contains(t, results, "gauge-name")
- require.InDelta(t, 12.34, results["gauge-name"], 0.01)
+ require.InDelta(t, 12, results["gauge-name"], 0.01)
// not included in string builder
bufAfter := strings.Builder{}
diff --git a/util/metrics/reporter.go b/util/metrics/reporter.go
index efecf6f659..19aef0c369 100644
--- a/util/metrics/reporter.go
+++ b/util/metrics/reporter.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/reporter_test.go b/util/metrics/reporter_test.go
index 7339c708db..4c522a9b22 100755
--- a/util/metrics/reporter_test.go
+++ b/util/metrics/reporter_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/runtime.go b/util/metrics/runtime.go
index 3f89ea761e..29d33e7814 100644
--- a/util/metrics/runtime.go
+++ b/util/metrics/runtime.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/runtime_test.go b/util/metrics/runtime_test.go
index 1032484466..32908670d6 100644
--- a/util/metrics/runtime_test.go
+++ b/util/metrics/runtime_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/service.go b/util/metrics/service.go
index e37f5c7341..5b1e395271 100644
--- a/util/metrics/service.go
+++ b/util/metrics/service.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/serviceCommon.go b/util/metrics/serviceCommon.go
index e0b26c8207..947e9c4404 100644
--- a/util/metrics/serviceCommon.go
+++ b/util/metrics/serviceCommon.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/tagcounter.go b/util/metrics/tagcounter.go
index 80689e7a8b..1daf30ba13 100644
--- a/util/metrics/tagcounter.go
+++ b/util/metrics/tagcounter.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/metrics/tagcounter_test.go b/util/metrics/tagcounter_test.go
index 9e8a507176..8c5991e195 100644
--- a/util/metrics/tagcounter_test.go
+++ b/util/metrics/tagcounter_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/process.go b/util/process.go
index 4ea34796a5..6a6d17d4ad 100644
--- a/util/process.go
+++ b/util/process.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/process_common.go b/util/process_common.go
index b2c1bc7081..03af826b0c 100644
--- a/util/process_common.go
+++ b/util/process_common.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/process_windows.go b/util/process_windows.go
index 4cad60df7e..7fe14d9742 100644
--- a/util/process_windows.go
+++ b/util/process_windows.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/rateLimit.go b/util/rateLimit.go
new file mode 100644
index 0000000000..c4e85c71e7
--- /dev/null
+++ b/util/rateLimit.go
@@ -0,0 +1,556 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package util
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math"
+ "math/rand"
+ "sort"
+ "sync"
+ "time"
+
+ "github.com/algorand/go-algorand/util/metrics"
+ "github.com/algorand/go-deadlock"
+)
+
+var errConManDropped = errors.New("congestionManager prevented client from consuming capacity")
+var errFailedConsume = errors.New("could not consume capacity from capacityQueue")
+var errERLReservationExists = errors.New("client already has a reservation")
+var errCapacityReturn = errors.New("could not replace capacity to channel")
+
+// ElasticRateLimiter holds and distributes capacity through capacityQueues
+// Capacity consumers are given an error if there is no capacity available for them,
+// and a "capacityGuard" structure they can use to return the capacity when finished
+type ElasticRateLimiter struct {
+ MaxCapacity int
+ CapacityPerReservation int
+ sharedCapacity capacityQueue
+ capacityByClient map[ErlClient]capacityQueue
+ clientLock deadlock.RWMutex
+ // CongestionManager and enable flag
+ cm CongestionManager
+ enableCM bool
+ congestionControlCounter *metrics.Counter
+}
+
+// ErlClient clients must support OnClose for reservation closing
+type ErlClient interface {
+ OnClose(func())
+}
+
+// capacity is an empty structure used for loading and draining queues
+type capacity struct {
+}
+
+// Capacity Queue wraps and maintains a channel of opaque capacity structs
+type capacityQueue chan capacity
+
+// ErlCapacityGuard is the structure returned to clients so they can release the capacity when needed
+// they also inform the congestion manager of events
+type ErlCapacityGuard struct {
+ cq capacityQueue
+ cm CongestionManager
+}
+
+// Release will put capacity back into the queue attached to this capacity guard
+func (cg *ErlCapacityGuard) Release() error {
+ if cg.cq == nil {
+ return nil
+ }
+ select {
+ case cg.cq <- capacity{}:
+ return nil
+ default:
+ return errCapacityReturn
+ }
+}
+
+// Served will notify the CongestionManager that this resource has been served, informing the Service Rate
+func (cg *ErlCapacityGuard) Served() {
+ if cg.cm != nil {
+ cg.cm.Served(time.Now())
+ }
+}
+
+func (q capacityQueue) blockingRelease() {
+ q <- capacity{}
+}
+
+func (q capacityQueue) blockingConsume() {
+ <-q
+}
+
+func (q capacityQueue) consume(cm CongestionManager) (ErlCapacityGuard, error) {
+ select {
+ case <-q:
+ return ErlCapacityGuard{
+ cq: q,
+ cm: cm,
+ }, nil
+ default:
+ return ErlCapacityGuard{}, errFailedConsume
+ }
+}
+
+// NewElasticRateLimiter creates an ElasticRateLimiter and initializes maps
+// maxCapacity: the total (absolute maximum) number of capacity units vended by this ERL at a given time
+// reservedCapacity: the number of capacity units to be reserved per client
+// cmWindow: the window duration of data collection for congestion management, passed to the congestion manager
+// conmanCount: the metric to increment when the congestion manager proposes dropping a request
+func NewElasticRateLimiter(
+ maxCapacity int,
+ reservedCapacity int,
+ cmWindow time.Duration,
+ conmanCount *metrics.Counter) *ElasticRateLimiter {
+ ret := ElasticRateLimiter{
+ MaxCapacity: maxCapacity,
+ CapacityPerReservation: reservedCapacity,
+ capacityByClient: map[ErlClient]capacityQueue{},
+ sharedCapacity: capacityQueue(make(chan capacity, maxCapacity)),
+ congestionControlCounter: conmanCount,
+ }
+ congestionManager := NewREDCongestionManager(
+ cmWindow,
+ maxCapacity)
+ ret.cm = congestionManager
+ // fill the sharedCapacity
+ for i := 0; i < maxCapacity; i++ {
+ ret.sharedCapacity.blockingRelease()
+ }
+ return &ret
+}
+
+// Start will start any underlying component of the ElasticRateLimiter
+func (erl *ElasticRateLimiter) Start() {
+ if erl.cm != nil {
+ erl.cm.Start()
+ }
+}
+
+// Stop will stop any underlying component of the ElasticRateLimiter
+func (erl *ElasticRateLimiter) Stop() {
+ if erl.cm != nil {
+ erl.cm.Stop()
+ }
+}
+
+// EnableCongestionControl turns on the flag that the ERL uses to check with its CongestionManager
+func (erl *ElasticRateLimiter) EnableCongestionControl() {
+ erl.clientLock.Lock()
+ defer erl.clientLock.Unlock()
+ erl.enableCM = true
+}
+
+// DisableCongestionControl turns off the flag that the ERL uses to check with its CongestionManager
+func (erl *ElasticRateLimiter) DisableCongestionControl() {
+ erl.clientLock.Lock()
+ defer erl.clientLock.Unlock()
+ erl.enableCM = false
+}
+
+// ConsumeCapacity will dispense one capacity from either the resource's reservedCapacity,
+// and will return a guard who can return capacity when the client is ready
+// Returns an error if the capacity could not be vended, which could be:
+// - there is not sufficient free capacity to assign a reserved capacity block
+// - there is no reserved or shared capacity available for the client
+func (erl *ElasticRateLimiter) ConsumeCapacity(c ErlClient) (*ErlCapacityGuard, error) {
+ var q capacityQueue
+ var err error
+ var exists bool
+ var enableCM bool
+ // get the client's queue
+ erl.clientLock.RLock()
+ q, exists = erl.capacityByClient[c]
+ enableCM = erl.enableCM
+ erl.clientLock.RUnlock()
+
+ // Step 0: Check for, and create a capacity reservation if needed
+ if !exists {
+ q, err = erl.openReservation(c)
+ if err != nil {
+ return nil, err
+ }
+ // if the client has been given a new reservation, make sure it cleans up OnClose
+ c.OnClose(func() { erl.closeReservation(c) })
+
+ // if this reservation is newly created, directly (blocking) take a capacity
+ q.blockingConsume()
+ return &ErlCapacityGuard{cq: q, cm: erl.cm}, nil
+ }
+
+ // Step 1: Attempt consumption from the reserved queue
+ cg, err := q.consume(erl.cm)
+ if err == nil {
+ if erl.cm != nil {
+ erl.cm.Consumed(c, time.Now()) // notify the congestion manager that this client consumed from this queue
+ }
+ return &cg, nil
+ }
+ // Step 2: Potentially gate shared queue access if the congestion manager disallows it
+ if erl.cm != nil &&
+ enableCM &&
+ erl.cm.ShouldDrop(c) {
+ if erl.congestionControlCounter != nil {
+ erl.congestionControlCounter.Inc(nil)
+ }
+ return nil, errConManDropped
+ }
+ // Step 3: Attempt consumption from the shared queue
+ cg, err = erl.sharedCapacity.consume(erl.cm)
+ if err != nil {
+ return nil, err
+ }
+ if erl.cm != nil {
+ erl.cm.Consumed(c, time.Now()) // notify the congestion manager that this client consumed from this queue
+ }
+ return &cg, nil
+}
+
+// openReservation creates an entry in the ElasticRateLimiter's reservedCapacity map,
+// and optimistically transfers capacity from the sharedCapacity to the reservedCapacity
+func (erl *ElasticRateLimiter) openReservation(c ErlClient) (capacityQueue, error) {
+ erl.clientLock.Lock()
+ defer erl.clientLock.Unlock()
+ if _, exists := erl.capacityByClient[c]; exists {
+ return capacityQueue(nil), errERLReservationExists
+ }
+ // guard against overprovisioning, if there is less than a reservedCapacity amount left
+ remaining := erl.MaxCapacity - (erl.CapacityPerReservation * len(erl.capacityByClient))
+ if erl.CapacityPerReservation > remaining {
+ return capacityQueue(nil), fmt.Errorf("not enough capacity to reserve for client: %d remaining, %d requested", remaining, erl.CapacityPerReservation)
+ }
+ // make capacity for the provided client
+ q := capacityQueue(make(chan capacity, erl.CapacityPerReservation))
+ erl.capacityByClient[c] = q
+ // create a thread to drain the capacity from sharedCapacity in a blocking way
+ // and move it to the reservation, also in a blocking way
+ go func() {
+ for i := 0; i < erl.CapacityPerReservation; i++ {
+ erl.sharedCapacity.blockingConsume()
+ q.blockingRelease()
+ }
+ }()
+ return q, nil
+}
+
+// closeReservation will remove the client mapping to capacity channel,
+// and will kick off a routine to drain the capacity and replace it to the shared capacity
+func (erl *ElasticRateLimiter) closeReservation(c ErlClient) {
+ erl.clientLock.Lock()
+ defer erl.clientLock.Unlock()
+ q, exists := erl.capacityByClient[c]
+ // guard clauses, and preventing the ElasticRateLimiter from draining its own sharedCapacity
+ if !exists || q == erl.sharedCapacity {
+ return
+ }
+ delete(erl.capacityByClient, c)
+ // start a routine to consume capacity from the closed reservation, and return it to the sharedCapacity
+ go func() {
+ for i := 0; i < erl.CapacityPerReservation; i++ {
+ q.blockingConsume()
+ erl.sharedCapacity.blockingRelease()
+ }
+ }()
+}
+
+// CongestionManager is an interface for tracking events which happen to capacityQueues
+type CongestionManager interface {
+ Start()
+ Stop()
+ Consumed(c ErlClient, t time.Time)
+ Served(t time.Time)
+ ShouldDrop(c ErlClient) bool
+}
+
+type event struct {
+ c ErlClient
+ t time.Time
+}
+
+type shouldDropQuery struct {
+ c ErlClient
+ ret chan bool
+}
+
+// "Random Early Detection" congestion manager,
+// will propose to drop messages proportional to the caller's request rate vs Average Service Rate
+type redCongestionManager struct {
+ runLock *deadlock.Mutex
+ running bool
+ window time.Duration
+ consumed chan event
+ served chan event
+ shouldDropQueries chan shouldDropQuery
+ targetRate float64
+ targetRateRefreshTicks int
+ // exp is applied as an exponential factor in shouldDrop. 1 would be linearly proportional, higher values punish noisy neighbors more
+ exp float64
+ // consumed is the only value tracked by-queue. The others are calculated in-total
+ // TODO: If we desire later, we can add mappings onto release/done for more insight
+ consumedByClient map[ErlClient]*[]time.Time
+ serves []time.Time
+ // synchronization for unit tests
+ ctx context.Context
+ ctxCancel context.CancelFunc
+ wg sync.WaitGroup
+}
+
+// NewREDCongestionManager creates a Congestion Manager which will watches capacityGuard activity,
+// and regularly calculates a Target Service Rate, and can give "Should Drop" suggestions
+func NewREDCongestionManager(d time.Duration, bsize int) *redCongestionManager {
+ ret := redCongestionManager{
+ runLock: &deadlock.Mutex{},
+ window: d,
+ consumed: make(chan event, bsize),
+ served: make(chan event, bsize),
+ shouldDropQueries: make(chan shouldDropQuery, bsize),
+ targetRateRefreshTicks: bsize / 10, // have the Congestion Manager refresh its target rates every 10% through the queue
+ consumedByClient: map[ErlClient]*[]time.Time{},
+ exp: 4,
+ wg: sync.WaitGroup{},
+ }
+ return &ret
+}
+
+// Consumed implements CongestionManager by putting an event on the consumed channel,
+// to be processed by the Start() loop
+func (cm *redCongestionManager) Consumed(c ErlClient, t time.Time) {
+ select {
+ case cm.consumed <- event{
+ c: c,
+ t: t,
+ }:
+ default:
+ }
+}
+
+// Served implements CongestionManager by putting an event on the done channel,
+// to be processed by the Start() loop
+func (cm *redCongestionManager) Served(t time.Time) {
+ select {
+ case cm.served <- event{
+ t: t,
+ }:
+ default:
+ }
+}
+
+// ShouldDrop implements CongestionManager by putting a query shouldDropQueries channel,
+// and blocks on the response to return synchronously to the caller
+// if an error should prevent the query from running, the result is defaulted to false
+func (cm *redCongestionManager) ShouldDrop(c ErlClient) bool {
+ ret := make(chan bool)
+ select {
+ case cm.shouldDropQueries <- shouldDropQuery{
+ c: c,
+ ret: ret,
+ }:
+ return <-ret
+ default:
+ return false
+ }
+}
+
+// Start will kick off a goroutine to consume activity from the different activity channels,
+// as well as service queries about if a given capacityQueue should drop
+func (cm *redCongestionManager) Start() {
+ // check if the maintainer is already running to ensure there is only one routine
+ cm.runLock.Lock()
+ defer cm.runLock.Unlock()
+ if cm.running {
+ return
+ }
+ cm.ctx, cm.ctxCancel = context.WithCancel(context.Background())
+ cm.running = true
+ cm.wg.Add(1)
+ go cm.run()
+}
+
+func (cm *redCongestionManager) run() {
+ tick := 0
+ targetRate := float64(0)
+ consumedByClient := map[ErlClient]*[]time.Time{}
+ serves := []time.Time{}
+ lastServiceRateUpdate := time.Now()
+ exit := false
+ for {
+ select {
+ // prioritize shouldDropQueries
+ case query := <-cm.shouldDropQueries:
+ cutoff := time.Now().Add(-1 * cm.window)
+ prune(consumedByClient[query.c], cutoff)
+ query.ret <- cm.shouldDrop(targetRate, query.c, consumedByClient[query.c])
+ default:
+ select {
+ // "should drop" queries
+ case query := <-cm.shouldDropQueries:
+ cutoff := time.Now().Add(-1 * cm.window)
+ prune(consumedByClient[query.c], cutoff)
+ query.ret <- cm.shouldDrop(targetRate, query.c, consumedByClient[query.c])
+ // consumed events -- a client has consumed capacity from a queue
+ case e := <-cm.consumed:
+ if consumedByClient[e.c] == nil {
+ ts := []time.Time{}
+ consumedByClient[e.c] = &ts
+ }
+ *(consumedByClient[e.c]) = append(*(consumedByClient[e.c]), e.t)
+ // served events -- the capacity has been totally served
+ case e := <-cm.served:
+ serves = append(serves, e.t)
+ // check for context Done, and flag the thread for shutdown
+ case <-cm.ctx.Done():
+ exit = true
+ }
+
+ }
+ // recalculate the service rate every N ticks, or every 100ms
+ // also calculate if the routine is going to exit
+ tick = (tick + 1) % cm.targetRateRefreshTicks
+ if tick == 0 || time.Now().After(lastServiceRateUpdate.Add(100*time.Millisecond)) || exit {
+ lastServiceRateUpdate = time.Now()
+ cutoff := time.Now().Add(-1 * cm.window)
+ prune(&serves, cutoff)
+ for c := range consumedByClient {
+ if prune(consumedByClient[c], cutoff) == 0 {
+ delete(consumedByClient, c)
+ }
+ }
+ targetRate = 0
+ // targetRate is the average service rate per client per second
+ if len(consumedByClient) > 0 {
+ serviceRate := float64(len(serves)) / float64(cm.window/time.Second)
+ targetRate = serviceRate / float64(len(consumedByClient))
+ }
+ }
+ if exit {
+ cm.setTargetRate(targetRate)
+ cm.setConsumedByClient(consumedByClient)
+ cm.setServes(serves)
+ cm.runLock.Lock()
+ defer cm.runLock.Unlock()
+ cm.running = false
+ cm.wg.Done()
+ return
+ }
+ }
+}
+
+func (cm *redCongestionManager) Stop() {
+ cm.ctxCancel()
+ cm.wg.Wait()
+}
+
+func (cm *redCongestionManager) setTargetRate(tr float64) {
+ cm.targetRate = tr
+}
+
+func (cm *redCongestionManager) setConsumedByClient(cbc map[ErlClient]*[]time.Time) {
+ cm.consumedByClient = cbc
+}
+
+func (cm *redCongestionManager) setServes(ts []time.Time) {
+ cm.serves = ts
+}
+
+func (cm *redCongestionManager) arrivalRateFor(arrivals *[]time.Time) float64 {
+ clientArrivalRate := float64(0)
+ if arrivals != nil {
+ clientArrivalRate = float64(len(*arrivals)) / float64(cm.window/time.Second)
+ }
+ return clientArrivalRate
+}
+
+// shouldDrop ultimately makes the recommendation to drop a given request through some fairness probability.
+// Comparing this behavior with the behavior of a basic Random Early Detection system:
+// A standard RED model will drop any message with chance proportional to its queue's fullness. The more full, the more random dropping is applied to all clients.
+// In this RED model, there is an application of fairness, in which the chance a client's request is dropped is proportional to their individual arrival rate, vs a per-client service rate.
+// A behavior example is as follows:
+// client1 makes 100 requests over a given sliding window (10s for this example)
+// client2 makes 200 requests over the window
+// all 300 requests were served over the window
+//
+// This means:
+// - client1's arrival rate is 100/10 = 10/s
+// - client2's arrival rate is 200/10 = 20/s
+// - the service rate is 300/10 = 30/s
+// - the *target rate* is the service rate per client: 30/2 = 15/s
+//
+// When a shouldDrop request is made:
+// - client1 shouldDrop: 10 / 15 > random float ?
+// - client2 shouldDrop: 20 / 15 > random float ?
+// - Additionally, the arrival and service rates are raised to an exponential power, to increase contrast.
+//
+// client2 will be throttled because it is making requests in excess of its target rate.
+// client1 will be throttled proportional to its usage of the service rate.
+// over time, client2 will fall in line with the appropriate service rate, while other clients will be able to use the newly freed capacity
+// The net effect is that clients who are disproportionately noisy are dropped more often,
+// while quieter ones are are dropped less often.
+// The reason this works is that the serviceRate represents the ability for the given resource to be serviced (ie, the rate at which work is dequeued).
+// When congestion management is required, the service should attempt a fair distribution of servicing to all clients.
+// clients who are making requests in excess of our known ability to fairly service requests should be reduced.
+func (cm *redCongestionManager) shouldDrop(targetRate float64, c ErlClient, arrivals *[]time.Time) bool {
+ // clients who have "never" been seen do not get dropped
+ clientArrivalRate := cm.arrivalRateFor(arrivals)
+ if clientArrivalRate == 0 {
+ return false
+ }
+ // if targetRate is 0, it means we haven't had any activity to calculate (or there is not enough data)
+ // it should not drop in this case
+ if targetRate == 0 {
+ return false
+ }
+ // A random float is selected, and the arrival rate of the given client is
+ // turned to a ratio against targetRate. the congestion manager recommends to drop activity
+ // proportional to its overuse above the targetRate
+ r := rand.Float64()
+ return (math.Pow(clientArrivalRate, cm.exp) / math.Pow(targetRate, cm.exp)) > r
+}
+
+func prune(ts *[]time.Time, cutoff time.Time) int {
+ // guard against nil lists
+ if ts == nil {
+ return 0
+ }
+ // guard against empty lists
+ if len(*ts) == 0 {
+ return 0
+ }
+ // optimization: if the last element falls before the cutoff, prune the whole list without iteration
+ if (*ts)[len(*ts)-1].Before(cutoff) {
+ *ts = (*ts)[:0]
+ return 0
+ }
+ // optimization: if the list is longer than 50 elements, use a binary search to find the cutoff line
+ if len(*ts) > 50 {
+ i := sort.Search(len(*ts), func(i int) bool { return (*ts)[i].After(cutoff) })
+ *ts = (*ts)[i:]
+ return len(*ts)
+ }
+ // find the first inserted timestamp *after* the cutoff, and cut everything behind it off
+ for i, t := range *ts {
+ if t.After(cutoff) {
+ *ts = (*ts)[i:]
+ return len(*ts)
+ }
+ }
+ // if no values are after the cutoff, clear the array and give back a 0
+ *ts = (*ts)[:0]
+ return 0
+}
diff --git a/util/rateLimit_test.go b/util/rateLimit_test.go
new file mode 100644
index 0000000000..669960ecfc
--- /dev/null
+++ b/util/rateLimit_test.go
@@ -0,0 +1,291 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package util
+
+import (
+ "testing"
+ "time"
+
+ "github.com/algorand/go-algorand/test/partitiontest"
+ "github.com/stretchr/testify/assert"
+)
+
+type mockClient string
+
+type mockCongestionControl struct{}
+
+func (cg mockCongestionControl) Start() {}
+func (cg mockCongestionControl) Stop() {}
+func (cg mockCongestionControl) Consumed(c ErlClient, t time.Time) {}
+func (cg mockCongestionControl) Served(t time.Time) {}
+func (cg mockCongestionControl) ShouldDrop(c ErlClient) bool { return true }
+
+func (c mockClient) OnClose(func()) {
+ return
+}
+
+func TestNewElasticRateLimiter(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ erl := NewElasticRateLimiter(100, 10, time.Second, nil)
+
+ assert.Equal(t, len(erl.sharedCapacity), 100)
+ assert.Equal(t, len(erl.capacityByClient), 0)
+}
+
+func TestElasticRateLimiterCongestionControlled(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client := mockClient("client")
+ erl := NewElasticRateLimiter(3, 2, time.Second, nil)
+ // give the ERL a congestion controler with well defined behavior for testing
+ erl.cm = mockCongestionControl{}
+
+ _, err := erl.ConsumeCapacity(client)
+ // because the ERL gives capacity to a reservation, and then asynchronously drains capacity from the share,
+ // wait a moment before testing the size of the sharedCapacity
+ time.Sleep(100 * time.Millisecond)
+ assert.Equal(t, 1, len(erl.capacityByClient[client]))
+ assert.Equal(t, 1, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+ erl.EnableCongestionControl()
+ _, err = erl.ConsumeCapacity(client)
+ assert.Equal(t, 0, len(erl.capacityByClient[client]))
+ assert.Equal(t, 1, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+ _, err = erl.ConsumeCapacity(client)
+ assert.Equal(t, 0, len(erl.capacityByClient[client]))
+ assert.Equal(t, 1, len(erl.sharedCapacity))
+ assert.Error(t, err)
+
+ erl.DisableCongestionControl()
+ _, err = erl.ConsumeCapacity(client)
+ assert.Equal(t, 0, len(erl.capacityByClient[client]))
+ assert.Equal(t, 0, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+}
+
+func TestReservations(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client1 := mockClient("client1")
+ client2 := mockClient("client2")
+ erl := NewElasticRateLimiter(4, 1, time.Second, nil)
+
+ _, err := erl.ConsumeCapacity(client1)
+ // because the ERL gives capacity to a reservation, and then asynchronously drains capacity from the share,
+ // wait a moment before testing the size of the sharedCapacity
+ time.Sleep(100 * time.Millisecond)
+ assert.Equal(t, 1, len(erl.capacityByClient))
+ assert.NoError(t, err)
+
+ _, err = erl.ConsumeCapacity(client2)
+ // because the ERL gives capacity to a reservation, and then asynchronously drains capacity from the share,
+ // wait a moment before testing the size of the sharedCapacity
+ time.Sleep(100 * time.Millisecond)
+ assert.Equal(t, 2, len(erl.capacityByClient))
+ assert.NoError(t, err)
+
+ erl.closeReservation(client1)
+ assert.Equal(t, 1, len(erl.capacityByClient))
+ erl.closeReservation(client2)
+ assert.Equal(t, 0, len(erl.capacityByClient))
+}
+
+func TestConsumeReleaseCapacity(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client := mockClient("client")
+ erl := NewElasticRateLimiter(4, 3, time.Second, nil)
+
+ c1, err := erl.ConsumeCapacity(client)
+ // because the ERL gives capacity to a reservation, and then asynchronously drains capacity from the share,
+ // wait a moment before testing the size of the sharedCapacity
+ time.Sleep(100 * time.Millisecond)
+ assert.Equal(t, 2, len(erl.capacityByClient[client]))
+ assert.Equal(t, 1, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+ _, err = erl.ConsumeCapacity(client)
+ assert.Equal(t, 1, len(erl.capacityByClient[client]))
+ assert.Equal(t, 1, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+ _, err = erl.ConsumeCapacity(client)
+ assert.Equal(t, 0, len(erl.capacityByClient[client]))
+ assert.Equal(t, 1, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+ // remember this capacity, as it is a shared capacity
+ c4, err := erl.ConsumeCapacity(client)
+ assert.Equal(t, 0, len(erl.capacityByClient[client]))
+ assert.Equal(t, 0, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+ _, err = erl.ConsumeCapacity(client)
+ assert.Equal(t, 0, len(erl.capacityByClient[client]))
+ assert.Equal(t, 0, len(erl.sharedCapacity))
+ assert.Error(t, err)
+
+ // now release the capacity and observe the items return to the correct places
+ err = c1.Release()
+ assert.Equal(t, 1, len(erl.capacityByClient[client]))
+ assert.Equal(t, 0, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+ // now release the capacity and observe the items return to the correct places
+ err = c4.Release()
+ assert.Equal(t, 1, len(erl.capacityByClient[client]))
+ assert.Equal(t, 1, len(erl.sharedCapacity))
+ assert.NoError(t, err)
+
+}
+
+func TestREDCongestionManagerShouldDrop(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client := mockClient("client")
+ other := mockClient("other")
+ red := NewREDCongestionManager(time.Second*10, 10000)
+ // calculate the target rate every request for most accurate results
+ red.targetRateRefreshTicks = 1
+ red.Start()
+ // indicate that the arrival rate is essentially 1/s
+ for i := 0; i < 10; i++ {
+ red.Consumed(client, time.Now())
+ }
+ // indicate that the service rate is essentially 0.9/s
+ for i := 0; i < 9; i++ {
+ red.Served(time.Now())
+ }
+ // allow the statistics to catch up before asserting
+ time.Sleep(100 * time.Millisecond)
+ // the service rate should be 0.9/s, and the arrival rate for this client should be 1/s
+ // for this reason, it should always drop the message
+ for i := 0; i < 100; i++ {
+ assert.True(t, red.ShouldDrop(client))
+ }
+ // this caller hasn't consumed any capacity before, so it won't need to drop
+ for i := 0; i < 10; i++ {
+ assert.False(t, red.ShouldDrop(other))
+ }
+ // allow the congestion manager to consume and process the given messages
+ time.Sleep(100 * time.Millisecond)
+ red.Stop()
+ assert.Equal(t, 10, len(*red.consumedByClient[client]))
+ assert.Equal(t, float64(1), red.arrivalRateFor(red.consumedByClient[client]))
+ assert.Equal(t, 0.0, red.arrivalRateFor(red.consumedByClient[other]))
+ assert.Equal(t, 0.9, red.targetRate)
+}
+
+func TestREDCongestionManagerShouldntDrop(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client := mockClient("client")
+ red := NewREDCongestionManager(time.Second*10, 10000)
+ // calculate the target rate every request for most accurate results
+ red.targetRateRefreshTicks = 1
+ red.Start()
+
+ // indicate that the arrival rate is essentially 0.1/s!
+ red.Consumed(client, time.Now())
+
+ // drive 10k messages, in batches of 500, with 100ms sleeps
+ for i := 0; i < 20; i++ {
+ for j := 0; j < 500; j++ {
+ red.Served(time.Now())
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ // the service rate should be 1000/s, and the arrival rate for this client should be 0.1/s
+ // for this reason, shouldDrop should almost certainly return false (true only 1/100k times)
+ for i := 0; i < 10; i++ {
+ assert.False(t, red.ShouldDrop(client))
+ }
+ // allow the congestion manager to consume and process the given messages
+ time.Sleep(1000 * time.Millisecond)
+ red.Stop()
+ assert.Equal(t, 1, len(*red.consumedByClient[client]))
+ assert.Equal(t, 10000, len(red.serves))
+ assert.Equal(t, 0.1, red.arrivalRateFor(red.consumedByClient[client]))
+ assert.Equal(t, float64(1000), red.targetRate)
+}
+
+func TestREDCongestionManagerTargetRate(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client := mockClient("client")
+ red := NewREDCongestionManager(time.Second*10, 10000)
+ red.Start()
+ red.Consumed(client, time.Now())
+ red.Consumed(client, time.Now())
+ red.Consumed(client, time.Now())
+ red.Served(time.Now())
+ red.Served(time.Now())
+ red.Served(time.Now())
+ // allow the congestion manager to consume and process the given messages
+ time.Sleep(100 * time.Millisecond)
+ red.Stop()
+ assert.Equal(t, 0.3, red.arrivalRateFor(red.consumedByClient[client]))
+ assert.Equal(t, 0.3, red.targetRate)
+}
+
+func TestREDCongestionManagerPrune(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client := mockClient("client")
+ red := NewREDCongestionManager(time.Second*10, 10000)
+ red.Start()
+ red.Consumed(client, time.Now().Add(-11*time.Second))
+ red.Consumed(client, time.Now().Add(-11*time.Second))
+ red.Consumed(client, time.Now().Add(-11*time.Second))
+ red.Consumed(client, time.Now())
+ red.Served(time.Now().Add(-11 * time.Second))
+ red.Served(time.Now().Add(-11 * time.Second))
+ red.Served(time.Now().Add(-11 * time.Second))
+ red.Served(time.Now())
+ // allow the congestion manager to consume and process the given messages
+ time.Sleep(100 * time.Millisecond)
+ red.Stop()
+ assert.Equal(t, 0.1, red.arrivalRateFor(red.consumedByClient[client]))
+ assert.Equal(t, 0.1, red.targetRate)
+}
+
+func TestREDCongestionManagerStopStart(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ client := mockClient("client")
+ red := NewREDCongestionManager(time.Second*10, 10000)
+ red.Start()
+ red.Consumed(client, time.Now())
+ red.Consumed(client, time.Now())
+ red.Consumed(client, time.Now())
+ red.Served(time.Now())
+ red.Served(time.Now())
+ red.Served(time.Now())
+ // allow the congestion manager to consume and process the given messages
+ time.Sleep(100 * time.Millisecond)
+ red.Stop()
+ assert.Equal(t, 0.3, red.arrivalRateFor(red.consumedByClient[client]))
+ assert.Equal(t, 0.3, red.targetRate)
+ // Do it all again, but with 2 calls instead of 3 and 4 serves instead of 3
+ red.Start()
+ red.Consumed(client, time.Now())
+ red.Consumed(client, time.Now())
+ red.Served(time.Now())
+ red.Served(time.Now())
+ red.Served(time.Now())
+ red.Served(time.Now())
+ // allow the congestion manager to consume and process the given messages
+ time.Sleep(100 * time.Millisecond)
+ red.Stop()
+ assert.Equal(t, 0.2, red.arrivalRateFor(red.consumedByClient[client]))
+ assert.Equal(t, 0.4, red.targetRate)
+}
diff --git a/util/s3/fileIterator.go b/util/s3/fileIterator.go
index d5adbd0961..b4f4ea82da 100644
--- a/util/s3/fileIterator.go
+++ b/util/s3/fileIterator.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go
index 43f017443a..8c70f14eda 100644
--- a/util/s3/s3Helper.go
+++ b/util/s3/s3Helper.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -22,6 +22,7 @@ import (
"io"
"os"
"path/filepath"
+ "reflect"
"regexp"
"runtime"
"strconv"
@@ -32,9 +33,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
-
- "github.com/algorand/go-algorand/util"
- "github.com/algorand/go-algorand/util/codecs"
)
const (
@@ -45,9 +43,6 @@ const (
s3DefaultReleaseBucket = "algorand-releases"
s3DefaultUploadBucket = "algorand-uploads"
s3DefaultRegion = "us-east-1"
-
- downloadAction = "download"
- uploadAction = "upload"
)
// Helper encapsulates the s3 session state for interactive with our default S3 bucket with appropriate credentials
@@ -84,20 +79,12 @@ func getS3Region() (region string) {
// MakeS3SessionForUploadWithBucket upload to bucket
func MakeS3SessionForUploadWithBucket(awsBucket string) (helper Helper, err error) {
- creds, err := getCredentials(uploadAction, awsBucket)
- if err != nil {
- return
- }
- return makeS3Session(creds, awsBucket)
+ return makeS3Session(awsBucket)
}
// MakeS3SessionForDownloadWithBucket download from bucket
func MakeS3SessionForDownloadWithBucket(awsBucket string) (helper Helper, err error) {
- creds, err := getCredentials(downloadAction, awsBucket)
- if err != nil {
- return
- }
- return makeS3Session(creds, awsBucket)
+ return makeS3Session(awsBucket)
}
// UploadFileStream sends file as stream to s3
@@ -114,84 +101,43 @@ func (helper *Helper) UploadFileStream(targetFile string, reader io.Reader) erro
return nil
}
-type s3Keys struct {
- ID string
- Secret string
-}
-
-func getCredentials(action string, awsBucket string) (creds *credentials.Credentials, err error) {
- awsID, awsKey := getAWSCredentials()
- credentailsRequired := checkCredentialsRequired(action, awsBucket)
- if !credentailsRequired && (awsID == "" || awsKey == "") {
- return credentials.AnonymousCredentials, nil
- }
- err = validateS3Credentials(awsID, awsKey)
- if err != nil {
+func validateS3Bucket(awsBucket string) (err error) {
+ if awsBucket == "" {
+ err = fmt.Errorf("bucket name is empty")
return
}
- creds = credentials.NewStaticCredentials(awsID, awsKey, "")
return
-
}
-func loadS3KeysFromFile(keyFile string) (keys s3Keys, err error) {
- err = codecs.LoadObjectFromFile(keyFile, &keys)
- return
-}
-
-func getAWSCredentials() (awsID string, awsKey string) {
- awsID, _ = os.LookupEnv("AWS_ACCESS_KEY_ID")
- awsKey, _ = os.LookupEnv("AWS_SECRET_ACCESS_KEY")
-
- // If not in environment, try to load from s3.json file in bin dir
- if awsID == "" || awsKey == "" {
- baseDir, err := util.ExeDir()
- if err == nil {
- keyFile := filepath.Join(baseDir, "s3.json")
- keys, err := loadS3KeysFromFile(keyFile)
- if err == nil {
- awsID = keys.ID
- awsKey = keys.Secret
- }
- }
- }
- return
-}
-
-func validateS3Credentials(awsID string, awsKey string) (err error) {
- if awsID == "" || awsKey == "" {
- err = fmt.Errorf("AWS credentials must be specified in environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY")
+func makeS3Session(bucket string) (helper Helper, err error) {
+ err = validateS3Bucket(bucket)
+ if err != nil {
return
}
- return
-}
-func validateS3Bucket(awsBucket string) (err error) {
- if awsBucket == "" {
- err = fmt.Errorf("bucket name is empty")
- return
+ awsConfig := &aws.Config{
+ CredentialsChainVerboseErrors: aws.Bool(true),
+ Region: aws.String(getS3Region()),
}
- return
-}
-func checkCredentialsRequired(action string, bucketName string) (required bool) {
- required = true
- if action == downloadAction && bucketName == s3DefaultReleaseBucket {
- required = false
+ // s3DefaultReleaseBucket should be public, use AnonymousCredentials
+ if bucket == s3DefaultReleaseBucket {
+ awsConfig.Credentials = credentials.AnonymousCredentials
}
- return
-}
-func makeS3Session(credentials *credentials.Credentials, bucket string) (helper Helper, err error) {
- err = validateS3Bucket(bucket)
+ sess, err := session.NewSessionWithOptions(session.Options{
+ SharedConfigState: session.SharedConfigEnable,
+ Config: *awsConfig,
+ })
if err != nil {
return
}
- sess, err := session.NewSession(&aws.Config{Region: aws.String(getS3Region()),
- Credentials: credentials})
- if err != nil {
- return
+
+ // use AnonymousCredentials if none are found
+ if creds, err := sess.Config.Credentials.Get(); err != nil && !reflect.DeepEqual(creds, credentials.AnonymousCredentials) {
+ sess.Config.Credentials = credentials.AnonymousCredentials
}
+
helper = Helper{
session: sess,
bucket: bucket,
@@ -294,7 +240,7 @@ func (helper *Helper) GetPackageFilesVersion(channel string, pkgFiles string, sp
func GetVersionFromName(name string) (version uint64, err error) {
re := regexp.MustCompile(`_(\d*)\.(\d*)\.(\d*)`)
submatchAll := re.FindAllStringSubmatch(name, -1)
- if submatchAll == nil || len(submatchAll) == 0 || len(submatchAll[0]) != 4 {
+ if len(submatchAll) == 0 || len(submatchAll[0]) != 4 {
err = errors.New("unable to parse version from filename " + name)
return
}
diff --git a/util/s3/s3Helper_test.go b/util/s3/s3Helper_test.go
index 6c46095015..c0502a3b01 100644
--- a/util/s3/s3Helper_test.go
+++ b/util/s3/s3Helper_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -114,8 +114,6 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) {
const emptyBucket = ""
type args struct {
awsBucket string
- awsID string
- awsSecret string
}
tests := []struct {
name string
@@ -123,21 +121,12 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) {
wantHelper Helper
wantErr bool
}{
- {name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
- {name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
- {name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
- {name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
- {name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
- // public upload bucket requires AWS credentials for uploads
- {name: "test6", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false},
- {name: "test7", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
- {name: "test8", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
- {name: "test9", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
+ {name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
+ {name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
+ {name: "test6", args: args{awsBucket: publicUploadBucket}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID)
- os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret)
gotHelper, err := MakeS3SessionForUploadWithBucket(tt.args.awsBucket)
if (err != nil) != tt.wantErr {
t.Errorf("MakeS3SessionForUploadWithBucket() error = %v, wantErr %v", err, tt.wantErr)
@@ -158,8 +147,6 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) {
const emptyBucket = ""
type args struct {
awsBucket string
- awsID string
- awsSecret string
}
tests := []struct {
name string
@@ -167,21 +154,12 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) {
wantHelper Helper
wantErr bool
}{
- {name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
- {name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
- {name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
- {name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
- {name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
- // public release bucket does not require AWS credentials for downloads
- {name: "test6", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
- {name: "test7", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
- {name: "test8", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
- {name: "test9", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
+ {name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
+ {name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
+ {name: "test6", args: args{awsBucket: publicReleaseBucket}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID)
- os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret)
gotHelper, err := MakeS3SessionForDownloadWithBucket(tt.args.awsBucket)
if (err != nil) != tt.wantErr {
t.Errorf("MakeS3SessionForDownloadWithBucket() error = %v, wantErr %v", err, tt.wantErr)
diff --git a/util/sleep.go b/util/sleep.go
index 0d3a60acf0..3ad397bf0c 100644
--- a/util/sleep.go
+++ b/util/sleep.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/sleep_linux.go b/util/sleep_linux.go
index 02d4503036..1540fce736 100644
--- a/util/sleep_linux.go
+++ b/util/sleep_linux.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/sleep_linux_32.go b/util/sleep_linux_32.go
index 50a0e696c2..8dd445e2ac 100644
--- a/util/sleep_linux_32.go
+++ b/util/sleep_linux_32.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/sleep_linux_64.go b/util/sleep_linux_64.go
index b2f7a69dbe..0fff615e76 100644
--- a/util/sleep_linux_64.go
+++ b/util/sleep_linux_64.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/tar/tar.go b/util/tar/tar.go
index 68d03b51f1..2ffc78c1bb 100644
--- a/util/tar/tar.go
+++ b/util/tar/tar.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/tar/untar.go b/util/tar/untar.go
index e6de065a66..fcbccc755f 100644
--- a/util/tar/untar.go
+++ b/util/tar/untar.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/tcpinfo.go b/util/tcpinfo.go
index 2b4c69d294..c387bba33f 100644
--- a/util/tcpinfo.go
+++ b/util/tcpinfo.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/tcpinfo_darwin.go b/util/tcpinfo_darwin.go
index cae19d06d4..ecb06ab668 100644
--- a/util/tcpinfo_darwin.go
+++ b/util/tcpinfo_darwin.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/tcpinfo_linux.go b/util/tcpinfo_linux.go
index 3da707e1cd..8cf1687aed 100644
--- a/util/tcpinfo_linux.go
+++ b/util/tcpinfo_linux.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
@@ -56,6 +56,7 @@ func getConnTCPInfo(raw syscall.RawConn) (*TCPInfo, error) {
// linuxTCPInfo is based on linux include/uapi/linux/tcp.h struct tcp_info
//revive:disable:var-naming
+//nolint:structcheck // complains about unused fields that are rqeuired to match C tcp_info struct
type linuxTCPInfo struct {
state uint8
ca_state uint8
diff --git a/util/tcpinfo_noop.go b/util/tcpinfo_noop.go
index a155ed9298..7eecba7584 100644
--- a/util/tcpinfo_noop.go
+++ b/util/tcpinfo_noop.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/timers/frozen.go b/util/timers/frozen.go
index a9aa6c1b71..e6487b1a8a 100644
--- a/util/timers/frozen.go
+++ b/util/timers/frozen.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/timers/interface.go b/util/timers/interface.go
index 6c8c493e12..e96aced757 100644
--- a/util/timers/interface.go
+++ b/util/timers/interface.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/timers/monotonic.go b/util/timers/monotonic.go
index 80b1a7de45..70db87da3e 100644
--- a/util/timers/monotonic.go
+++ b/util/timers/monotonic.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/timers/monotonic_test.go b/util/timers/monotonic_test.go
index b9fecdbdbb..f8821b300b 100644
--- a/util/timers/monotonic_test.go
+++ b/util/timers/monotonic_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/tokens/tokens.go b/util/tokens/tokens.go
index 930f030bbe..1064a5863c 100644
--- a/util/tokens/tokens.go
+++ b/util/tokens/tokens.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/util.go b/util/util.go
index a19a5b0690..0ecf357344 100644
--- a/util/util.go
+++ b/util/util.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/util_windows.go b/util/util_windows.go
index 5a533b655d..fa91a4f8a8 100644
--- a/util/util_windows.go
+++ b/util/util_windows.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/uuid/uuid.go b/util/uuid/uuid.go
index ab99a5a3d5..e159ecf46e 100644
--- a/util/uuid/uuid.go
+++ b/util/uuid/uuid.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/uuid/uuid_test.go b/util/uuid/uuid_test.go
index 6656672256..17914af3cc 100644
--- a/util/uuid/uuid_test.go
+++ b/util/uuid/uuid_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
diff --git a/util/watchdogStreamReader.go b/util/watchdogStreamReader.go
index 8842483bd3..a54ca376bb 100644
--- a/util/watchdogStreamReader.go
+++ b/util/watchdogStreamReader.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
+// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify