From 1b2d549451e94b60607e81ad992eebf09a054fc6 Mon Sep 17 00:00:00 2001 From: Benjamin Chan Date: Wed, 12 Jun 2019 12:55:50 -0400 Subject: [PATCH 01/54] fix flaky agreement test: make sortition deterministic --- agreement/service_test.go | 92 +++++++++++++++++++++++++-------------- crypto/rand.go | 4 +- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/agreement/service_test.go b/agreement/service_test.go index dc5fbe3e88..0c759193d6 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -583,6 +583,18 @@ func (l amCoserviceListener) dec(sum uint) { } } +// copied from fuzzer/ledger_test.go. We can merge once a refactor seems necessary. +func generatePseudoRandomVRF(keynum int) *crypto.VRFSecrets { + seed := [32]byte{} + seed[0] = byte(keynum % 255) + seed[1] = byte(keynum / 255) + pk, sk := crypto.VrfKeygenFromSeed(seed) + return &crypto.VRFSecrets{ + PK: pk, + SK: sk, + } +} + func createTestAccountsAndBalances(t *testing.T, numNodes int, rootSeed []byte) (accounts []account.Participation, balances map[basics.Address]basics.BalanceRecord) { off := int(rand.Uint32() >> 2) // prevent name collision from running tests more than once @@ -590,44 +602,63 @@ func createTestAccountsAndBalances(t *testing.T, numNodes int, rootSeed []byte) accounts = make([]account.Participation, numNodes) balances = make(map[basics.Address]basics.BalanceRecord, numNodes) var seed crypto.Seed - if len(rootSeed) < 32 { - crypto.RandBytes(seed[:]) - } else { - copy(seed[:], rootSeed) - } + copy(seed[:], rootSeed) for i := 0; i < numNodes; i++ { - stake := basics.MicroAlgos{Raw: 1000000} - firstValid := basics.Round(0) - lastValid := basics.Round(1000) - - rootAccess, err := db.MakeAccessor(t.Name()+"root"+strconv.Itoa(i+off), false, true) - - if err != nil { - panic(err) - } - - seed = sha256.Sum256(seed[:]) - root, err := account.ImportRoot(rootAccess, seed) - if err != nil { - panic(err) + var rootAddress basics.Address + // add new account rootAddress to db + { + rootAccess, err := db.MakeAccessor(t.Name()+"root"+strconv.Itoa(i+off), false, true) + if err != nil { + panic(err) + } + seed = sha256.Sum256(seed[:]) // rehash every node to get different root addresses + root, err := account.ImportRoot(rootAccess, seed) + if err != nil { + panic(err) + } + rootAddress = root.Address() } - rootAddress := root.Address() - - partAccess, err := db.MakeAccessor(t.Name()+"part"+strconv.Itoa(i+off), false, true) - if err != nil { - panic(err) + var v *crypto.OneTimeSignatureSecrets + firstValid := basics.Round(0) + lastValid := basics.Round(1000) + // generate new participation keys + { + // Compute how many distinct participation keys we should generate + keyDilution := config.Consensus[protocol.ConsensusCurrentVersion].DefaultKeyDilution + firstID := basics.OneTimeIDForRound(firstValid, keyDilution) + lastID := basics.OneTimeIDForRound(lastValid, keyDilution) + numBatches := lastID.Batch - firstID.Batch + 1 + + // Generate them + v = crypto.GenerateOneTimeSignatureSecrets(firstID.Batch, numBatches) } - accounts[i], err = account.FillDBWithParticipationKeys(partAccess, rootAddress, firstValid, lastValid, config.Consensus[protocol.ConsensusCurrentVersion].DefaultKeyDilution) - if err != nil { - panic(err) + // save partkeys to db + { + partAccess, err := db.MakeAccessor(t.Name()+"part"+strconv.Itoa(i+off), false, true) + if err != nil { + panic(err) + } + accounts[i] = account.Participation{ + Parent: rootAddress, + VRF: generatePseudoRandomVRF(i), + Voting: v, + FirstValid: firstValid, + LastValid: lastValid, + Store: partAccess, + } + err = accounts[i].Persist() + if err != nil { + panic(err) + } } + // expose balances for future ledger creation acctData := basics.AccountData{ Status: basics.Online, - MicroAlgos: stake, + MicroAlgos: basics.MicroAlgos{Raw: 1000000}, VoteID: accounts[i].VotingSecrets().OneTimeSignatureVerifier, SelectionID: accounts[i].VRFSecrets().PK, } @@ -657,8 +688,7 @@ func setupAgreement(t *testing.T, numNodes int, traceLevel traceLevel, ledgerFac bufCap := 1000 // max number of buffered messages // system state setup: keygen, stake initialization - accounts, balances := createTestAccountsAndBalances(t, numNodes, nil) - + accounts, balances := createTestAccountsAndBalances(t, numNodes, (&[32]byte{})[:]) baseLedger := makeTestLedger(balances) // logging @@ -669,7 +699,6 @@ func setupAgreement(t *testing.T, numNodes int, traceLevel traceLevel, ledgerFac log.SetLevel(logging.Debug) // node setup - clocks := make([]timers.Clock, numNodes) ledgers := make([]Ledger, numNodes) dbAccessors := make([]db.Accessor, numNodes) @@ -736,7 +765,6 @@ func setupAgreement(t *testing.T, numNodes int, traceLevel traceLevel, ledgerFac panic(r) } } - return baseNetwork, baseLedger, cleanupFn, services, clocks, ledgers, am } diff --git a/crypto/rand.go b/crypto/rand.go index 7bb61b7c9f..6f7dca89a0 100644 --- a/crypto/rand.go +++ b/crypto/rand.go @@ -69,14 +69,14 @@ func RandBytes(buf []byte) { // MakePRNG creates a new PRNG from an initial seed. The implementation is // based on HMAC_DRBG. All random bytes from the PRNG will be determined by -// the initial seed value. +// the initial seed value. Used by test code only. func MakePRNG(seed []byte) *PRNG { return &PRNG{ d: drbg.New(seed), } } -// RandBytes implements the RNG interface for the PRNG. +// RandBytes implements the RNG interface for the PRNG. Used by test code only. func (prng *PRNG) RandBytes(buf []byte) { n, err := prng.d.Read(buf) if err != nil { From f677d386d959dd192b661f610c450c1b9685219b Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Wed, 12 Jun 2019 14:36:55 -0400 Subject: [PATCH 02/54] fix ledger ErrNoEntry issue due to sqlite (#6) * test concurrent SQL writes and reads We see unexpected behavior when we have 1 thread doing inserts, and 2 threads doing selects of recently-inserted rows. One of the threads doing selects will get sql.ErrNoRows even though it is querying a row that was already committed by an insert. The issue might be that shared-cache connections provide serializability but not strict serializability; see: http://mailinglists.sqlite.org/cgi-bin/mailman/private/sqlite-users/2019-June/084813.html * workaround for sqlite issue: never use shared cache The shared cache isn't that important for our workload, and I can't reproduce the ledger bug when running in a no-shared-cache configuration. By using private caches (no shared cache), we also maintain the performance optimization of allowing concurrent reads and writes (and, specifically, TestDBConcurrency passes). --- util/db/dbutil.go | 2 -- util/db/dbutil_test.go | 59 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/util/db/dbutil.go b/util/db/dbutil.go index c6c22d4e03..3d4051be2f 100644 --- a/util/db/dbutil.go +++ b/util/db/dbutil.go @@ -212,8 +212,6 @@ func URI(filename string, readOnly bool, memory bool) string { } if memory { uri += "&mode=memory" - } - if readOnly || memory { uri += "&cache=shared" } return uri diff --git a/util/db/dbutil_test.go b/util/db/dbutil_test.go index aabd216f19..60362bb071 100644 --- a/util/db/dbutil_test.go +++ b/util/db/dbutil_test.go @@ -20,6 +20,8 @@ import ( "database/sql" "errors" "fmt" + "sync" + "sync/atomic" "testing" "github.com/stretchr/testify/require" @@ -202,3 +204,60 @@ func TestDBConcurrency(t *testing.T) { }) require.NoError(t, err) } + +func TestDBConcurrencyRW(t *testing.T) { + fn := fmt.Sprintf("/dev/shm/%s.%d.sqlite3", t.Name(), crypto.RandUint64()) + acc, err := MakeAccessor(fn, false, false) + require.NoError(t, err) + + acc2, err := MakeAccessor(fn, true, false) + require.NoError(t, err) + + err = acc.Atomic(func(tx *sql.Tx) error { + _, err := tx.Exec("CREATE TABLE t (a INTEGER PRIMARY KEY)") + return err + }) + require.NoError(t, err) + + var lastInsert int64 + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + defer atomic.StoreInt64(&lastInsert, -1) + for i := int64(0); i < 10000; i++ { + errw := acc.Atomic(func(tx *sql.Tx) error { + _, err := tx.Exec("INSERT INTO t (a) VALUES (?)", i) + return err + }) + atomic.StoreInt64(&lastInsert, i) + require.NoError(t, errw) + } + }() + + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + id := atomic.LoadInt64(&lastInsert) + if id == 0 { + continue + } + if id < 0 { + break + } + var x int64 + errsel := acc2.Atomic(func(tx *sql.Tx) error { + return tx.QueryRow("SELECT a FROM t WHERE a=?", id).Scan(&x) + }) + if errsel != nil { + t.Errorf("selecting %d: %v", id, errsel) + } + require.Equal(t, x, id) + } + }() + } + + wg.Wait() +} From 53e59ac22792c151f2caa3ebf18185d023817d0a Mon Sep 17 00:00:00 2001 From: algobolson <45948765+algobolson@users.noreply.github.com> Date: Thu, 13 Jun 2019 12:51:44 -0400 Subject: [PATCH 03/54] switch RPM to use dedicated rpm@algorand.com key (#11) --- installer/rpm/RPM-GPG-KEY-Algorand | 109 +++++++---------------------- installer/rpm/algorand.repo | 2 +- 2 files changed, 28 insertions(+), 83 deletions(-) diff --git a/installer/rpm/RPM-GPG-KEY-Algorand b/installer/rpm/RPM-GPG-KEY-Algorand index d503252cf7..71d2149c0a 100755 --- a/installer/rpm/RPM-GPG-KEY-Algorand +++ b/installer/rpm/RPM-GPG-KEY-Algorand @@ -1,85 +1,30 @@ -----BEGIN PGP PUBLIC KEY BLOCK----- -mQGNBFz/pAIBDADXyIjyU4pjcyTelWUBoqLHgBp0aACFFZdvm/86t5BSGpqVdQmS -sxg2s5QfChcmWbBnScTI61ojyDPXDEyelUgjXu5p2Ujz4QWLizlAyUGt9P7fHwkB -RR6cYNFtp2xztVHE00aLMbwbmdpv8tbMbBZDgUW0gTYT1Ha17baBBKQL/u+RypOx -fPRwNsnwJ8rZVJYgHqkOc49rL1t2aCdWztXs54Bdd0j39B2NBdd6WGjaOwdkp10s -xrsKCAyDQg/XU2G30ypAxa7zueIqvPkW27dKc4W9juUzoiRNUepUuChfVM+uwkF4 -IHjZvB3II4n0+SLLnw8kovEdQtbDZSc78Lj9bF/5Q1cJR3lTG2gGV69UWO/kKRER -mqxjGRjQZrJ5wScLtnZ9aDAJ+HLNSXh/eE+zBynMcF2VBEwFRsYjmxRvu063ZKdl -n+yANmGG8kcjc2lrx4f+mEth/i0BNpUJYJl0Y4Yh1QP1hAo8C7w1tfqKsKD5z/UJ -5mhozyw0l87JU/sAEQEAAbQmQWxnb3JhbmQgZGV2ZWxvcGVycyA8ZGV2QGFsZ29y -YW5kLmNvbT6JAc4EEwEKADgWIQRhHelKOW8BNZxyr1btrNKdoQpOpgUCXP+kAgIb -AwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRDtrNKdoQpOpmyGC/0eQrg0lYNz -7OWRBjgvvMtL9uODxXRc7C2tMzIwlqaJ/ZqCYimLbwDg9RJSdCFa9EYIpbXn9Dsz -6wjiyRd4TQuEom10b2CY5Pm/FJB5p/LHctdSSb3IM0kDaQSmfo8wW7aTdlwOYexz -zSI2MzNYt4HWEkjp5MJbmGZSuGPRWI2OL1irDJUvphoK6pvFYUeocTsufHi46ZOl -4T9s8+9EWv8jPPiIXAGmUbK3cNXvwGTe+rmEBwXC3l864x1X/bDVG9ZbAOJRolje -IKUhdqh7DAWj1NVwx+KPGfqZIdeNDz2ATf9ZEWAbYRKhd6C6tbMIU9rXrpMPnFKk -Q7+EXh+lH2BgMcatnnIrBfj9bdnUHpisA+cWkIL7XwpQVTHv/uyeORqFL632/Z+B -WkSjs+fO72umzXQUh6x7q/B8emaKrljE6L7C1BLfJJnKz4+jTRwSo5EJS6oW6eda -ZpUuJm+VTA6NTcJs4+WlqFe+pZQeslZLom1S1H6m6czyTV7Z/zXNc7q5AY0EXP+k -AgEMAN/JndhoZAdKqFzBBh/P1an9c2LHeG70fLFAEplgkB0ndmqQQYtZqAshOVO4 -8KEuEYJnugj4/NSUgJUcUwq5E+rrC8r2ZIYm1vW95NaeAzXpMvDVZ414GYuUXpSu -oMoF+VAJn+hkilLpvAQFGvsvE70BokIQiDp2s5bZpvQFScH+nUlDl3plG6hWjmDU -VNRx6YPCrFTScB+8Ap585dOIqlkWZIyqn89l2K+ceetMfjUyIlaWO+VkTxaHIAt6 -prPN8UrAy6vZmyFMZZrI1+/9U0JaHRrogSIE4lrJqdbz+wfX7UPepUR/DJUpNwFc -eKnPb7fwkJEC2DfCpsRcX1jgB4vYhhDgh6X0qdvNr8udc0uWoHGmUzO/WeRdHVBG -4twTnscxbViMA0skH6J/F4Y8qm1KQ6rQjO0duKNYvyFjDpx1tpX7bburKMMK1Zo8 -lZSL7ZD0PIO2RrURK1tecIZP9TaP1qCvhKdR4xW8OzGhO/cPdNAaC5t5qdGW/FtL -Eh6/SQARAQABiQG2BBgBCgAgFiEEYR3pSjlvATWccq9W7azSnaEKTqYFAlz/pAIC -GwwACgkQ7azSnaEKTqb4dQv+KY80JMfxOBYOj1VM31Zc67T/C8nD7M3qI3ylkXf8 -eK3vi8DRUaBbz50b8zWadTW2ohUe3PQJoheRUQyVFQbO2BiD+qxkmxUJ71jXW5Fq -cB3hcx1BFePkWz1IlsVdTFtVPJVyBCnOFz+zQwBD9pX1lBoVtK9Wh/7jUaWoc6h3 -mXEWHREnJwrdAcxYVSZuiZEBAvESjpe61KaCiacTquxbOIPPPjt08jQzE3SciRKM -wWqp/mvx3hWWFTI5lMvppeEKaNb/C/NTmheVJnt25O8v7Y1KA8ENWnBltmgEcQe9 -tU86vZ2nagtJW03Rj+AYTQzg7MeHis+iWlOGYlTtev4hVz4Lmiv2LoMjhK7lpAth -u6MtcPKxP/nC9se4m2lO86711qPkgS/f9Rcp7vlwcThr9lOtiIRj5e2afapSzeh1 -QdLcO25IFC2J1ed9S+GD2uBxFRDB/D8Pr3J5jvUUCeaYnVhCRrWqN+1tGhHUedDe -yVjCxLOgqpmA0uZgGnpmTPpBuQGNBFz/qNkBDADAjrCex5F5tnfmAmPL2j3G/bAY -4d4g3ybGnsX0AzLmiBJ+X54wxtPLHRwFWr04yD6B9dzMln4S3yuYIkj1tfuSIuu3 -DifbwDOixPNK7yFuFD14Or4n9d1ehN8KexIXmgnFu1cPHecglEihtK2TzvycN6J/ -ri2xcGnTEQFgQoCIohiSK53tj/+BXZS6LyQCek8Zdm9dBJm2Ed796sjIlxXOhqbI -g+GP7VksIIgLVdHmbvrPLdZrKiXrmBxpEmWBILSWPslDwkobpC+Q9V6jUE+qjypF -/Dut4PB8F+7iK+VZaEmsdNx6HjkQdgLcFDwhzb2UjDlO+EmWdBZb2gCmrUQIbSmh -gXIgIa2n0U+qQrWYkRUXSD130mPzIK2dvysO4eKYuikNOwUw/0EIfg+GWus7ISnx -eDehopTffxVdQovSXckMAgXbf/zrr4LmPOTl0v3q6V7cpmG/Up4HB4dEzn4zclo0 -Lfd2c6slsM4D/pX53j/hvgh5ddKystSl2lO7OzsAEQEAAYkDcgQYAQoAJhYhBGEd -6Uo5bwE1nHKvVu2s0p2hCk6mBQJc/6jZAhsCBQkDwmcAAcAJEO2s0p2hCk6mwPQg -BBkBCgAdFiEE8usQMbz1qT5q5sJ+GUfcABZXaNIFAlz/qNkACgkQGUfcABZXaNLU -ewv+IHfJMcq8AXc6po4jHdeD8w5pqkluw3LL9Zr/dwrr5J5ej+ev9dr365GPMWxi -3hZMNWCvalk5lpen2lIBpBbIYnCpsS3VLczk9g3xBJbHuIYC+as7q2eEXS+Ei2HJ -O4x6m9jkCb0YqMkfTsQPiqr5rXgNqPFX79b5e8awU/f/ldJEJFMPfHXvQsP5skqF -czIjwgms1p84PAUrPAWE7RzQEpe0f9dsuXtwOTE6r54TGThpxbZlKPlD2ij9nDqf -gFNFTaRFEa2fCEODzVZkIRtYtg2oOn8fc6PqY8U+aEWYC98QO0d4Rzs+EgRu7KgJ -FdJEbb8YntZzNaCZt72+ilR9qA4AX6/ehfbROYlAib/oY14NdMJSksVDgNVrKWnZ -DkEwHB0Jdkm00glNNcTbQnYHWKHn3LIngAhKDQa00/axgvqZxmzses2RsCVgRtAu -OjWz4qO2qSwbwEYcul/JCWRn2BA8Bm5cFmUPv8dzROJw6AdUp0IQNCIncy6W+jSh -AH5iwn8L+wU5GuY2t1cNuyWxH4g37wdsL/LI2qcoZEKoMjh9cQAVuBOukcAR1KIu -xxUC9FWvfrJhCaPdt45WYPS4gSGI1GAbH4LwDW8gPV7FmEkSmzXXmOhKxjtAGx7D -M8vMq6hqRhK8vWMfoQyL4MTDtiJKeBXKYmILNx5hBgMy9T9nMBA4vCNFfQB+1+Ou -dpHOuTApBpGX+nF38zOV0LhRtOyukL9UKG3XIH0weWPBNHOLAtj4GwXA2MuPWgsV -adBgAHd/nEJtnF+kj4P79DulPtpyoHEY8z8Ee51mulyFllcQvlv0cDFhXme2b1Mp -rErIeRO5VQq0+luOOtxujybeXfOmrhTQlDaDCeI1MUQLmwFHi5GD4VuLGD2da+jR -512tRd5lBFOOQgeq/B9ffvIf/I6uXAqJdIllmkTWkAbQWCFbgt4fKKaZJOcCdEQ0 -/q8+bDDMCHVuVuM1zmy+Z7PR6VMBFMcI8XAwUIaNAMpAADtqKXAvq+iQgilTp9DB -oCU73hn6+7kBjQRc/6mWAQwA35SzNifCKFWb+iX+OcPj3h4FTlHW55zrSvJLhnTG -SARz9odxzVCzq3JYS4VsTNAHmdXr2TV+Gs4HsU1IZzX5fs7N+rA8qi/eOyctEZjm -ZS0+U3fseofh00N6vwNSU3CIRcLQvJy7XvoTFGQ7n6QZ000SHMUkH9Hf3A9dHmcJ -mYHXHl24CSlzik0FkxGhN+yGIpn6eh+caAG2x559q+lWq5fh6OyMpdkD73Y+2ybS -c1FYUopzaf1XOACRvi9qOQE5gq9RVdoqDecTB3LEmCEAxCKir074MvbLNBgFUo/J -QYoJJOgsIGjyTvP2PhySpt/AeYTG5hBI7dOjvt0oDBGT4IA+8CDy57N2yx8MxdNO -H0oe+8CUZh3RK0wZsVpamKAViRpYkXFFHbnphsGXDo/LrzoWHAKdk3+rPknNSSrm -u8l+UUf7ZV771B7JYrmXHpZh2eZyNKtqqPr4lSgTdOEsj2IBpeNx/EgE7chE/FsP -umOR60fQPH3o6ndbbjuVIb/VABEBAAGJAbwEGAEKACYWIQRhHelKOW8BNZxyr1bt -rNKdoQpOpgUCXP+plgIbDAUJA8JnAAAKCRDtrNKdoQpOprafC/9GWT0iAfhOL9y5 -6W34EaWHpUZTbTWOtKJzmf6T2s3Sz7zozB/GnGFR/92hrfKRDx1YhGabthbwhepu -/gkkhvkHYv+IhNA+Wt0BcvXmcDFwjH88W88vupjdXc4EU9eyx+0JI17M6/GrBGOw -WDUX9ok+dAW/RW5pReBTYZy2Q+TCrBvDHRRha2SAk2nQXGshHgUDD8XOp7HmG69s -20kAFOWLVSgBexelLPmF/NMpOLHUSW3zF4Ca+C6vH62Ob78ANf3Wytvz5mgIq47j -iZxOhKorukl1m3jJ+IaXamXsPvF37D5wBnA/JRYRXzHTK3LpZsJmbV5zjb0a2yZA -16KXscwl8VLpr7NmwywM+rgaGNtN5WEUw2KA69O+IeLydBsVBJCbSDCGaS1a4NDa -wAkN3cg1XMoDNO5+kRr4C9Nj7A8LOYBIFQezfqPjGP6cbAnbhdefM1QRc9IvW75Z -yH2U78m7vGV5jf5FqKUrytzNYmcqWruiNMHDonGlzaqpmq5K3H8= -=IQbt +mQENBF0BeeQBCADIUu66SGiqaFFcKY18unyFGvq2VU/uoHUiKbMsagvbHxggXeMk +x6t4E1XJ89wmoQJNuxVGGNSaCtdEh65+l5XRrzKJUDb/9CLuf2dvVLWq0sZAo48z +3KW8Si1yrh7rOiOmX46wG5cIhPrZuBtmETtvo8FDrQD2QdlB2dhMhHOnFvgt7m3E +3CX60MNrFmaCREq7IAXHbFc3kcYKfhcCn57GXOBOWdSQgSvCT2sL6ENPmfZKuAqf +tydhRGxfTJhGgym3W6v2UhuGFWjVzmbYRb/nkpG/FVjQovvXASBwZCU84spiL0oJ +PwyYIwut3vTTQu/TpQEgzhhY8kIze13YbP35ABEBAAG0H0FsZ29yYW5kIFJQTSA8 +cnBtQGFsZ29yYW5kLmNvbT6JAVQEEwEIAD4WIQQTk8G1ae02MJH9u2YimZ7Zkai6 +QQUCXQF55AIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAimZ7Z +kai6QWMfB/4mlI/tTx4zEtXQLebpRSpCQtOf3S+mBTglvbrpEsqXlqsAcupm4Dhd +cKDkEl59QE/1rG0hVMU4TWxVr517djZPO4uJu5mDbG7MDe5DxTQuKrNpsYGgSl4m +KaVE9qBZGKm3EGCzFRyS6kN3o0o8FJphSB/5ZlkNZDfCT7tuLklv8MhdSrrsj8vH +nQZK8erPwscI8SPkfEKoRBNuUNXBSrgz0R7ICUI+zoppLtr6Xh50ZL3pvuAKME9c ++10MnvMudewXlkpChayYv+01AzdS14uAA15DbOnET+Ew2tsHSaNUQMUJ6vhph+Ny +hwNdr1bcXzuTzyCrQDSA4KE4/soLFZsVuQENBF0BeeQBCADAHPBKt9mVLh0BP4tx +2ODuA9V6VxYNvViLWxNEtm2qEXdvk+HtGOQv0Fxwwegs6S7OIIMrLlEXXBvpsXp/ +nDefgkNhw7EgzzoIwzpg3MPfhzIKdyFOTiXo0wt9xJh+pnbwMLfe0r4geY1J1d3D +CYdEJh0u/a/KJQZ3zBmXHpkij/IcgsyM4cuh1VCheBNxW0EI8fxlhLtMLD2z5NEz +TUU8MiwL5NlpMC6AMvpbTeBfCvSSeIo+KowdDGd67yWrY1DWq+AQ9DooonE2Z2oq +MyUG2ykpH9SnreZqKOoRlwU+9lxt8LKdZxFTyiG73A40bP988e3QE3M2Tuz0Ei2j +09x1ABEBAAGJATwEGAEIACYWIQQTk8G1ae02MJH9u2YimZ7Zkai6QQUCXQF55AIb +DAUJA8JnAAAKCRAimZ7Zkai6QT6bCACdnrW9Sp2LyfHLKNb3j/WOWX15r8vCcjdG +Xn8bVkwB2KApsICSnW9xtQPBMfoQwlAFcJuaIFcEIl0TyWRamLdRFh+773IKoQgw +YZ0WDx6sNnUB/e9aCuio1kAxmfEXWvEWu7JLN4J5FFmgzJs2JYNSeBRm4kbPjwe/ +zKdBBMK05ND348Jnduj1kZ6fwWKWIue3+nIVUCxyZao5dRp6zhEPtN5gDFpc18uT +kXNB/j0KjwzFUOB3u1ioOMxfbiga9B2fJZO/OSsm+3S4AGNCrn96hRCHnsgPehGT +AZEyE+M1SYulIyylMIcWwOVWUSKxmeSR20g1H83wWZqNpYmdgY24 +=gLD6 -----END PGP PUBLIC KEY BLOCK----- diff --git a/installer/rpm/algorand.repo b/installer/rpm/algorand.repo index dd0f37b4f5..1c3bb57069 100644 --- a/installer/rpm/algorand.repo +++ b/installer/rpm/algorand.repo @@ -3,4 +3,4 @@ name=Algorand baseurl=https://releases.algorand.com/rpm/stable/ enabled=1 gpgcheck=1 -gpgkey=https://releases.algorand.com/key.pub +gpgkey=https://releases.algorand.com/rpm/rpm_algorand.pub From 6535597ffbd1c12448d71d436fa6dc9fe8e5787f Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 13 Jun 2019 13:24:17 -0400 Subject: [PATCH 04/54] mark conffiles as such in debian and rpm packages (#10) --- installer/debian/conffiles | 2 ++ installer/rpm/algorand.spec | 6 +++--- scripts/build_deb.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 installer/debian/conffiles diff --git a/installer/debian/conffiles b/installer/debian/conffiles new file mode 100644 index 0000000000..4aa4aa1622 --- /dev/null +++ b/installer/debian/conffiles @@ -0,0 +1,2 @@ +/etc/apt/apt.conf.d/50algorand-upgrades +/var/lib/algorand/genesis.json diff --git a/installer/rpm/algorand.spec b/installer/rpm/algorand.spec index 60e839f24c..35537ac137 100644 --- a/installer/rpm/algorand.spec +++ b/installer/rpm/algorand.spec @@ -77,15 +77,15 @@ fi /usr/bin/goal /var/lib/algorand/config.json.example /var/lib/algorand/system.json -/var/lib/algorand/genesis.json +%config(noreplace) /var/lib/algorand/genesis.json %if %{RELEASE_GENESIS_PROCESS} != "x" /var/lib/algorand/genesis/devnet/genesis.json /var/lib/algorand/genesis/testnet/genesis.json /var/lib/algorand/genesis/mainnet/genesis.json %endif /lib/systemd/system/algorand.service -/etc/cron.hourly/0yum-algorand-hourly.cron -/etc/yum/yum-cron-algorand.conf +%config(noreplace) /etc/cron.hourly/0yum-algorand-hourly.cron +%config(noreplace) /etc/yum/yum-cron-algorand.conf /etc/pki/rpm-gpg/RPM-GPG-KEY-Algorand /usr/lib/algorand/yum.repos.d/algorand.repo diff --git a/scripts/build_deb.sh b/scripts/build_deb.sh index 7c743d0cef..a125b2b26e 100755 --- a/scripts/build_deb.sh +++ b/scripts/build_deb.sh @@ -101,7 +101,7 @@ for f in "${unattended_upgrades_files[@]}"; do done mkdir -p ${PKG_ROOT}/DEBIAN -debian_files=("control" "postinst" "prerm" "postrm") +debian_files=("control" "postinst" "prerm" "postrm" "conffiles") for ctl in "${debian_files[@]}"; do # Copy first, to preserve permissions, then overwrite to fill in template. cp -a installer/debian/${ctl} ${PKG_ROOT}/DEBIAN/${ctl} From f1af13d7ffb2d32bec403cbeb237621580aace7e Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 13 Jun 2019 13:26:18 -0400 Subject: [PATCH 05/54] make ledger.ErrNoEntry more informative (#3) --- .../algod/api/server/v1/handlers/handlers.go | 9 ++-- ledger/blockdb.go | 12 ++--- ledger/blockqueue.go | 53 +++++++++++++------ ledger/error.go | 12 +++++ node/node.go | 5 +- rpcs/ledgerService.go | 27 +++++----- 6 files changed, 76 insertions(+), 42 deletions(-) diff --git a/daemon/algod/api/server/v1/handlers/handlers.go b/daemon/algod/api/server/v1/handlers/handlers.go index f93c60077a..1e4aa6ae7b 100644 --- a/daemon/algod/api/server/v1/handlers/handlers.go +++ b/daemon/algod/api/server/v1/handlers/handlers.go @@ -792,9 +792,12 @@ func Transactions(ctx lib.ReqContext, w http.ResponseWriter, r *http.Request) { txs, err = ctx.Node.ListTxns(addr, basics.Round(fR), basics.Round(lR)) if err != nil { - if err == ledger.ErrNoEntry && !ctx.Node.IsArchival() { - lib.ErrorResponse(w, http.StatusInternalServerError, err, errBlockHashBeenDeletedArchival, ctx.Log) - return + switch err.(type) { + case ledger.ErrNoEntry: + if !ctx.Node.IsArchival() { + lib.ErrorResponse(w, http.StatusInternalServerError, err, errBlockHashBeenDeletedArchival, ctx.Log) + return + } } lib.ErrorResponse(w, http.StatusInternalServerError, err, err.Error(), ctx.Log) diff --git a/ledger/blockdb.go b/ledger/blockdb.go index 398c0bd855..ad86a439b9 100644 --- a/ledger/blockdb.go +++ b/ledger/blockdb.go @@ -18,7 +18,6 @@ package ledger import ( "database/sql" - "errors" "fmt" "github.com/mattn/go-sqlite3" @@ -29,9 +28,6 @@ import ( "github.com/algorand/go-algorand/protocol" ) -// ErrNoEntry is the error indicating the ledger does not contain an entry for a requested Round -var ErrNoEntry = errors.New("ledger does not have entry") - var blockSchema = []string{ `CREATE TABLE IF NOT EXISTS blocks ( rnd integer primary key, @@ -76,7 +72,7 @@ func blockGet(tx *sql.Tx, rnd basics.Round) (blk bookkeeping.Block, err error) { err = tx.QueryRow("SELECT blkdata FROM blocks WHERE rnd=?", rnd).Scan(&buf) if err != nil { if err == sql.ErrNoRows { - err = ErrNoEntry + err = ErrNoEntry{Round: rnd} } return @@ -91,7 +87,7 @@ func blockGetHdr(tx *sql.Tx, rnd basics.Round) (hdr bookkeeping.BlockHeader, err err = tx.QueryRow("SELECT hdrdata FROM blocks WHERE rnd=?", rnd).Scan(&buf) if err != nil { if err == sql.ErrNoRows { - err = ErrNoEntry + err = ErrNoEntry{Round: rnd} } return @@ -107,7 +103,7 @@ func blockGetCert(tx *sql.Tx, rnd basics.Round) (blk bookkeeping.Block, cert agr err = tx.QueryRow("SELECT blkdata, certdata FROM blocks WHERE rnd=?", rnd).Scan(&blkbuf, &certbuf) if err != nil { if err == sql.ErrNoRows { - err = ErrNoEntry + err = ErrNoEntry{Round: rnd} } return @@ -132,7 +128,7 @@ func blockGetAux(tx *sql.Tx, rnd basics.Round) (blk bookkeeping.Block, aux evalA err = tx.QueryRow("SELECT blkdata, auxdata FROM blocks WHERE rnd=?", rnd).Scan(&blkbuf, &auxbuf) if err != nil { if err == sql.ErrNoRows { - err = ErrNoEntry + err = ErrNoEntry{Round: rnd} } return diff --git a/ledger/blockqueue.go b/ledger/blockqueue.go index aac38c5e78..fe23901661 100644 --- a/ledger/blockqueue.go +++ b/ledger/blockqueue.go @@ -188,29 +188,49 @@ func (bq *blockQueue) putBlock(blk bookkeeping.Block, cert agreement.Certificate return nil } -func (bq *blockQueue) checkEntry(r basics.Round) (e *blockEntry, checkDisk bool) { +func (bq *blockQueue) checkEntry(r basics.Round) (e *blockEntry, lastCommitted basics.Round, latest basics.Round, err error) { bq.mu.Lock() defer bq.mu.Unlock() + // To help the caller form a more informative ErrNoEntry + lastCommitted = bq.lastCommitted + latest = bq.lastCommitted + basics.Round(len(bq.q)) + if r > bq.lastCommitted+basics.Round(len(bq.q)) { - return nil, false + return nil, lastCommitted, latest, ErrNoEntry{ + Round: r, + Latest: latest, + Committed: lastCommitted, + } } if r <= bq.lastCommitted { - return nil, true + return nil, lastCommitted, latest, nil + } + + return &bq.q[r-bq.lastCommitted-1], lastCommitted, latest, nil +} + +func updateErrNoEntry(err error, lastCommitted basics.Round, latest basics.Round) error { + if err != nil { + switch errt := err.(type) { + case ErrNoEntry: + errt.Committed = lastCommitted + errt.Latest = latest + return errt + } } - return &bq.q[int(r-bq.lastCommitted-1)], false + return err } func (bq *blockQueue) getBlock(r basics.Round) (blk bookkeeping.Block, err error) { - e, checkDisk := bq.checkEntry(r) + e, lastCommitted, latest, err := bq.checkEntry(r) if e != nil { return e.block, nil } - if !checkDisk { - err = ErrNoEntry + if err != nil { return } @@ -219,17 +239,17 @@ func (bq *blockQueue) getBlock(r basics.Round) (blk bookkeeping.Block, err error blk, err0 = blockGet(tx, r) return err0 }) + err = updateErrNoEntry(err, lastCommitted, latest) return } func (bq *blockQueue) getBlockHdr(r basics.Round) (hdr bookkeeping.BlockHeader, err error) { - e, checkDisk := bq.checkEntry(r) + e, lastCommitted, latest, err := bq.checkEntry(r) if e != nil { return e.block.BlockHeader, nil } - if !checkDisk { - err = ErrNoEntry + if err != nil { return } @@ -238,17 +258,17 @@ func (bq *blockQueue) getBlockHdr(r basics.Round) (hdr bookkeeping.BlockHeader, hdr, err0 = blockGetHdr(tx, r) return err0 }) + err = updateErrNoEntry(err, lastCommitted, latest) return } func (bq *blockQueue) getBlockCert(r basics.Round) (blk bookkeeping.Block, cert agreement.Certificate, err error) { - e, checkDisk := bq.checkEntry(r) + e, lastCommitted, latest, err := bq.checkEntry(r) if e != nil { return e.block, e.cert, nil } - if !checkDisk { - err = ErrNoEntry + if err != nil { return } @@ -257,17 +277,17 @@ func (bq *blockQueue) getBlockCert(r basics.Round) (blk bookkeeping.Block, cert blk, cert, err0 = blockGetCert(tx, r) return err0 }) + err = updateErrNoEntry(err, lastCommitted, latest) return } func (bq *blockQueue) getBlockAux(r basics.Round) (blk bookkeeping.Block, aux evalAux, err error) { - e, checkDisk := bq.checkEntry(r) + e, lastCommitted, latest, err := bq.checkEntry(r) if e != nil { return e.block, e.aux, nil } - if !checkDisk { - err = ErrNoEntry + if err != nil { return } @@ -276,5 +296,6 @@ func (bq *blockQueue) getBlockAux(r basics.Round) (blk bookkeeping.Block, aux ev blk, aux, err0 = blockGetAux(tx, r) return err0 }) + err = updateErrNoEntry(err, lastCommitted, latest) return } diff --git a/ledger/error.go b/ledger/error.go index 2f1f895c0f..c05c753e34 100644 --- a/ledger/error.go +++ b/ledger/error.go @@ -52,3 +52,15 @@ type ProtocolError protocol.ConsensusVersion func (err ProtocolError) Error() string { return fmt.Sprintf("protocol not supported: %s", err) } + +// ErrNoEntry is used to indicate that a block is not present in the ledger. +type ErrNoEntry struct { + Round basics.Round + Latest basics.Round + Committed basics.Round +} + +// Error satisfies builtin interface `error` +func (err ErrNoEntry) Error() string { + return fmt.Sprintf("ledger does not have entry %d (latest %d, committed %d)", err.Round, err.Latest, err.Committed) +} diff --git a/node/node.go b/node/node.go index 3c0f0c206e..c2de2c2927 100644 --- a/node/node.go +++ b/node/node.go @@ -768,9 +768,10 @@ func (node *AlgorandFullNode) oldKeyDeletionThread() { // r. The params come from agreement.ParamsRound(r), which is r-2. hdr, err := node.ledger.BlockHdr(agreement.ParamsRound(r)) if err != nil { - if err == ledger.ErrNoEntry { + switch err.(type) { + case ledger.ErrNoEntry: // No need to warn; expected during catchup. - } else { + default: node.log.Warnf("Cannot look up block %d for deleting ephemeral keys: %v", agreement.ParamsRound(r), err) } } else { diff --git a/rpcs/ledgerService.go b/rpcs/ledgerService.go index d846f3f8c7..e87e662e44 100644 --- a/rpcs/ledgerService.go +++ b/rpcs/ledgerService.go @@ -151,20 +151,21 @@ func (ls *LedgerService) ServeHTTP(response http.ResponseWriter, request *http.R return } encodedBlockCert, err := ls.encodedBlockCert(round) - switch err { - case ledger.ErrNoEntry: - // entry cound not be found. - response.Header().Set("Cache-Control", ledgerResponseMissingBlockCacheControl) - response.WriteHeader(http.StatusNotFound) - return - default: - // unexpected error. - logging.Base().Warnf("ServeHTTP : failed to retrieve block %d %v", round, err) - response.WriteHeader(http.StatusInternalServerError) - return - case nil: - // no error ! keep going. + if err != nil { + switch err.(type) { + case ledger.ErrNoEntry: + // entry cound not be found. + response.Header().Set("Cache-Control", ledgerResponseMissingBlockCacheControl) + response.WriteHeader(http.StatusNotFound) + return + default: + // unexpected error. + logging.Base().Warnf("ServeHTTP : failed to retrieve block %d %v", round, err) + response.WriteHeader(http.StatusInternalServerError) + return + } } + response.Header().Set("Content-Type", ledgerResponseContentType) response.Header().Set("Content-Length", strconv.Itoa(len(encodedBlockCert))) response.Header().Set("Cache-Control", ledgerResponseHasBlockCacheControl) From 37f42ddabd81ef1a9f9e262b6cc5b48e1a766446 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 13 Jun 2019 13:26:35 -0400 Subject: [PATCH 06/54] cleanup Gopkg.toml based on suggestions from "dep ensure" (#4) --- Gopkg.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index f9b0cca36b..c4ec479f80 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -31,7 +31,6 @@ [[prune.project]] name = "github.com/karalabe/hid" - non-go = false unused-packages = false # The current version of logrus, v1.2.0, implements UnmarshalText but not MarshalText. @@ -75,10 +74,6 @@ name = "github.com/jmoiron/sqlx" version = "1.2.0" -[[constraint]] - name = "gopkg.in/yaml.v2" - version = "2.2.2" - [[constraint]] name = "github.com/algorand/websocket" branch = "master" From 788cbc05ffacd6ad75a9fbae8d108acd04c5bccb Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 13 Jun 2019 13:26:43 -0400 Subject: [PATCH 07/54] websocket: terminate flushThread on Close() without writeTimeout (#5) --- Gopkg.lock | 4 ++-- vendor/github.com/algorand/websocket/.travis.yml | 1 + vendor/github.com/algorand/websocket/conn.go | 12 ++++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d3cc0fd5dd..5ef58f2560 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -21,11 +21,11 @@ [[projects]] branch = "master" - digest = "1:b9c9493bbc4f42ccf5cb7b28fe0303f236696820042180b7715a50f6e1457508" + digest = "1:8f6b71091c3cf5069b57669db97a79fb5939f82eae39f0d33cebb0496a8a1bb6" name = "github.com/algorand/websocket" packages = ["."] pruneopts = "UT" - revision = "a55d40ee35db4f6d093c397796790fe9dc1706a3" + revision = "8e576782c555b1e0edd8acb347649be04ff82d7f" [[projects]] digest = "1:f84c885bd17de475b92997e6e9972465e3c533894dd82223c0f62eda8852cc83" diff --git a/vendor/github.com/algorand/websocket/.travis.yml b/vendor/github.com/algorand/websocket/.travis.yml index 0212203dd9..a5021130cd 100644 --- a/vendor/github.com/algorand/websocket/.travis.yml +++ b/vendor/github.com/algorand/websocket/.travis.yml @@ -7,6 +7,7 @@ matrix: - go: 1.9.x - go: 1.10.x - go: 1.11.x + - go: 1.12.x - go: tip allow_failures: - go: tip diff --git a/vendor/github.com/algorand/websocket/conn.go b/vendor/github.com/algorand/websocket/conn.go index 60be513b62..90125afb3e 100644 --- a/vendor/github.com/algorand/websocket/conn.go +++ b/vendor/github.com/algorand/websocket/conn.go @@ -270,6 +270,7 @@ type Conn struct { bwPresent bool bwLock sync.Mutex bwCond sync.Cond + bwFlushClose chan struct{} readRemaining int64 // bytes remaining in current frame. readFinal bool // true the current message has more frames. readLength int64 // Message size. @@ -326,7 +327,10 @@ func newConn(conn net.Conn, isServer bool, readBufferSize, writeBufferSize int, compressionLevel: defaultCompressionLevel, } c.bwCond.L = &c.bwLock - go c.flushThread() + if c.bwPresent { + c.bwFlushClose = make(chan struct{}) + go c.flushThread() + } c.SetCloseHandler(nil) c.SetPingHandler(nil) c.SetPongHandler(nil) @@ -361,6 +365,7 @@ func (c *Conn) CloseWithoutFlush() error { c.bwLock.Lock() defer c.bwLock.Unlock() if c.bw != nil { + close(c.bwFlushClose) c.bw = nil } c.bwCond.Signal() @@ -471,7 +476,10 @@ func (c *Conn) flushThread() { c.bwLock.Unlock() bwTimeout.Reset(writeTimeout) - <-bwTimeout.C + select { + case <-bwTimeout.C: + case <-c.bwFlushClose: + } c.bwLock.Lock() if c.bw == nil { From d3e0e3423c711990063983e050852fe17a5cc716 Mon Sep 17 00:00:00 2001 From: David Shoots Date: Thu, 13 Jun 2019 10:34:58 -0700 Subject: [PATCH 08/54] [GOAL2-769] Fix update script for joining mainnet (#9) New installs should be able to specify -g mainnet to directly join mainnet. Updating an existing installation should preserve the genesis.json network even if someone manually replaced the original genesis.json file (ignore wallet-genesis.id). --- cmd/updater/update.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/updater/update.sh b/cmd/updater/update.sh index b26d286709..7ed7aeaa3e 100755 --- a/cmd/updater/update.sh +++ b/cmd/updater/update.sh @@ -76,7 +76,7 @@ while [ "$1" != "" ]; do -g) shift GENESIS_NETWORK_DIR=$1 - GENESIS_NETWORK_DIR_SPEC=-g $1 + GENESIS_NETWORK_DIR_SPEC="-g $1" ;; -b) shift @@ -330,7 +330,7 @@ function copy_genesis_files() { function check_for_new_ledger() { CURDATADIR=$1 echo "Checking for new ledger in ${CURDATADIR}" - EXISTING_VER="$(head -n 1 ${CURDATADIR}/wallet-genesis.id)" + EXISTING_VER=$(${UPDATESRCDIR}/bin/algod -d ${CURDATADIR} -g ${CURDATADIR}/genesis.json -G) if [ -z $EXISTING_VER ]; then if [ -z ${GENESIS_NETWORK_DIR} ]; then @@ -359,10 +359,10 @@ function check_for_new_ledger() { # changed the file itself in a compatible way. cp ${UPDATESRCDIR}/genesis/${GENESIS_NETWORK_DIR}/genesis.json ${CURDATADIR} + echo ${NEW_VER} > ${CURDATADIR}/wallet-genesis.id if [ "${NEW_VER}" != "${EXISTING_VER}" ]; then echo "New genesis ID, resetting wallets" NEW_LEDGER=1 - echo ${NEW_VER} > ${CURDATADIR}/wallet-genesis.id reset_wallets_for_new_ledger ${CURDATADIR} import_rootkeys ${CURDATADIR} From 188e1b9006ba9692feccf5b85079eec05ca12092 Mon Sep 17 00:00:00 2001 From: algobolson <45948765+algobolson@users.noreply.github.com> Date: Thu, 13 Jun 2019 13:40:07 -0400 Subject: [PATCH 09/54] skip flakey test (#13) --- logging/telemetryhook_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/logging/telemetryhook_test.go b/logging/telemetryhook_test.go index 9d7ca25144..8fec50ce59 100644 --- a/logging/telemetryhook_test.go +++ b/logging/telemetryhook_test.go @@ -137,6 +137,7 @@ func TestAsyncTelemetryHook_Close(t *testing.T) { } func TestAsyncTelemetryHook_QueueDepth(t *testing.T) { + t.Skip("flakey test can fail on slow test systems") a := require.New(t) t.Parallel() From 689c0c2cb2a5e59f8117bd45f134c41ee583d590 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 13 Jun 2019 14:29:06 -0400 Subject: [PATCH 10/54] goal clerk inspect: print msig PKs using base32+checksum (#16) * goal clerk inspect: print msig PKs using base32+checksum * put boilerplate on top of inspect.go --- cmd/goal/clerk.go | 6 ++- cmd/goal/inspect.go | 116 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 cmd/goal/inspect.go diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index 594ddf38f5..f229247541 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -359,7 +359,11 @@ var inspectCmd = &cobra.Command{ if err != nil { reportErrorf(txDecodeError, txFilename, err) } - fmt.Printf("%s[%d]\n%s\n\n", txFilename, count, string(protocol.EncodeJSON(txn))) + sti, err := inspectTxn(txn) + if err != nil { + reportErrorf(txDecodeError, txFilename, err) + } + fmt.Printf("%s[%d]\n%s\n\n", txFilename, count, string(protocol.EncodeJSON(sti))) } } }, diff --git a/cmd/goal/inspect.go b/cmd/goal/inspect.go new file mode 100644 index 0000000000..5024c27338 --- /dev/null +++ b/cmd/goal/inspect.go @@ -0,0 +1,116 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package main + +import ( + "fmt" + "reflect" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/protocol" +) + +// inspectSignedTxn is isomorphic to SignedTxn but uses different +// types to print public keys using algorand's address format +// (base32 + checksum) in JSON, instead of the default base64. +type inspectSignedTxn struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Sig crypto.Signature `codec:"sig"` + Msig inspectMultisigSig `codec:"msig"` + Txn transactions.Transaction `codec:"txn"` +} + +// inspectMultisigSig is isomorphic to MultisigSig but uses different +// types to print public keys using algorand's address format in JSON. +type inspectMultisigSig struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Version uint8 `codec:"v"` + Threshold uint8 `codec:"thr"` + Subsigs []inspectMultisigSubsig `codec:"subsig"` +} + +// inspectMultisigSig is isomorphic to MultisigSig but uses different +// types to print public keys using algorand's address format in JSON. +type inspectMultisigSubsig struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Key basics.Address `codec:"pk"` + Sig crypto.Signature `codec:"s"` +} + +func inspectTxn(stxn transactions.SignedTxn) (sti inspectSignedTxn, err error) { + sti = txnToInspect(stxn) + if !reflect.DeepEqual(stxn, txnFromInspect(sti)) { + err = fmt.Errorf("non-idempotent transformation to inspectSignedTxn (DeepEqual)") + } + if !reflect.DeepEqual(protocol.Encode(sti), protocol.Encode(stxn)) { + err = fmt.Errorf("non-idempotent transformation to inspectSignedTxn (protocol.Encode)") + } + return +} + +func txnToInspect(stxn transactions.SignedTxn) inspectSignedTxn { + return inspectSignedTxn{ + Txn: stxn.Txn, + Sig: stxn.Sig, + Msig: msigToInspect(stxn.Msig), + } +} + +func txnFromInspect(sti inspectSignedTxn) transactions.SignedTxn { + return transactions.SignedTxn{ + Txn: sti.Txn, + Sig: sti.Sig, + Msig: msigFromInspect(sti.Msig), + } +} + +func msigToInspect(msig crypto.MultisigSig) inspectMultisigSig { + res := inspectMultisigSig{ + Version: msig.Version, + Threshold: msig.Threshold, + } + + for _, subsig := range msig.Subsigs { + res.Subsigs = append(res.Subsigs, inspectMultisigSubsig{ + Sig: subsig.Sig, + Key: basics.Address(subsig.Key), + }) + } + + return res +} + +func msigFromInspect(msi inspectMultisigSig) crypto.MultisigSig { + res := crypto.MultisigSig{ + Version: msi.Version, + Threshold: msi.Threshold, + } + + for _, subsig := range msi.Subsigs { + res.Subsigs = append(res.Subsigs, crypto.MultisigSubsig{ + Sig: subsig.Sig, + Key: crypto.PublicKey(subsig.Key), + }) + } + + return res +} From e61ca7025894af70ba35bce186d2cc25d1add005 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 13 Jun 2019 14:55:12 -0400 Subject: [PATCH 11/54] Specify what version of go we use in the README. (#17) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50d5272616..7fb3705c56 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Our [developer website][developer site url] has the most up to date information ## Building from source ## -Development is done using the [Go Programming Language](https://golang.org/), and this document assumes that you have a functioning environment setup. If you need assistance setting up an environment please visit the [official Go documentation website](https://golang.org/doc/). +Development is done using the [Go Programming Language](https://golang.org/) version 1.12.x, and this document assumes that you have a functioning environment setup. If you need assistance setting up an environment please visit the [official Go documentation website](https://golang.org/doc/). ### Linux / OSX ### From e91385f8d01d4466bd85c6b5f8c30b2602acb95d Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 13 Jun 2019 14:58:26 -0400 Subject: [PATCH 12/54] Disable failing test, and re-enable integration stage. (#14) --- .travis.yml | 28 +++++++++++++-------------- test/scripts/run_integration_tests.sh | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6baae5b47a..af5f20621b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,26 +33,26 @@ jobs: script: - scripts/travis/build.sh || travis_terminate 1; - travis_wait 90 scripts/travis/test.sh -# - # same stage, parallel job -# os: linux -# env: -# - BUILD_TYPE: "integration" -# script: -# - scripts/travis/build.sh || travis_terminate 1; -# - travis_wait 90 scripts/travis/test.sh + - # same stage, parallel job + os: linux + env: + - BUILD_TYPE: "integration" + script: + - scripts/travis/build.sh || travis_terminate 1; + - travis_wait 90 scripts/travis/test.sh - stage: build_release os: linux script: - scripts/travis/build.sh || travis_terminate 1; - travis_wait 90 scripts/travis/test.sh -# - # same stage, parallel job -# os: linux -# env: -# - BUILD_TYPE: "integration" -# script: -# - scripts/travis/build.sh || travis_terminate 1; -# - travis_wait 90 scripts/travis/test.sh + - # same stage, parallel job + os: linux + env: + - BUILD_TYPE: "integration" + script: + - scripts/travis/build.sh || travis_terminate 1; + - travis_wait 90 scripts/travis/test.sh - # same stage, parallel job os: osx script: diff --git a/test/scripts/run_integration_tests.sh b/test/scripts/run_integration_tests.sh index ec9e84ba01..c0b3a5100b 100755 --- a/test/scripts/run_integration_tests.sh +++ b/test/scripts/run_integration_tests.sh @@ -12,7 +12,7 @@ cd ${GOPATH}/src/github.com/algorand/go-algorand # Run more comprehensive tests (not just 'go test' tests) CHANNEL=$(./scripts/travis/channel_for_branch.sh) -./test/scripts/test_running_install_and_update.sh -c "${CHANNEL}" +#./test/scripts/test_running_install_and_update.sh -c "${CHANNEL}" #./test/scripts/test_update_rollback.sh -c "${CHANNEL}" # Test deploying, running, and deleting a local private network From 49c619410d9c9367ead6983fb15afc28baf292ea Mon Sep 17 00:00:00 2001 From: Benjamin Chan Date: Thu, 13 Jun 2019 15:04:54 -0400 Subject: [PATCH 13/54] fix filter for next round period 0 votes This fixes a deviation from the spec. Currently, the code (accidentally) filters all votePresent and voteVerified events from the next round, if myPlayer.Period is large. Instead, whether we filter votes from the next round should not be a function of current period. As specified in the spec, we allow votes with vote.Round = player.Round + 1 and vote.Period == 0 and vote.Step = {propose,soft,cert,next} --- agreement/voteAggregator.go | 9 +++-- agreement/voteAggregator_test.go | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/agreement/voteAggregator.go b/agreement/voteAggregator.go index c27775d655..5f58ed2b76 100644 --- a/agreement/voteAggregator.go +++ b/agreement/voteAggregator.go @@ -248,8 +248,13 @@ func voteFresh(proto protocol.ConsensusVersion, freshData freshnessData, vote un return fmt.Errorf("filtered vote from bad round: player.Round=%v; vote.Round=%v", freshData.PlayerRound, vote.R.Round) } - if freshData.PlayerRound+1 == vote.R.Round && (vote.R.Period > 0 || vote.R.Step > next) { - return fmt.Errorf("filtered future vote from bad period or step: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) + if freshData.PlayerRound+1 == vote.R.Round { + if (vote.R.Period > 0 || vote.R.Step > next) { + return fmt.Errorf("filtered future vote from bad period or step: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) + } else { + // pipeline votes from next round period 0 + return nil + } } switch vote.R.Period { diff --git a/agreement/voteAggregator_test.go b/agreement/voteAggregator_test.go index a2ca0b822d..841769afe7 100644 --- a/agreement/voteAggregator_test.go +++ b/agreement/voteAggregator_test.go @@ -777,3 +777,65 @@ func TestVoteAggregatorFiltersVotePresentPeriod(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, res, "VotePresent not correctly filtered") } + +func TestVoteAggregatorFiltersVoteNextRound(t *testing.T) { + // Set up a composed test machine + rRouter := new(rootRouter) + rRouter.update(player{}, 0, false) + voteM := &ioAutomataConcrete{ + listener: rRouter.voteRoot, + routerCtx: rRouter, + } + helper := voteMakerHelper{} + helper.Setup() + b := testCaseBuilder{} + + // define a current player state for freshness testing + lastConcludingStep := next + msgTemplate := filterableMessageEvent{ + FreshnessData: freshnessData{ + PlayerRound: round(10), + PlayerPeriod: period(10), + PlayerStep: next + 5, + PlayerLastConcluding: lastConcludingStep, + }, + } + // generate old next vote in next round, period 0, step 1; make sure it is accepted + pV := helper.MakeRandomProposalValue() + uv := helper.MakeUnauthenticatedVote(t, 0, round(11), period(0), soft, *pV) + inMsg := msgTemplate // copy + inMsg.messageEvent = messageEvent{ + T: votePresent, + Input: message{ + UnauthenticatedVote: uv, + }, + } + b.AddInOutPair(inMsg, emptyEvent{}) + + // next round, period 0, step > next should be rejected + uv = helper.MakeUnauthenticatedVote(t, 1, round(11), period(0), next+1, *pV) + inMsg = msgTemplate // copy + inMsg.messageEvent = messageEvent{ + T: votePresent, + Input: message{ + UnauthenticatedVote: uv, + }, + } + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + + // next round, period 1 should be rejected + uv = helper.MakeUnauthenticatedVote(t, 1, round(11), period(1), soft, *pV) + inMsg = msgTemplate // copy + inMsg.messageEvent = messageEvent{ + T: votePresent, + Input: message{ + UnauthenticatedVote: uv, + }, + } + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + + // finalize + res, err := b.Build().Validate(voteM) + require.NoError(t, err) + require.NoErrorf(t, res, "Votes from next round not correctly filtered") +} From aa2ec6814395913ce285ddd3c06604800bd8d0b7 Mon Sep 17 00:00:00 2001 From: Benjamin Chan Date: Thu, 13 Jun 2019 15:19:14 -0400 Subject: [PATCH 14/54] run make lint --- agreement/voteAggregator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/agreement/voteAggregator.go b/agreement/voteAggregator.go index 5f58ed2b76..ced4cb600b 100644 --- a/agreement/voteAggregator.go +++ b/agreement/voteAggregator.go @@ -251,10 +251,9 @@ func voteFresh(proto protocol.ConsensusVersion, freshData freshnessData, vote un if freshData.PlayerRound+1 == vote.R.Round { if (vote.R.Period > 0 || vote.R.Step > next) { return fmt.Errorf("filtered future vote from bad period or step: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) - } else { - // pipeline votes from next round period 0 - return nil } + // pipeline votes from next round period 0 + return nil } switch vote.R.Period { From a89961c252dacca4e7e1df813cc23ba227945307 Mon Sep 17 00:00:00 2001 From: Evan Richard Date: Fri, 14 Jun 2019 11:24:58 -0400 Subject: [PATCH 15/54] Check and return error in AlgorandFullNode::BroadcastSignedTxn (#26) AlgorandFullNode::BroadcastSignedTxn checks, logs, and returns an error if an error occurs at any step, except for the final step where the transaction is sent to the networking stack for broadcast. This commit checks, logs, and returns that error if it occurs as well. --- node/node.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/node.go b/node/node.go index c2de2c2927..e1bb402e1b 100644 --- a/node/node.go +++ b/node/node.go @@ -450,7 +450,11 @@ func (node *AlgorandFullNode) BroadcastSignedTxn(signed transactions.SignedTxn) return transactions.Txid{}, err } - node.net.Broadcast(context.TODO(), protocol.TxnTag, protocol.Encode(signed), true, nil) + err = node.net.Broadcast(context.TODO(), protocol.TxnTag, protocol.Encode(signed), true, nil) + if err != nil { + node.log.Infof("failure broadcasting transaction to network: %v - transaction was %+v", err, signed) + return transactions.Txid{}, err + } node.log.Infof("Sent signed tx %s", signed.ID()) return signed.ID(), nil } From dde4f1bca74d4a9b9b3b8b55ab237efac82df56a Mon Sep 17 00:00:00 2001 From: Derek Leung Date: Fri, 14 Jun 2019 15:05:46 -0400 Subject: [PATCH 16/54] Describe how to report vulnerabilities in CONTRIBUTING.md. (#22) Link to the Algorand vulnerability submission form and bug bounty program. --- CONTRIBUTING.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5cb2732fb..18591da8b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,21 +6,31 @@ The algorand project is composed of several repositories on GitHub. Specifically # Filing Issues -Did you discover a bug? Do you have a feature request? Filing issues is an easy way anyone can contribute and helps us improve Algorand. We use GitHub Issues to track all known bugs and feature requests. +Did you discover a bug? Do you have a feature request? Filing issues is an easy way anyone can contribute and helps us improve Algorand. We use GitHub Issues to track all known bugs and feature requests. Before logging an issue be sure to check current issues, verify that your [node is synced](https://developer.algorand.org/docs/introduction-installing-node#sync-node), check the [Developer Frequently Asked Questions](https://developer.algorand.org/docs/developer-faq) and [GitHub issues][issues_url] to see if your issue is described there. If you’d like to contribute to any of the repositories, please file a [GitHub issue][issues_url] using the issues menu item. Make sure to specify whether you are describing a bug or a new enhancement using the **Bug report** or **Feature request** button. -See the GitHub help guide for more information on [filing an issue](https://help.github.com/en/articles/creating-an-issue) +See the GitHub help guide for more information on [filing an issue](https://help.github.com/en/articles/creating-an-issue). + +## Vulnerabilities + +Please don't create issues for any security vulnerabilities. Instead, we would appreciate it if you reported them through our [vulnerability disclosure form][vuln_url]. This allows us to distribute a fix before the vulnerability is exploited. + +Additionally, if you believe that you've discovered a security vulnerability, you might qualify for our bug bounty program. Visit our [bug bounty site][bug_bounty_url] for details. + +If you have any questions, don't hesitate to contact us at security@algorand.com. # Contribution Model For each of our repositories we use the same model for contributing code. Developers wanting to contribute must create pull requests. This process is described in the GitHub [Creating a pull request from a fork](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork) documentation. Each pull request should be initiated against the master branch in the Algorand repository. After a pull request is submitted the core development team will review the submission and communicate with the developer using the comments sections of the PR. After the submission is reviewed and approved, it will be merged into the master branch of the source. These changes will be merged to our release branch on the next viable release date. For the SDKs, this may be immediate. Changes to the node software may take more time as we must ensure and verify the security, as well as apply protocol upgrades in an orderly way. +Again, if you have a patch for a critical security vulnerability, please use our [vulnerability disclosure form][vuln_url] instead of creating a PR. We'll follow up with you on distributing the patch before we merge it. + # Code Guidelines -For Go code we use the [Golang guidelines defined here](https://golang.org/doc/effective_go.html) +For Go code we use the [Golang guidelines defined here](https://golang.org/doc/effective_go.html). * Code must adhere to the official Go formatting guidelines (i.e. uses gofmt). * We use **gofmt** and **golint**. Also make sure to run `make fix` and `make generate` before opening a pull request. * Code must be documented adhering to the official Go commentary guidelines. @@ -34,3 +44,5 @@ For Java code we use [Oracle’s standard formatting rules for Java](https://www The core development team monitors the Algorand community forums and regularly responds to questions and suggestions. Issues and Pull Requests are handled on GitHub. [issues_url]: https://github.com/algorand/go-algorand/issues +[vuln_url]: https://www.algorand.com/resources/blog/security +[bug_bounty_url]: https://bugcrowd.com/algorand From e4dc33abc8e2ed5e5330e90535d649fb08dc03fa Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Fri, 14 Jun 2019 15:10:30 -0400 Subject: [PATCH 17/54] goal clerk inspect: more base32+checksum fixes (#24) Actually display the checksum for multisig PKs; also display txn Sender, Receiver, and CloseRemainderTo using the same address encoding; and fix the counter for displaying multiple txns in a file. --- cmd/goal/clerk.go | 1 + cmd/goal/inspect.go | 122 +++++++++++++++++++++++++++++++++++---- cmd/goal/inspect_test.go | 89 ++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 cmd/goal/inspect_test.go diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index f229247541..e324ad6d93 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -364,6 +364,7 @@ var inspectCmd = &cobra.Command{ reportErrorf(txDecodeError, txFilename, err) } fmt.Printf("%s[%d]\n%s\n\n", txFilename, count, string(protocol.EncodeJSON(sti))) + count++ } } }, diff --git a/cmd/goal/inspect.go b/cmd/goal/inspect.go index 5024c27338..40c8e3e1ba 100644 --- a/cmd/goal/inspect.go +++ b/cmd/goal/inspect.go @@ -32,9 +32,9 @@ import ( type inspectSignedTxn struct { _struct struct{} `codec:",omitempty,omitemptyarray"` - Sig crypto.Signature `codec:"sig"` - Msig inspectMultisigSig `codec:"msig"` - Txn transactions.Transaction `codec:"txn"` + Sig crypto.Signature `codec:"sig"` + Msig inspectMultisigSig `codec:"msig"` + Txn inspectTransaction `codec:"txn"` } // inspectMultisigSig is isomorphic to MultisigSig but uses different @@ -52,32 +52,90 @@ type inspectMultisigSig struct { type inspectMultisigSubsig struct { _struct struct{} `codec:",omitempty,omitemptyarray"` - Key basics.Address `codec:"pk"` + Key checksumAddress `codec:"pk"` Sig crypto.Signature `codec:"s"` } +// inspectTransaction is isomorphic to Transaction but uses different +// types to print public keys using algorand's address format in JSON. +type inspectTransaction struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Type protocol.TxType `codec:"type"` + inspectTxnHeader + transactions.KeyregTxnFields + inspectPaymentTxnFields +} + +// inspectTxnHeader is isomorphic to Header but uses different +// types to print public keys using algorand's address format in JSON. +type inspectTxnHeader struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Sender checksumAddress `codec:"snd"` + Fee basics.MicroAlgos `codec:"fee"` + FirstValid basics.Round `codec:"fv"` + LastValid basics.Round `codec:"lv"` + Note []byte `codec:"note"` + GenesisID string `codec:"gen"` + GenesisHash crypto.Digest `codec:"gh"` +} + +// inspectPaymentTxnFields is isomorphic to Header but uses different +// types to print public keys using algorand's address format in JSON. +type inspectPaymentTxnFields struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Receiver checksumAddress `codec:"rcv"` + Amount basics.MicroAlgos `codec:"amt"` + CloseRemainderTo checksumAddress `codec:"close"` +} + +// checksumAddress is a checksummed address, for use with text encodings +// like JSON. +type checksumAddress basics.Address + +// UnmarshalText implements the encoding.TextUnmarshaler interface +func (a *checksumAddress) UnmarshalText(text []byte) error { + addr, err := basics.UnmarshalChecksumAddress(string(text)) + if err != nil { + return err + } + + *a = checksumAddress(addr) + return nil +} + +// MarshalText implements the encoding.TextMarshaler interface +func (a checksumAddress) MarshalText() (text []byte, err error) { + addr := basics.Address(a) + return []byte(addr.GetChecksumAddress().String()), nil +} + func inspectTxn(stxn transactions.SignedTxn) (sti inspectSignedTxn, err error) { - sti = txnToInspect(stxn) - if !reflect.DeepEqual(stxn, txnFromInspect(sti)) { + sti = stxnToInspect(stxn) + if !reflect.DeepEqual(stxn, stxnFromInspect(sti)) { err = fmt.Errorf("non-idempotent transformation to inspectSignedTxn (DeepEqual)") + return } if !reflect.DeepEqual(protocol.Encode(sti), protocol.Encode(stxn)) { err = fmt.Errorf("non-idempotent transformation to inspectSignedTxn (protocol.Encode)") + return } return } -func txnToInspect(stxn transactions.SignedTxn) inspectSignedTxn { +func stxnToInspect(stxn transactions.SignedTxn) inspectSignedTxn { return inspectSignedTxn{ - Txn: stxn.Txn, + Txn: txnToInspect(stxn.Txn), Sig: stxn.Sig, Msig: msigToInspect(stxn.Msig), } } -func txnFromInspect(sti inspectSignedTxn) transactions.SignedTxn { +func stxnFromInspect(sti inspectSignedTxn) transactions.SignedTxn { return transactions.SignedTxn{ - Txn: sti.Txn, + Txn: txnFromInspect(sti.Txn), Sig: sti.Sig, Msig: msigFromInspect(sti.Msig), } @@ -92,7 +150,7 @@ func msigToInspect(msig crypto.MultisigSig) inspectMultisigSig { for _, subsig := range msig.Subsigs { res.Subsigs = append(res.Subsigs, inspectMultisigSubsig{ Sig: subsig.Sig, - Key: basics.Address(subsig.Key), + Key: checksumAddress(subsig.Key), }) } @@ -114,3 +172,45 @@ func msigFromInspect(msi inspectMultisigSig) crypto.MultisigSig { return res } + +func txnToInspect(txn transactions.Transaction) inspectTransaction { + return inspectTransaction{ + Type: txn.Type, + inspectTxnHeader: inspectTxnHeader{ + Sender: checksumAddress(txn.Sender), + Fee: txn.Fee, + FirstValid: txn.FirstValid, + LastValid: txn.LastValid, + Note: txn.Note, + GenesisID: txn.GenesisID, + GenesisHash: txn.GenesisHash, + }, + KeyregTxnFields: txn.KeyregTxnFields, + inspectPaymentTxnFields: inspectPaymentTxnFields{ + Receiver: checksumAddress(txn.Receiver), + Amount: txn.Amount, + CloseRemainderTo: checksumAddress(txn.CloseRemainderTo), + }, + } +} + +func txnFromInspect(txi inspectTransaction) transactions.Transaction { + return transactions.Transaction{ + Type: txi.Type, + Header: transactions.Header{ + Sender: basics.Address(txi.Sender), + Fee: txi.Fee, + FirstValid: txi.FirstValid, + LastValid: txi.LastValid, + Note: txi.Note, + GenesisID: txi.GenesisID, + GenesisHash: txi.GenesisHash, + }, + KeyregTxnFields: txi.KeyregTxnFields, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: basics.Address(txi.Receiver), + Amount: txi.Amount, + CloseRemainderTo: basics.Address(txi.CloseRemainderTo), + }, + } +} diff --git a/cmd/goal/inspect_test.go b/cmd/goal/inspect_test.go new file mode 100644 index 0000000000..66f243ce08 --- /dev/null +++ b/cmd/goal/inspect_test.go @@ -0,0 +1,89 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/protocol" +) + +func TestInspect(t *testing.T) { + var err error + + var empty transactions.SignedTxn + _, err = inspectTxn(empty) + require.NoError(t, err) + + var payment transactions.SignedTxn + crypto.RandBytes(payment.Sig[:]) + payment.Txn.Type = protocol.PaymentTx + crypto.RandBytes(payment.Txn.Sender[:]) + crypto.RandBytes(payment.Txn.Receiver[:]) + payment.Txn.Fee.Raw = crypto.RandUint64() + payment.Txn.Amount.Raw = crypto.RandUint64() + payment.Txn.FirstValid = basics.Round(crypto.RandUint64()) + payment.Txn.LastValid = basics.Round(crypto.RandUint64()) + _, err = inspectTxn(payment) + require.NoError(t, err) + + var keyreg transactions.SignedTxn + crypto.RandBytes(keyreg.Sig[:]) + keyreg.Txn.Type = protocol.KeyRegistrationTx + crypto.RandBytes(keyreg.Txn.Sender[:]) + keyreg.Txn.Fee.Raw = crypto.RandUint64() + keyreg.Txn.FirstValid = basics.Round(crypto.RandUint64()) + keyreg.Txn.LastValid = basics.Round(crypto.RandUint64()) + crypto.RandBytes(keyreg.Txn.VotePK[:]) + crypto.RandBytes(keyreg.Txn.SelectionPK[:]) + _, err = inspectTxn(keyreg) + require.NoError(t, err) + + var full transactions.SignedTxn + crypto.RandBytes(full.Sig[:]) + full.Msig.Version = uint8(crypto.RandUint64()) + full.Msig.Threshold = uint8(crypto.RandUint64()) + full.Msig.Subsigs = make([]crypto.MultisigSubsig, 2) + crypto.RandBytes(full.Msig.Subsigs[0].Key[:]) + crypto.RandBytes(full.Msig.Subsigs[0].Sig[:]) + crypto.RandBytes(full.Msig.Subsigs[1].Key[:]) + crypto.RandBytes(full.Msig.Subsigs[1].Sig[:]) + full.Txn.Type = protocol.UnknownTx + crypto.RandBytes(full.Txn.Sender[:]) + full.Txn.Fee.Raw = crypto.RandUint64() + full.Txn.FirstValid = basics.Round(crypto.RandUint64()) + full.Txn.LastValid = basics.Round(crypto.RandUint64()) + full.Txn.Note = make([]byte, 256) + crypto.RandBytes(full.Txn.Note[:]) + full.Txn.GenesisID = "testid" + crypto.RandBytes(full.Txn.GenesisHash[:]) + crypto.RandBytes(full.Txn.VotePK[:]) + crypto.RandBytes(full.Txn.SelectionPK[:]) + full.Txn.VoteFirst = basics.Round(crypto.RandUint64()) + full.Txn.VoteLast = basics.Round(crypto.RandUint64()) + full.Txn.VoteKeyDilution = crypto.RandUint64() + full.Txn.Amount.Raw = crypto.RandUint64() + crypto.RandBytes(full.Txn.Receiver[:]) + crypto.RandBytes(full.Txn.CloseRemainderTo[:]) + _, err = inspectTxn(full) + require.NoError(t, err) +} From 15708a62fd68ab9b7a5129f5f0c6e058ce041573 Mon Sep 17 00:00:00 2001 From: algoradam <37638838+algoradam@users.noreply.github.com> Date: Fri, 14 Jun 2019 20:59:28 +0000 Subject: [PATCH 18/54] Add .gitattributes file (#29) In particular, tell GitHub's language autodetection feature that crypto/libsodium-fork/* is vendored code. This way GitHub will show go-algorand as a Go project rather than a C project. See https://github.com/github/linguist#vendored-code --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..55d3330392 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +crypto/libsodium-fork/* linguist-vendored From 979da1a360ccc49bdfed7561e6b49488d13b832b Mon Sep 17 00:00:00 2001 From: Derek Leung Date: Fri, 14 Jun 2019 17:02:48 -0400 Subject: [PATCH 19/54] Support streaming in the coroner debug tool. (#33) This commit allows streaming cadaver files into the coroner debug tool. This improves the memory footprint of processing of large cadaver files, and it also allows a user or other tools to inspect the output of coroner as quickly as it is processed. This commit removes support for relative round bounds for trimming coroner, since end-relative round bounds are hard to compute while streaming. --- agreement/actor.go | 4 +- agreement/autopsy.go | 540 +++++++++++++++++++++++------------------- agreement/trace.go | 25 +- debug/coroner/main.go | 128 +++------- 4 files changed, 353 insertions(+), 344 deletions(-) diff --git a/agreement/actor.go b/agreement/actor.go index 195edb4d29..20f066c9da 100644 --- a/agreement/actor.go +++ b/agreement/actor.go @@ -135,11 +135,11 @@ type ioLoggedActor struct { func (l ioLoggedActor) handle(h routerHandle, e event) []action { if l.tracer.level >= top { - fmt.Printf("%23v => %23v: %v\n", "", l.T(), e) + fmt.Fprintf(l.tracer.w, "%23v => %23v: %v\n", "", l.T(), e) } a := l.checkedActor.handle(h, e) if l.tracer.level >= top { - fmt.Printf("%23v <= %23v: %v\n", "", l.T(), a) + fmt.Fprintf(l.tracer.w, "%23v <= %23v: %v\n", "", l.T(), a) } return a } diff --git a/agreement/autopsy.go b/agreement/autopsy.go index 81a3746130..549a916729 100644 --- a/agreement/autopsy.go +++ b/agreement/autopsy.go @@ -21,6 +21,7 @@ import ( "io" "os" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" ) @@ -33,44 +34,42 @@ import ( type Autopsy struct { io.Reader io.Closer -} -// PrepareAutopsyFromInputStream prepares an autopsy from std in. -func PrepareAutopsyFromInputStream() (*Autopsy, error) { - a := new(Autopsy) - a.Reader = os.Stdin - a.Closer = os.Stdin - return a, nil + cdvs <-chan cdvInstance } -type multiCloser struct { - closers []io.Closer +// AutopsyBounds defines the range of rounds and periods spanned by a single +// invocation of a cadaver-generating process. +type AutopsyBounds struct { + // Start and End are inclusive here. + StartRound uint64 + StartPeriod uint64 + EndRound uint64 + EndPeriod uint64 } -func (m *multiCloser) Close() error { - for _, c := range m.closers { - err := c.Close() - if err != nil { - return err - } - } - return nil -} - -// MultiCloser returns a Closer that closes all the given closers. -func MultiCloser(closers ...io.Closer) io.Closer { - r := make([]io.Closer, len(closers)) - copy(r, closers) - return &multiCloser{r} +// PrepareAutopsyFromStream prepares an autopsy from a given ReadCloser. +// +// nextBounds is called with a sequence number for each new invocation of a +// cadaver-generating process (a "run"). +// +// done is called with the total number of runs and any error encountered while +// performing the autopsy. +func PrepareAutopsyFromStream(stream io.ReadCloser, nextBounds func(int, AutopsyBounds), done func(int, error)) (*Autopsy, error) { + return prepareStreamingAutopsy(stream, stream, nextBounds, done), nil } // PrepareAutopsy prepares an autopsy from a cadaver filename. -func PrepareAutopsy(cadaverBaseFilename string) (*Autopsy, error) { +// +// nextBounds is called with a sequence number for each new invocation of a +// cadaver-generating process (a "run"). +// +// done is called with the total number of runs and any error encountered while +// performing the autopsy. +func PrepareAutopsy(cadaverBaseFilename string, nextBounds func(int, AutopsyBounds), done func(int, error)) (*Autopsy, error) { name0 := cadaverBaseFilename + ".archive" // read the archive file first name1 := cadaverBaseFilename - a := new(Autopsy) - in1, err := os.Open(name1) if err != nil { return nil, err @@ -79,272 +78,168 @@ func PrepareAutopsy(cadaverBaseFilename string) (*Autopsy, error) { if err != nil { if os.IsNotExist(err) { // only one file created - a.Reader = in1 - a.Closer = in1 - return a, nil + return prepareStreamingAutopsy(in1, in1, nextBounds, done), nil } return nil, err } - a.Reader = io.MultiReader(in0, in1) - a.Closer = MultiCloser(in0, in1) - return a, nil -} -// ExtractCdvs returns all the autopsied cadaver sequences contained in an autopsy. -func (a *Autopsy) ExtractCdvs() (seqs []AutopsiedCdv, reterr error) { - for { - s, _, err := a.ExtractNextCdv(nil) - if err != nil { - reterr = err - return - } - if s.Empty() { - return - } - seqs = append(seqs, s) - } + return prepareStreamingAutopsy(io.MultiReader(in0, in1), makeMultiCloser(in0, in1), nextBounds, done), nil } -// ExtractNextCdv extracts the next AutopsiedCdv from an Autopsy and calls the -// given callback every time it extracts a single AutopsyTrace. -// -// headSkipped indicates how many events were skipped from the head of -// the cadaver. -// -// AutopsiedCdv may be partial - that is, it may not have a metadata entry, esp. if the archive -// file was too big and overwritten. -// -// traces may be set if reterr != nil. -func (a *Autopsy) ExtractNextCdv(h func(AutopsyTrace) (bool, error)) (aCdv AutopsiedCdv, headSkipped int, reterr error) { - recording := false - var acc AutopsyTrace - var accs AutopsyTraceSeq - - var err error - defer func() { - if recording { - accs = append(accs, acc) - if h != nil { - _, reterr = h(acc) - } - } - if err != nil && err.Error() == "EOF" { - reterr = nil - } - aCdv.T = accs - }() +type multiCloser struct { + closers []io.Closer +} - for { // terminates automatically on EOF - var t cadaverEntryType - err = protocol.DecodeStream(a, &t) +func (m *multiCloser) Close() error { + for _, c := range m.closers { + err := c.Close() if err != nil { - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode cadaverEntryType: %v", err) - return + return err } + } + return nil +} - switch t { - case cadaverEOSEntry: - // if cadaver sequence, terminate. This indicates a crash, or any new cadaver process. - // else, no-op. - if recording { - return - } - case cadaverPlayerEntry: - if recording { - if len(acc.e) != len(acc.a) && len(acc.e) != len(acc.a)+1 { // last event may have resulted in a process failure - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: events do not align with actions: %d != %d (+1)", len(acc.e), len(acc.a)) - return - } - accs = append(accs, acc) - if h != nil { - keepGoing, err := h(acc) - if err != nil || !keepGoing { - reterr = err - return - } - } - } - recording = true +// makeMultiCloser returns a Closer that closes all the given closers. +func makeMultiCloser(closers ...io.Closer) io.Closer { + r := make([]io.Closer, len(closers)) + copy(r, closers) + return &multiCloser{r} +} - acc = AutopsyTrace{} - err = protocol.DecodeStream(a, &acc.x) - if err != nil { - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode player: %v", err) - return - } +type autopsyTrace struct { + x player + m CadaverMetadata - if len(accs) == 0 { - aCdv.StartRound = int64(acc.x.Round) - aCdv.StartPeriod = int64(acc.x.Period) - } - aCdv.EndRound = int64(acc.x.Round) - aCdv.EndPeriod = int64(acc.x.Period) - case cadaverEventEntry: - var et eventType - err = protocol.DecodeStream(a, &et) - if err != nil { - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode eventType: %v", err) - return - } + p <-chan autopsyPair +} - e := zeroEvent(et) - err = protocol.DecodeStream(a, &e) - if err != nil { - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode event: %v", err) - return - } +type cdvInstance <-chan autopsyTrace - if recording { - acc.e = append(acc.e, e) - } else { - headSkipped++ - } +func prepareStreamingAutopsy(r io.Reader, c io.Closer, nextBounds func(int, AutopsyBounds), done func(int, error)) *Autopsy { + a := new(Autopsy) + a.Reader = r + a.Closer = c - case cadaverActionEntry: - var n int - err = protocol.DecodeStream(a, &n) - if err != nil { - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode number of actions: %v", err) - } + ch := make(chan cdvInstance) + go func() { + defer func() { + close(ch) + }() - var as []action - for i := 0; i < n; i++ { - var at actionType - err = protocol.DecodeStream(a, &at) - if err != nil { - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode actionType: %v", err) - return - } + for n := 0; ; n++ { + tch := make(chan autopsyTrace) + ch <- tch - zA := zeroAction(at) - err = protocol.DecodeStream(a, &zA) - if err != nil { - fmt.Printf("Action type: %v\n", at.String()) - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode action: %v", err) - return - } + bounds, empty, err := a.extractNextCdv(tch) - as = append(as, zA) + if !empty { + nextBounds(n, bounds) } - if recording { - acc.a = append(acc.a, as) - } // headSkipped is accounted for already - - case cadaverMetaEntry: - // note that we can read multiple of these for a singe "cadaver seq" during normal operation if a sequence spans multiple - // files (due to fileTargetSize); the latest one gets printed (for now). - err = protocol.DecodeStream(a, &aCdv.M) if err != nil { - reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode meta entry sequence number: %v", err) + close(tch) + done(n, err) + return + } + if empty { + close(tch) + done(n, nil) return } } - } + }() + a.cdvs = ch + return a } -// An AutopsyTrace is the explict trace extracted from a cadaver -// file for a single (round, period) pair. -type AutopsyTrace struct { - x player - e []event - a [][]action +type switchableWriter struct { + io.Writer + disabled bool } -// Dump is another convenience function for live streaming autopsy -func (a AutopsyTrace) Dump() { - fmt.Printf("autopsy: player state is %+v ", a.x) - - for i := range a.e { - fmt.Printf("e: %v\n", a.e[i]) - fmt.Printf("actual: %v\n", a.a[i]) - } +func (w *switchableWriter) Enable() { + w.disabled = false } -// An AutopsyTraceSeq is a slice of traces extracted from the autopsy. -type AutopsyTraceSeq []AutopsyTrace - -// An AutopsiedCdv is an ordered slice of AutopsyTraces, corresponding to -// one contiguous cadaver log sequence (e.g. before crashing) -type AutopsiedCdv struct { - T AutopsyTraceSeq - M CadaverMetadata - StartRound int64 - StartPeriod int64 - EndRound int64 - EndPeriod int64 +func (w *switchableWriter) Disable() { + w.disabled = true } -// Empty returns true if AutopsiedCdv is empty (e.g. read from empty autopsy file) -func (seq AutopsiedCdv) Empty() bool { - return seq.T == nil +func (w switchableWriter) Write(p []byte) (n int, err error) { + if w.disabled { + return len(p), nil + } + return w.Writer.Write(p) } -// FilterBefore removes all traces smaller than the given round. Returns -// trimmed sequence and the first round of the first trace in the trimmed sequence. -func (seq AutopsyTraceSeq) FilterBefore(first int64) (AutopsyTraceSeq, int64) { - var nextFirstRound int64 - for i := range seq { - nextFirstRound = int64(seq[i].x.Round) - if nextFirstRound >= first { - // we want to keep seq[i] - return seq[i:], nextFirstRound - } - } - return nil, nextFirstRound +type switchableWriteCloser struct { + switchableWriter + io.Closer } -// FilterAfter removes all traces larger than the given round. Returns -// a trimmed seq and the last round of the last trace in the trimmed seq. -func (seq AutopsyTraceSeq) FilterAfter(last int64) (AutopsyTraceSeq, int64) { - var prevLastRound int64 - for i := range seq { - if int64(seq[i].x.Round) > last { - // discard seq[i] and everything after; return round of seq[i-1] - return seq[:i], prevLastRound - } - prevLastRound = int64(seq[i].x.Round) - } - // nothing to discard... - return seq[:], prevLastRound +// AutopsyFilter represents a window of rounds to be filtered from the autopsy +// output. +type AutopsyFilter struct { + Enabled bool // do not filter if this is false + First basics.Round // first round to emit output for; inclusive + Last basics.Round // last round to emit output for; inclusive } // DumpString dumps a textual representation of the AutopsyCdvs to the // given io.Writer. -func DumpString(cdvs []AutopsiedCdv, w io.Writer) { +func (a *Autopsy) DumpString(filter AutopsyFilter, w0 io.Writer) (version string) { + w := &switchableWriter{Writer: w0} var playerTracer tracer playerTracer.level = all playerTracer.log = serviceLogger{logging.Base()} + playerTracer.w = w var router rootRouter // TODO this could become inaccurate with orphaned events - for _, aCdv := range cdvs { - fmt.Fprintf(w, "autopsy: metadata: %v >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n", aCdv.M) + for cdv := range a.cdvs { + first := true + + for tr := range cdv { + if first { + first = false + fmt.Fprintf(w, "autopsy: metadata: %v >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n", tr.m) + version = tr.m.VersionCommitHash + } - for _, tr := range aCdv.T { player := tr.x + + if filter.Enabled { + if player.Round < filter.First || player.Round > filter.Last { + w.Disable() + } else { + w.Enable() + } + } + fmt.Fprintln(w, "autopsy:-") fmt.Fprintln(w, "autopsy:===================================") - DumpPlayerStr(w, player, router, "actual") + dumpPlayerStr(w, player, router, "actual") var p actor = ioLoggedActor{checkedActor{actor: &player, actorContract: playerContract{}}, playerTracer} router.root = p - for i, e := range tr.e { - player, _ = router.submitTop(&playerTracer, player, e) - if i == len(tr.a) { + for pair := range tr.p { + player, _ = router.submitTop(&playerTracer, player, pair.e) + if !pair.aok { break } - fmt.Fprintf(w, "actual: %v\n", tr.a[i]) + fmt.Fprintf(w, "actual: %v\n", pair.a) fmt.Fprintln(w, "autopsy:===================================") - DumpPlayerStr(w, player, router, "predicted") + dumpPlayerStr(w, player, router, "predicted") } } } + return } -// DumpPlayerStr prints useful state of the player, tagging the output with string -func DumpPlayerStr(w io.Writer, p player, r rootRouter, tag string) { +// dumpPlayerStr prints useful state of the player, tagging the output with string. +func dumpPlayerStr(w io.Writer, p player, r rootRouter, tag string) { playerCopy := p playerCopy.Pending = proposalTable{} fmt.Fprintf(w, "autopsy: (%s) player state is %+v (len(player.Pending = %d))\n", tag, playerCopy, len(p.Pending.Pending)) @@ -392,34 +287,197 @@ func DumpPlayerStr(w io.Writer, p player, r rootRouter, tag string) { // DumpMessagePack dumps a msgpack representation of the AutopsiedCdvs to the // given io.Writer. -func DumpMessagePack(cdvs []AutopsiedCdv, w io.WriteCloser) { +func (a *Autopsy) DumpMessagePack(filter AutopsyFilter, w0 io.WriteCloser) (version string) { + w := &switchableWriteCloser{switchableWriter: switchableWriter{Writer: w0}, Closer: w0} var playerTracer tracer playerTracer.log = serviceLogger{logging.Base()} + playerTracer.w = w var router rootRouter // TODO this could become inaccurate with orphaned events - for _, aCdv := range cdvs { + for cdv := range a.cdvs { + first := true + // reset cadaver for every cdv seq (so we don't miss caching player state) c := cadaver{} c.overrideSetup = true c.out = &cadaverHandle{WriteCloser: w} - protocol.EncodeStream(c.out, cadaverMetaEntry) - protocol.EncodeStream(c.out, aCdv.M) + for tr := range cdv { + if first { + first = false + protocol.EncodeStream(c.out, cadaverMetaEntry) + protocol.EncodeStream(c.out, tr.m) + version = tr.m.VersionCommitHash + } - for _, tr := range aCdv.T { player := tr.x var p actor = checkedActor{actor: &player, actorContract: playerContract{}} router.root = p - for i, e := range tr.e { - c.traceInput(player.Round, player.Period, player, tr.e[i]) - if i < len(tr.a) { - c.traceOutput(player.Round, player.Period, player, tr.a[i]) + if filter.Enabled { + if player.Round < filter.First || player.Round > filter.Last { + w.Disable() + } else { + w.Enable() + } + } + + for pair := range tr.p { + c.traceInput(player.Round, player.Period, player, pair.e) + if pair.aok { + c.traceOutput(player.Round, player.Period, player, pair.a) } - player, _ = router.submitTop(&playerTracer, player, e) + player, _ = router.submitTop(&playerTracer, player, pair.e) // TODO can check correspondence here } } protocol.EncodeStream(c.out, cadaverEOSEntry) } + return +} + +type autopsyPair struct { + e event + a []action + aok bool +} + +func (a *Autopsy) extractNextCdv(ch chan<- autopsyTrace) (bounds AutopsyBounds, empty bool, reterr error) { + empty = true + + recording := false + var acc autopsyTrace + + var pch chan autopsyPair + var err error + defer func() { + if recording { + empty = false + close(pch) + close(ch) + } + if err != nil && err.Error() == "EOF" { + reterr = nil + } + }() + + expectAction := false // if false, event is expected; else action + var accp autopsyPair + + for { // terminates automatically on EOF + var t cadaverEntryType + err = protocol.DecodeStream(a, &t) + if err != nil { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode cadaverEntryType: %v", err) + return + } + + switch t { + case cadaverEOSEntry: + // if cadaver sequence, terminate. This indicates a crash, or any new cadaver process. + // else, no-op. + if recording { + return + } + case cadaverPlayerEntry: + if recording { + empty = false + close(pch) + } + + pch = make(chan autopsyPair, 0) + acc = autopsyTrace{m: acc.m, p: pch} + err = protocol.DecodeStream(a, &acc.x) + if err != nil { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode player: %v", err) + return + } + expectAction = false + + bounds.EndRound = uint64(acc.x.Round) + bounds.EndPeriod = uint64(acc.x.Period) + + if !recording { + // first time + bounds.StartRound = uint64(acc.x.Round) + bounds.StartPeriod = uint64(acc.x.Period) + } + recording = true + + ch <- acc + + case cadaverEventEntry: + var et eventType + err = protocol.DecodeStream(a, &et) + if err != nil { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode eventType: %v", err) + return + } + + e := zeroEvent(et) + err = protocol.DecodeStream(a, &e) + if err != nil { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode event: %v", err) + return + } + + if recording { + if expectAction { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: expected action but got event") + return + } + accp.e = e + expectAction = !expectAction + } + + case cadaverActionEntry: + var n int + err = protocol.DecodeStream(a, &n) + if err != nil { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode number of actions: %v", err) + } + + var as []action + for i := 0; i < n; i++ { + var at actionType + err = protocol.DecodeStream(a, &at) + if err != nil { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode actionType: %v", err) + return + } + + zA := zeroAction(at) + err = protocol.DecodeStream(a, &zA) + if err != nil { + fmt.Printf("Action type: %v\n", at.String()) + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode action: %v", err) + return + } + + as = append(as, zA) + } + + if recording { + if !expectAction { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: expected event but got action") + return + } + accp.aok = true + accp.a = as + pch <- accp + + accp = autopsyPair{} + expectAction = !expectAction + } + + case cadaverMetaEntry: + // note that we can read multiple of these for a singe "cadaver seq" during normal operation if a sequence spans multiple + // files (due to fileTargetSize); the latest one gets printed (for now). + err = protocol.DecodeStream(a, &acc.m) + if err != nil { + reterr = fmt.Errorf("Autopsy.ExtractNextCdv: failed to decode meta entry sequence number: %v", err) + return + } + } + } } diff --git a/agreement/trace.go b/agreement/trace.go index f570b9cab8..92a1cd7d51 100644 --- a/agreement/trace.go +++ b/agreement/trace.go @@ -18,6 +18,8 @@ package agreement import ( "fmt" + "io" + "os" "strings" "time" @@ -51,6 +53,8 @@ type tracer struct { log serviceLogger + w io.Writer + // Tracer is now a little stateful (for ad-hoc logging) // parent state machines/routers are responsible for making sure tracer // picks up the right state. Optional. @@ -74,6 +78,7 @@ func makeTracer(log serviceLogger, cadaverFilename string, cadaverSizeTarget uin t.log = log t.verboseReports = verboseReportFlag t.timingReports = timingReportFlag + t.w = os.Stdout fileSizeTarget := int64(cadaverSizeTarget) if fileSizeTarget == 0 { @@ -126,21 +131,21 @@ func (t *tracer) setMetadata(metadata tracerMetadata) { func (t *tracer) ein(src, dest stateMachineTag, e event, r round, p period, s step) { t.seq++ if t.level >= all { - // fmt.Printf("%v %3v %23v -> %23v: %30v\n", t.tag, t.seq, src, dest, e) - fmt.Printf("%v] %23v -> %23v: %30v\n", t.tag, src, dest, e) + // fmt.Fprintf(t.w, "%v %3v %23v -> %23v: %30v\n", t.tag, t.seq, src, dest, e) + fmt.Fprintf(t.w, "%v] %23v -> %23v: %30v\n", t.tag, src, dest, e) } } func (t *tracer) eout(src, dest stateMachineTag, e event, r round, p period, s step) { t.seq++ if t.level >= all { - // fmt.Printf("%v %3v %23v <- %23v: %30v\n", t.tag, t.seq, src, dest, e) - fmt.Printf("%v] %23v <- %23v: %30v\n", t.tag, src, dest, e) + // fmt.Fprintf(t.w, "%v %3v %23v <- %23v: %30v\n", t.tag, t.seq, src, dest, e) + fmt.Fprintf(t.w, "%v] %23v <- %23v: %30v\n", t.tag, src, dest, e) } else if t.level >= key { switch e.t() { case proposalAccepted, proposalCommittable, softThreshold, certThreshold, nextThreshold: - // fmt.Printf("%v %3v %23v <- %23v: %30v\n", t.tag, t.seq, src, dest, e) - fmt.Printf("%v] %23v <- %23v: %30v\n", t.tag, src, dest, e) + // fmt.Fprintf(t.w, "%v %3v %23v <- %23v: %30v\n", t.tag, t.seq, src, dest, e) + fmt.Fprintf(t.w, "%v] %23v <- %23v: %30v\n", t.tag, src, dest, e) } } } @@ -148,8 +153,8 @@ func (t *tracer) eout(src, dest stateMachineTag, e event, r round, p period, s s func (t *tracer) ainTop(src, dest stateMachineTag, state player, e event, r round, p period, s step) { t.seq++ if t.level >= top { - // fmt.Printf("%v %3v %23v => %23v: %30v\n", t.tag, t.seq, src, dest, e) - fmt.Printf("%v] %23v => %23v: %30v\n", t.tag, src, dest, e) + // fmt.Fprintf(t.w, "%v %3v %23v => %23v: %30v\n", t.tag, t.seq, src, dest, e) + fmt.Fprintf(t.w, "%v] %23v => %23v: %30v\n", t.tag, src, dest, e) } } @@ -165,8 +170,8 @@ func (t *tracer) aoutTop(src, dest stateMachineTag, as []action, r round, p peri t.seq++ if t.level >= top { - // fmt.Printf("%v %3v %23v <= %23v: %.30v\n", t.tag, t.seq, src, dest, as) - fmt.Printf("%v] %23v <= %23v: %.30v\n", t.tag, src, dest, as) + // fmt.Fprintf(t.w, "%v %3v %23v <= %23v: %.30v\n", t.tag, t.seq, src, dest, as) + fmt.Fprintf(t.w, "%v] %23v <= %23v: %.30v\n", t.tag, src, dest, as) } } diff --git a/debug/coroner/main.go b/debug/coroner/main.go index 44b6b6d9c4..91db4f25d8 100644 --- a/debug/coroner/main.go +++ b/debug/coroner/main.go @@ -26,53 +26,45 @@ import ( "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" ) var numRegex = regexp.MustCompile(`^\d+$`) -var posOffRegex = regexp.MustCompile(`^\+\d+$`) -var negOffRegex = regexp.MustCompile(`^\-\d+$`) -var filename = flag.String("file", "", "Name of the input cadaver file") +var filename = flag.String("file", "", "Name of the input cadaver file (otherwise, use stdin)") var versionCheck = flag.Bool("version", false, "Display current coroner build version and exit") - var printmsgpack = flag.Bool("msgpack", false, "If provided, emit msgpack instead of a string") -// note: these also take relative offsets given by "+" or "-" symbols -// e.g., the command -// coroner --skip-head -10 -// will give the last 10 rounds of the coroner. -// If relative is set, the removal is done relative to the minimum round in the -// trace if the given round is nonnegative. Otherwise, the removal is relative -// to the maximum round in the trace. var skipHead = flag.String("skip-head", "", "The first round to trim before") var skipTail = flag.String("skip-tail", "", "The last round to trim after") -func mustParse(data []byte) int64 { - x, err := strconv.ParseInt(string(data), 10, 64) +func mustParse(data []byte) uint64 { + x, err := strconv.ParseUint(string(data), 10, 64) if err != nil { log.Fatalf(`failed to parse round bound in "%s": %s`, string(data), err) } return x } -func parseRoundBound(s string) (bound int64, relative bool) { - data := []byte(s) - signfact := int64(1) - - switch { - case s == "": - case numRegex.Match(data): - bound = mustParse(numRegex.Find(data)) - case negOffRegex.Match(data): - signfact = -1 - fallthrough - case posOffRegex.Match(data): - relative = true - bound = mustParse(numRegex.Find(data[1:])) * signfact - default: - log.Fatalf(`failed to parse round bound in "%s": string does not match regex "^(+|-)\d+$"`, s) +func parseRoundBound(s string) uint64 { + if !numRegex.Match([]byte(s)) { + log.Fatalf(`failed to parse round bound in "%s": string does not match regex "^\d+$"`, s) + } + return mustParse(numRegex.Find([]byte(s))) +} + +func done(n int, err error) { + if n == 0 { + log.Println("coroner: no cadavers autopsied") + } + + if err != nil { + log.Println("coroner: failed to extract full autopsy trace:", err) } - return +} + +func nextBounds(i int, bounds agreement.AutopsyBounds) { + log.Printf("cadaver seq: %d\tstart(r,p): (%d,%d)\tend(r,p): (%d,%d)\n", i, bounds.StartRound, bounds.StartPeriod, bounds.EndRound, bounds.EndPeriod) } func main() { @@ -89,80 +81,34 @@ func main() { if *filename == "" { log.Println("coroner: no filename provided; reading from stdin...") - autopsy, err = agreement.PrepareAutopsyFromInputStream() + autopsy, err = agreement.PrepareAutopsyFromStream(os.Stdin, nextBounds, done) } else { - autopsy, err = agreement.PrepareAutopsy(*filename) + autopsy, err = agreement.PrepareAutopsy(*filename, nextBounds, done) } if err != nil { log.Fatalln("coroner: failed to prepare autopsy:", err) } defer autopsy.Close() - headRound, headRelative := parseRoundBound(*skipHead) - tailRound, tailRelative := parseRoundBound(*skipTail) - - autopsiedCdvs, err := autopsy.ExtractCdvs() - if err != nil { - log.Println("coroner: failed to extract full autopsy trace:", err) - log.Println("coroner: continuing after error...") - } - - if len(autopsiedCdvs) < 1 { - log.Println("coroner: no cadavers autopsied") - return - } - - firstMeta := autopsiedCdvs[0].M - log.Printf("coroner: Cadaver file generated with commit hash:\n%s\n", firstMeta.VersionCommitHash) - if firstMeta.VersionCommitHash != version.GetCommitHash() { - log.Printf("coroner: Cadaver version mismatches coroner version:\n(%s (cadaver) != %s (coroner))\n", firstMeta.VersionCommitHash, version.GetCommitHash()) - } - - cachedStartRound := autopsiedCdvs[0].StartRound + var filter agreement.AutopsyFilter if *skipHead != "" { - numCdvs := len(autopsiedCdvs) - first := headRound - if headRelative { - if headRound >= 0 { - first = cachedStartRound + headRound - } else { - first = autopsiedCdvs[numCdvs-1].EndRound + headRound - } - } - for i := range autopsiedCdvs { - if autopsiedCdvs[i].EndRound < first { - autopsiedCdvs = autopsiedCdvs[i+1:] - break - } - } - autopsiedCdvs[0].T, autopsiedCdvs[0].StartRound = autopsiedCdvs[0].T.FilterBefore(first) + filter.Enabled = true + filter.First = basics.Round(parseRoundBound(*skipHead)) } if *skipTail != "" { - numCdvs := len(autopsiedCdvs) - last := tailRound - if tailRelative { - if tailRound >= 0 { - last = cachedStartRound + tailRound - } else { - last = autopsiedCdvs[numCdvs-1].EndRound + tailRound - } - } - for i := range autopsiedCdvs { - if autopsiedCdvs[i].StartRound > last { - autopsiedCdvs = autopsiedCdvs[:i] - break - } - } - end := len(autopsiedCdvs) - autopsiedCdvs[end].T, autopsiedCdvs[end].EndRound = autopsiedCdvs[end].T.FilterAfter(last) + filter.Enabled = true + filter.Last = basics.Round(parseRoundBound(*skipTail)) } - for i := range autopsiedCdvs { - log.Printf("Cadaver Seq: %d\tstart: %d\tend: %d\n", i, autopsiedCdvs[i].StartRound, autopsiedCdvs[i].EndRound) - } + var commitHash string if *printmsgpack { - agreement.DumpMessagePack(autopsiedCdvs, os.Stdout) + commitHash = autopsy.DumpMessagePack(filter, os.Stdout) } else { - agreement.DumpString(autopsiedCdvs, os.Stdout) + commitHash = autopsy.DumpString(filter, os.Stdout) } + if commitHash != version.GetCommitHash() { + log.Printf("coroner: cadaver version mismatches coroner version:\n(%s (cadaver) != %s (coroner))\n", commitHash, version.GetCommitHash()) + } + + return } From 02ae086576ab8b6d11ba457f297e794097543b53 Mon Sep 17 00:00:00 2001 From: David Shoots Date: Sat, 15 Jun 2019 07:49:33 -0400 Subject: [PATCH 20/54] Tools for working with relay registration --- cmd/algorelay/commands.go | 45 ++++++ cmd/algorelay/dnsCmd.go | 293 ++++++++++++++++++++++++++++++++++++++ cmd/algorelay/eb.go | 29 ++++ 3 files changed, 367 insertions(+) create mode 100644 cmd/algorelay/commands.go create mode 100644 cmd/algorelay/dnsCmd.go create mode 100644 cmd/algorelay/eb.go diff --git a/cmd/algorelay/commands.go b/cmd/algorelay/commands.go new file mode 100644 index 0000000000..dad5cadf14 --- /dev/null +++ b/cmd/algorelay/commands.go @@ -0,0 +1,45 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func init() { +} + +var rootCmd = &cobra.Command{ + Use: "algorelay", + Short: "algorelay", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + // If no arguments passed, we should fallback to help + + cmd.HelpFunc()(cmd, args) + }, +} + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/cmd/algorelay/dnsCmd.go b/cmd/algorelay/dnsCmd.go new file mode 100644 index 0000000000..f9f41b684f --- /dev/null +++ b/cmd/algorelay/dnsCmd.go @@ -0,0 +1,293 @@ +package main + +import ( + "fmt" + "github.com/algorand/go-algorand/util/codecs" + "github.com/spf13/cobra" + "os" +) + +var ( + inputFile string +) + +func init() { + rootCmd.AddCommand(checkCmd) + + checkCmd.Flags().StringVarP(&inputFile, "inputfile", "i", "", "File containing Relay data") + checkCmd.MarkFlagRequired("inputfile") + + //dnsCmd.AddCommand(checkCmd) + //dnsCmd.AddCommand(addCmd) + //dnsCmd.AddCommand(deleteCmd) + //dnsCmd.AddCommand(listCmd) + // + //addCmd.Flags().StringVarP(&addFromName, "from", "f", "", "From name to add new DNS entry") + //addCmd.MarkFlagRequired("from") + //addCmd.Flags().StringVarP(&addToAddress, "to", "t", "", "To address to map new DNS entry to") + //addCmd.MarkFlagRequired("to") + // + //deleteCmd.Flags().StringVarP(&deleteNetwork, "network", "n", "", "Network name for records to delete") + //deleteCmd.MarkFlagRequired("network") + //deleteCmd.Flags().BoolVarP(&noPrompt, "no-prompt", "y", false, "No prompting for records deletion") + //deleteCmd.Flags().StringVarP(&excludePattern, "exclude", "e", "", "name records exclude pattern") + // + //listCmd.Flags().StringVarP(&listNetwork, "network", "n", "", "Domain name for records to list") + //listCmd.Flags().StringVarP(&recordType, "recordType", "t", "", "DNS record type to list (A, CNAME, SRV)") + //listCmd.MarkFlagRequired("network") +} + +//type byIP []net.IP +// +//func (a byIP) Len() int { return len(a) } +//func (a byIP) Less(i, j int) bool { return a[i].String() < a[j].String() } +//func (a byIP) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +func loadRelays(file string) []Relay { + var relays []Relay + err := codecs.LoadObjectFromFile(inputFile, &relays) + if err != nil { + panic(err) + } + return relays +} + +var checkCmd = &cobra.Command{ + Use: "check", + Short: "Check status of all relays", + Run: func(cmd *cobra.Command, args []string) { + relays := loadRelays(inputFile) + + for _, relay := range relays { + err := verifyRelayStatus(relay) + if err != nil { + fmt.Fprintf(os.Stderr, "Error verifying relay status %s: %s\n", relay.IPOrDNSName, err) + } + } + }, +} + +func verifyRelayStatus(relay Relay) error { + +} + +//var addCmd = &cobra.Command{ +// Use: "add", +// Short: "Add a DNS record", +// Long: "Adds a DNS record to map --from to --to, using A if to == IP or CNAME otherwise\n", +// Example: "algons dns add -f a.test.algodev.network -t r1.algodev.network\n" + +// "algons dns add -f a.test.algodev.network -t 192.168.100.10", +// Run: func(cmd *cobra.Command, args []string) { +// err := doAddDNS(addFromName, addToAddress) +// if err != nil { +// fmt.Fprintf(os.Stderr, "Error adding DNS entry: %v\n", err) +// os.Exit(1) +// } else { +// fmt.Printf("DNS Entry Added\n") +// } +// }, +//} +// +//var deleteCmd = &cobra.Command{ +// Use: "delete", +// Short: "Delete DNS and SRV records for a specified network", +// Run: func(cmd *cobra.Command, args []string) { +// if !doDeleteDNS(deleteNetwork, noPrompt, excludePattern) { +// os.Exit(1) +// } +// }, +//} +// +//func doAddDNS(from string, to string) (err error) { +// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() +// if err != nil { +// return fmt.Errorf("error getting DNS credentials: %v", err) +// } +// +// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) +// +// const priority = 1 +// const proxied = false +// +// // If we need to register anything, first register a DNS entry +// // to map our network DNS name to our public name (or IP) provided to nodecfg +// // Network HostName = eg r1.testnet.algorand.network +// isIP := net.ParseIP(to) != nil +// var recordType string +// if isIP { +// recordType = "A" +// } else { +// recordType = "CNAME" +// } +// cloudflareDNS.SetDNSRecord(context.Background(), recordType, from, to, cloudflare.AutomaticTTL, priority, proxied) +// +// return +//} +// +//func getClouldflareCredentials() (zoneID string, email string, authKey string, err error) { +// zoneID = os.Getenv("CLOUDFLARE_ZONE_ID") +// email = os.Getenv("CLOUDFLARE_EMAIL") +// authKey = os.Getenv("CLOUDFLARE_AUTH_KEY") +// if zoneID == "" || email == "" || authKey == "" { +// err = fmt.Errorf("one or more credentials missing from ENV") +// } +// return +//} +// +//func checkDNSRecord(dnsName string) { +// fmt.Printf("------------------------\nDNS Lookup: %s\n", dnsName) +// ips, err := net.LookupIP(dnsName) +// if err != nil { +// fmt.Printf("Cannot resolve %s: %v\n", dnsName, err) +// } else { +// sort.Sort(byIP(ips)) +// for _, ip := range ips { +// fmt.Printf("-> %s\n", ip.String()) +// } +// } +//} +// +//func checkSrvRecord(dnsBootstrap string) { +// fmt.Printf("------------------------\nSRV Lookup: %s\n", dnsBootstrap) +// +// _, addrs, err := net.LookupSRV("algobootstrap", "tcp", dnsBootstrap) +// if err != nil { +// if !strings.HasSuffix(err.Error(), "cannot unmarshal DNS message") { +// // we weren't able to get the SRV records. +// fmt.Printf("Cannot lookup SRV record for %s: %v\n", dnsBootstrap, err) +// return +// } +// +// var resolver network.Resolver +// _, addrs, err = resolver.LookupSRV(context.Background(), "algobootstrap", "tcp", dnsBootstrap) +// if err != nil { +// fmt.Printf("Cannot lookup SRV record for %s via neither default resolver nor via %s: %v\n", dnsBootstrap, resolver.EffectiveResolverDNS(), err) +// return +// } +// } +// +// for _, srv := range addrs { +// fmt.Printf("%s:%d\n", srv.Target, srv.Port) +// } +//} +// +//func doDeleteDNS(network string, noPrompt bool, excludePattern string) bool { +// +// if network == "" || network == "testnet" || network == "devnet" || network == "mainnet" { +// fmt.Fprintf(os.Stderr, "Deletion of network '%s' using this tool is not allowed\n", network) +// return false +// } +// +// var excludeRegex *regexp.Regexp +// if excludePattern != "" { +// var err error +// excludeRegex, err = regexp.Compile(excludePattern) +// if err != nil { +// fmt.Fprintf(os.Stderr, "specified regular expression exclude pattern ('%s') is not a valid regular expression : %v", excludePattern, err) +// return false +// } +// } +// +// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() +// if err != nil { +// fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) +// return false +// } +// +// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) +// +// idsToDelete := make(map[string]string) // Maps record ID to Name +// +// for _, service := range []string{"_algobootstrap", "_metrics"} { +// records, err := cloudflareDNS.ListDNSRecord(context.Background(), "SRV", service+"._tcp."+network+".algodev.network", "", "", "", "") +// if err != nil { +// fmt.Fprintf(os.Stderr, "Error listing SRV '%s' entries: %v\n", service, err) +// os.Exit(1) +// } +// for _, r := range records { +// if excludeRegex != nil { +// if excludeRegex.MatchString(r.Name) { +// fmt.Printf("Excluding SRV '%s' record: %s\n", service, r.Name) +// continue +// } +// } +// fmt.Printf("Found SRV '%s' record: %s\n", service, r.Name) +// idsToDelete[r.ID] = r.Name +// } +// } +// +// networkSuffix := "." + network + ".algodev.network" +// +// for _, recordType := range []string{"A", "CNAME"} { +// records, err := cloudflareDNS.ListDNSRecord(context.Background(), recordType, "", "", "", "", "") +// if err != nil { +// fmt.Fprintf(os.Stderr, "Error listing DNS '%s' entries: %v\n", recordType, err) +// os.Exit(1) +// } +// for _, r := range records { +// if strings.Index(r.Name, networkSuffix) > 0 { +// if excludeRegex != nil { +// if excludeRegex.MatchString(r.Name) { +// fmt.Printf("Excluding DNS '%s' record: %s\n", recordType, r.Name) +// continue +// } +// } +// fmt.Printf("Found DNS '%s' record: %s\n", recordType, r.Name) +// idsToDelete[r.ID] = r.Name +// } +// } +// } +// +// if len(idsToDelete) == 0 { +// fmt.Printf("No DNS/SRV records found\n") +// return true +// } +// +// var text string +// if !noPrompt { +// reader := bufio.NewReader(os.Stdin) +// fmt.Printf("Delete these %d entries (type 'yes' to delete)? ", len(idsToDelete)) +// text, _ = reader.ReadString('\n') +// text = strings.Replace(text, "\n", "", -1) +// } else { +// text = "yes" +// } +// +// if text == "yes" { +// for id, name := range idsToDelete { +// fmt.Fprintf(os.Stdout, "Deleting %s\n", name) +// err = cloudflareDNS.DeleteDNSRecord(context.Background(), id) +// if err != nil { +// fmt.Fprintf(os.Stderr, " !! error deleting %s: %v\n", name, err) +// } +// } +// } +// return true +//} +// +//func listEntries(listNetwork string, recordType string) { +// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() +// if err != nil { +// fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) +// return +// } +// +// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) +// recordTypes := []string{"A", "CNAME", "SRV"} +// if recordType != "" { +// recordTypes = []string{recordType} +// } +// for _, recType := range recordTypes { +// records, err := cloudflareDNS.ListDNSRecord(context.Background(), recType, "", "", "", "", "") +// if err != nil { +// fmt.Fprintf(os.Stderr, "Error listing DNS entries: %v\n", err) +// os.Exit(1) +// } +// +// for _, record := range records { +// if strings.HasSuffix(record.Name, listNetwork) { +// fmt.Printf("%v\n", record.Name) +// } +// } +// } +//} diff --git a/cmd/algorelay/eb.go b/cmd/algorelay/eb.go new file mode 100644 index 0000000000..a62b7fce37 --- /dev/null +++ b/cmd/algorelay/eb.go @@ -0,0 +1,29 @@ +package main + +import "time" + +const NodeRunnerKind = "NodeRunner" + +// NodeRunner entities are synchronized directly from db1 using `afdb1 syncebnoderunners` +// They should not be modified manually +type NodeRunner struct { + CompanyName string + InvestorID string + NbRequiredRelays int // number of relays required relays the investor needs to run in the node agreement + NodeRunnerToken string // token used by eb.algorand.foundation +} + +const RelayKind = "Relay" + +type Relay struct { + InvestorID string + ContactEmail string // comma separated list of emails + NodeProvider string + IPOrDNSName string + Specs string + Notes string + SubmissionTime time.Time + SRVRecordCreationTime time.Time + Telemetry string // GUID[:name] + MetricsEnabled bool +} From a9f0b8f60907bfefc1b7f1871d563c511a1a32a7 Mon Sep 17 00:00:00 2001 From: David Shoots Date: Sat, 15 Jun 2019 15:17:05 -0400 Subject: [PATCH 21/54] Completed CHECK functionality --- cmd/algorelay/dnsCmd.go | 293 ---------------------- cmd/algorelay/{ => eb}/eb.go | 7 +- cmd/algorelay/relayCmd.go | 474 +++++++++++++++++++++++++++++++++++ 3 files changed, 480 insertions(+), 294 deletions(-) delete mode 100644 cmd/algorelay/dnsCmd.go rename cmd/algorelay/{ => eb}/eb.go (72%) create mode 100644 cmd/algorelay/relayCmd.go diff --git a/cmd/algorelay/dnsCmd.go b/cmd/algorelay/dnsCmd.go deleted file mode 100644 index f9f41b684f..0000000000 --- a/cmd/algorelay/dnsCmd.go +++ /dev/null @@ -1,293 +0,0 @@ -package main - -import ( - "fmt" - "github.com/algorand/go-algorand/util/codecs" - "github.com/spf13/cobra" - "os" -) - -var ( - inputFile string -) - -func init() { - rootCmd.AddCommand(checkCmd) - - checkCmd.Flags().StringVarP(&inputFile, "inputfile", "i", "", "File containing Relay data") - checkCmd.MarkFlagRequired("inputfile") - - //dnsCmd.AddCommand(checkCmd) - //dnsCmd.AddCommand(addCmd) - //dnsCmd.AddCommand(deleteCmd) - //dnsCmd.AddCommand(listCmd) - // - //addCmd.Flags().StringVarP(&addFromName, "from", "f", "", "From name to add new DNS entry") - //addCmd.MarkFlagRequired("from") - //addCmd.Flags().StringVarP(&addToAddress, "to", "t", "", "To address to map new DNS entry to") - //addCmd.MarkFlagRequired("to") - // - //deleteCmd.Flags().StringVarP(&deleteNetwork, "network", "n", "", "Network name for records to delete") - //deleteCmd.MarkFlagRequired("network") - //deleteCmd.Flags().BoolVarP(&noPrompt, "no-prompt", "y", false, "No prompting for records deletion") - //deleteCmd.Flags().StringVarP(&excludePattern, "exclude", "e", "", "name records exclude pattern") - // - //listCmd.Flags().StringVarP(&listNetwork, "network", "n", "", "Domain name for records to list") - //listCmd.Flags().StringVarP(&recordType, "recordType", "t", "", "DNS record type to list (A, CNAME, SRV)") - //listCmd.MarkFlagRequired("network") -} - -//type byIP []net.IP -// -//func (a byIP) Len() int { return len(a) } -//func (a byIP) Less(i, j int) bool { return a[i].String() < a[j].String() } -//func (a byIP) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -func loadRelays(file string) []Relay { - var relays []Relay - err := codecs.LoadObjectFromFile(inputFile, &relays) - if err != nil { - panic(err) - } - return relays -} - -var checkCmd = &cobra.Command{ - Use: "check", - Short: "Check status of all relays", - Run: func(cmd *cobra.Command, args []string) { - relays := loadRelays(inputFile) - - for _, relay := range relays { - err := verifyRelayStatus(relay) - if err != nil { - fmt.Fprintf(os.Stderr, "Error verifying relay status %s: %s\n", relay.IPOrDNSName, err) - } - } - }, -} - -func verifyRelayStatus(relay Relay) error { - -} - -//var addCmd = &cobra.Command{ -// Use: "add", -// Short: "Add a DNS record", -// Long: "Adds a DNS record to map --from to --to, using A if to == IP or CNAME otherwise\n", -// Example: "algons dns add -f a.test.algodev.network -t r1.algodev.network\n" + -// "algons dns add -f a.test.algodev.network -t 192.168.100.10", -// Run: func(cmd *cobra.Command, args []string) { -// err := doAddDNS(addFromName, addToAddress) -// if err != nil { -// fmt.Fprintf(os.Stderr, "Error adding DNS entry: %v\n", err) -// os.Exit(1) -// } else { -// fmt.Printf("DNS Entry Added\n") -// } -// }, -//} -// -//var deleteCmd = &cobra.Command{ -// Use: "delete", -// Short: "Delete DNS and SRV records for a specified network", -// Run: func(cmd *cobra.Command, args []string) { -// if !doDeleteDNS(deleteNetwork, noPrompt, excludePattern) { -// os.Exit(1) -// } -// }, -//} -// -//func doAddDNS(from string, to string) (err error) { -// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() -// if err != nil { -// return fmt.Errorf("error getting DNS credentials: %v", err) -// } -// -// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) -// -// const priority = 1 -// const proxied = false -// -// // If we need to register anything, first register a DNS entry -// // to map our network DNS name to our public name (or IP) provided to nodecfg -// // Network HostName = eg r1.testnet.algorand.network -// isIP := net.ParseIP(to) != nil -// var recordType string -// if isIP { -// recordType = "A" -// } else { -// recordType = "CNAME" -// } -// cloudflareDNS.SetDNSRecord(context.Background(), recordType, from, to, cloudflare.AutomaticTTL, priority, proxied) -// -// return -//} -// -//func getClouldflareCredentials() (zoneID string, email string, authKey string, err error) { -// zoneID = os.Getenv("CLOUDFLARE_ZONE_ID") -// email = os.Getenv("CLOUDFLARE_EMAIL") -// authKey = os.Getenv("CLOUDFLARE_AUTH_KEY") -// if zoneID == "" || email == "" || authKey == "" { -// err = fmt.Errorf("one or more credentials missing from ENV") -// } -// return -//} -// -//func checkDNSRecord(dnsName string) { -// fmt.Printf("------------------------\nDNS Lookup: %s\n", dnsName) -// ips, err := net.LookupIP(dnsName) -// if err != nil { -// fmt.Printf("Cannot resolve %s: %v\n", dnsName, err) -// } else { -// sort.Sort(byIP(ips)) -// for _, ip := range ips { -// fmt.Printf("-> %s\n", ip.String()) -// } -// } -//} -// -//func checkSrvRecord(dnsBootstrap string) { -// fmt.Printf("------------------------\nSRV Lookup: %s\n", dnsBootstrap) -// -// _, addrs, err := net.LookupSRV("algobootstrap", "tcp", dnsBootstrap) -// if err != nil { -// if !strings.HasSuffix(err.Error(), "cannot unmarshal DNS message") { -// // we weren't able to get the SRV records. -// fmt.Printf("Cannot lookup SRV record for %s: %v\n", dnsBootstrap, err) -// return -// } -// -// var resolver network.Resolver -// _, addrs, err = resolver.LookupSRV(context.Background(), "algobootstrap", "tcp", dnsBootstrap) -// if err != nil { -// fmt.Printf("Cannot lookup SRV record for %s via neither default resolver nor via %s: %v\n", dnsBootstrap, resolver.EffectiveResolverDNS(), err) -// return -// } -// } -// -// for _, srv := range addrs { -// fmt.Printf("%s:%d\n", srv.Target, srv.Port) -// } -//} -// -//func doDeleteDNS(network string, noPrompt bool, excludePattern string) bool { -// -// if network == "" || network == "testnet" || network == "devnet" || network == "mainnet" { -// fmt.Fprintf(os.Stderr, "Deletion of network '%s' using this tool is not allowed\n", network) -// return false -// } -// -// var excludeRegex *regexp.Regexp -// if excludePattern != "" { -// var err error -// excludeRegex, err = regexp.Compile(excludePattern) -// if err != nil { -// fmt.Fprintf(os.Stderr, "specified regular expression exclude pattern ('%s') is not a valid regular expression : %v", excludePattern, err) -// return false -// } -// } -// -// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() -// if err != nil { -// fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) -// return false -// } -// -// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) -// -// idsToDelete := make(map[string]string) // Maps record ID to Name -// -// for _, service := range []string{"_algobootstrap", "_metrics"} { -// records, err := cloudflareDNS.ListDNSRecord(context.Background(), "SRV", service+"._tcp."+network+".algodev.network", "", "", "", "") -// if err != nil { -// fmt.Fprintf(os.Stderr, "Error listing SRV '%s' entries: %v\n", service, err) -// os.Exit(1) -// } -// for _, r := range records { -// if excludeRegex != nil { -// if excludeRegex.MatchString(r.Name) { -// fmt.Printf("Excluding SRV '%s' record: %s\n", service, r.Name) -// continue -// } -// } -// fmt.Printf("Found SRV '%s' record: %s\n", service, r.Name) -// idsToDelete[r.ID] = r.Name -// } -// } -// -// networkSuffix := "." + network + ".algodev.network" -// -// for _, recordType := range []string{"A", "CNAME"} { -// records, err := cloudflareDNS.ListDNSRecord(context.Background(), recordType, "", "", "", "", "") -// if err != nil { -// fmt.Fprintf(os.Stderr, "Error listing DNS '%s' entries: %v\n", recordType, err) -// os.Exit(1) -// } -// for _, r := range records { -// if strings.Index(r.Name, networkSuffix) > 0 { -// if excludeRegex != nil { -// if excludeRegex.MatchString(r.Name) { -// fmt.Printf("Excluding DNS '%s' record: %s\n", recordType, r.Name) -// continue -// } -// } -// fmt.Printf("Found DNS '%s' record: %s\n", recordType, r.Name) -// idsToDelete[r.ID] = r.Name -// } -// } -// } -// -// if len(idsToDelete) == 0 { -// fmt.Printf("No DNS/SRV records found\n") -// return true -// } -// -// var text string -// if !noPrompt { -// reader := bufio.NewReader(os.Stdin) -// fmt.Printf("Delete these %d entries (type 'yes' to delete)? ", len(idsToDelete)) -// text, _ = reader.ReadString('\n') -// text = strings.Replace(text, "\n", "", -1) -// } else { -// text = "yes" -// } -// -// if text == "yes" { -// for id, name := range idsToDelete { -// fmt.Fprintf(os.Stdout, "Deleting %s\n", name) -// err = cloudflareDNS.DeleteDNSRecord(context.Background(), id) -// if err != nil { -// fmt.Fprintf(os.Stderr, " !! error deleting %s: %v\n", name, err) -// } -// } -// } -// return true -//} -// -//func listEntries(listNetwork string, recordType string) { -// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() -// if err != nil { -// fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) -// return -// } -// -// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) -// recordTypes := []string{"A", "CNAME", "SRV"} -// if recordType != "" { -// recordTypes = []string{recordType} -// } -// for _, recType := range recordTypes { -// records, err := cloudflareDNS.ListDNSRecord(context.Background(), recType, "", "", "", "", "") -// if err != nil { -// fmt.Fprintf(os.Stderr, "Error listing DNS entries: %v\n", err) -// os.Exit(1) -// } -// -// for _, record := range records { -// if strings.HasSuffix(record.Name, listNetwork) { -// fmt.Printf("%v\n", record.Name) -// } -// } -// } -//} diff --git a/cmd/algorelay/eb.go b/cmd/algorelay/eb/eb.go similarity index 72% rename from cmd/algorelay/eb.go rename to cmd/algorelay/eb/eb.go index a62b7fce37..413b6b64a3 100644 --- a/cmd/algorelay/eb.go +++ b/cmd/algorelay/eb/eb.go @@ -1,4 +1,4 @@ -package main +package eb import "time" @@ -16,6 +16,7 @@ type NodeRunner struct { const RelayKind = "Relay" type Relay struct { + ID int64 // db key injected when loaded InvestorID string ContactEmail string // comma separated list of emails NodeProvider string @@ -26,4 +27,8 @@ type Relay struct { SRVRecordCreationTime time.Time Telemetry string // GUID[:name] MetricsEnabled bool + CheckTime time.Time // time the check was done + CheckSuccess bool // true if check was successful + CheckError string // non-empty if check error, contains the error + DNSAlias string // DNS Alias name used } diff --git a/cmd/algorelay/relayCmd.go b/cmd/algorelay/relayCmd.go new file mode 100644 index 0000000000..cb8d439d20 --- /dev/null +++ b/cmd/algorelay/relayCmd.go @@ -0,0 +1,474 @@ +package main + +import ( + "context" + "fmt" + "github.com/algorand/go-algorand/cmd/algorelay/eb" + "github.com/algorand/go-algorand/tools/network/cloudflare" + "github.com/algorand/go-algorand/util/codecs" + "github.com/spf13/cobra" + "net" + "os" + "strconv" + "strings" +) + +var ( + inputFileArg string + outputFileArg string + srvDomainArg string + nameDomainArg string + defaultPortArg uint16 + dnsBootstrapArg string + recordIDArg int64 + + cfEmail string + cfAuthKey string + cfSrvZoneID string + cfNameZoneID string +) + +var nameRecordTypes = []string{"A", "CNAME", "SRV"} +var srvRecordTypes = []string{"SRV"} + +const metricsPort = uint16(9100) + +func init() { + cfSrvZoneID = os.Getenv("CLOUDFLARE_SRV_ZONE_ID") + cfNameZoneID = os.Getenv("CLOUDFLARE_NAME_ZONE_ID") + cfEmail = os.Getenv("CLOUDFLARE_EMAIL") + cfAuthKey = os.Getenv("CLOUDFLARE_AUTH_KEY") + if cfSrvZoneID == "" || cfNameZoneID == "" || cfEmail == "" || cfAuthKey == "" { + panic("One or more credentials missing from ENV") + } + + rootCmd.AddCommand(checkCmd) + + checkCmd.Flags().StringVarP(&inputFileArg, "inputfile", "i", "", "File containing Relay data") + checkCmd.MarkFlagRequired("inputfile") + + checkCmd.Flags().StringVarP(&outputFileArg, "outputfile", "o", "", "File to output results to, as JSON") + + checkCmd.Flags().Int64Var(&recordIDArg, "id", 0, "Specific Datastore record ID to check (all if not specified)") + + checkCmd.Flags().StringVarP(&srvDomainArg, "srvdomain", "s", "", "Domain name for SRV records") + checkCmd.MarkFlagRequired("srvdomain") + checkCmd.Flags().StringVarP(&nameDomainArg, "namedomain", "n", "", "Domain name for A/CNAME records") + checkCmd.MarkFlagRequired("namedomain") + checkCmd.Flags().Uint16VarP(&defaultPortArg, "defaultport", "p", 4160, "Default listening port (eg 4160)") + checkCmd.MarkFlagRequired("defaultport") + checkCmd.Flags().StringVarP(&dnsBootstrapArg, "dnsbootstrap", "b", "", "Bootstrap name for SRV records (eg mainnet)") + checkCmd.MarkFlagRequired("dnsbootstrap") +} + +func loadRelays(file string) []eb.Relay { + var relays []eb.Relay + err := codecs.LoadObjectFromFile(file, &relays) + if err != nil { + panic(err) + } + return relays +} + +type checkResult struct { + ID int64 + Success bool + Error string `json:",omitempty"` +} + +var checkCmd = &cobra.Command{ + Use: "check", + Short: "Check status of all relays", + Run: func(cmd *cobra.Command, args []string) { + relays := loadRelays(inputFileArg) + + nameEntries, err := getReverseMappedEntries(cfNameZoneID, nameRecordTypes) + if err != nil { + panic(err) + } + + bootstrapSrvService := "_algobootstrap._tcp." + dnsBootstrapArg + "." + srvDomainArg + srvEntries, err := getSrvRecords(cfSrvZoneID, bootstrapSrvService) + if err != nil { + panic(err) + } + + metricsSrvService := "_metrics._tcp." + srvDomainArg + metricsEntries, err := getSrvRecords(cfSrvZoneID, metricsSrvService) + if err != nil { + panic(err) + } + + checkOne := recordIDArg != 0 + results := make([]checkResult,0) + anyCheckError := false + + for _, relay := range relays { + if checkOne && relay.ID != recordIDArg { + continue + } + + name, port, err := verifyRelayStatus(relay, nameDomainArg, srvDomainArg, defaultPortArg, nameEntries, srvEntries, metricsEntries) + if err != nil { + fmt.Fprintf(os.Stderr, "[%d] ERROR: %s: %s\n", relay.ID, relay.IPOrDNSName, err) + results = append(results, checkResult{ + ID: relay.ID, + Success: false, + Error: err.Error(), + }) + anyCheckError = true + } else { + fmt.Printf("[%d] OK: %s -> %s:%d\n", relay.ID, relay.IPOrDNSName, name, port) + results = append(results, checkResult{ + ID: relay.ID, + Success: true, + }) + } + + if checkOne { + break + } + } + + if outputFileArg != "" { + codecs.SaveObjectToFile(outputFileArg, &results, true) + } + + // Only return success if all checked out + if anyCheckError { + os.Exit(-1) + } + }, +} + +func verifyRelayStatus(relay eb.Relay, nameDomain string, srvDomain string, defaultPort uint16, nameEntries map[string]string, srvEntries map[string]uint16, metricsEntries map[string]uint16) (srvName string, srvPort uint16, err error) { + var port uint16 + target, portString, err := net.SplitHostPort(relay.IPOrDNSName) + if err != nil { + target = relay.IPOrDNSName + port = defaultPort + } else { + var port64 uint64 + port64, err = strconv.ParseUint(portString, 10, 16) + if err != nil { + return + } + port = uint16(port64) + } + + if port == 0 { + err = fmt.Errorf("%s - port cannot be zero", relay.IPOrDNSName) + return + } + + // Error if target has another name entry - target should be relay provider's domain so shouldn't be possible + if mapsTo, has := nameEntries[target]; has { + err = fmt.Errorf("relay target has a DNS Name entry and should not (%s -> %s)", target, mapsTo) + } + + names, err := getTargetDNSChain(nameEntries, target) + if err != nil { + return + } + + // Error if no entries + if len(names) == 1 { + err = fmt.Errorf("no DNS entries found mapping to %s in '%s'", target, nameDomain) + return + } + + topmost := names[len(names)-1] + + if topmost != relay.DNSAlias+"."+nameDomain { + err = fmt.Errorf("topmost DNS name is not the assigned DNS Alias (wanted: %s, found %s)", + relay.DNSAlias, topmost) + return + } + + var ensureEntry = func(use string, entries map[string]uint16, port uint16) error { + type srvMatch struct { + name string + port uint16 + } + + // Now check for SRV entries for anything in that chain + var matches []srvMatch + for _, name := range names { + entry, has := entries[name] + if has { + matches = append(matches, srvMatch{name, entry}) + } + } + + if len(matches) == 0 { + return fmt.Errorf("no %s SRV entries found mapping to %s in '%s'", use, target, srvDomain) + } + + if len(matches) > 1 { + return fmt.Errorf("multiple %s SRV entries found in the chain mapping to %s", use, target) + } + + if matches[0].name != topmost || matches[0].port != port { + return fmt.Errorf("existing %s SRV record mapped to intermediate DNS name or wrong port (wanted %s:%d, found %s:%d)", + use, topmost, port, matches[0].name, matches[0].port) + } + return nil + } + + err = ensureEntry("bootstrap", srvEntries, port) + if err != nil { + return + } + + err = ensureEntry("metrics", metricsEntries, metricsPort) + if relay.MetricsEnabled { + if err != nil { + return + } + } else if err == nil { + err = fmt.Errorf("metrics should not be registered for %s but it is", target) + } + + srvName = topmost + srvPort = port + return +} + +// Returns an array of names starting with the target ip/name and ending with the outermost reference +func getTargetDNSChain(nameEntries map[string]string, target string) (names []string, err error) { + target = strings.ToLower(target) + if err != nil { + return + } + + names = append(names, target) + for { + from, has := nameEntries[target] + if !has { + return + } + names = append(names, from) + target = from + } +} + +func getReverseMappedEntries(zoneID string, recordTypes []string) (reverseMap map[string]string, err error) { + reverseMap = make(map[string]string) + + cloudflareDNS := cloudflare.NewDNS(zoneID, cfEmail, cfAuthKey) + + for _, recType := range recordTypes { + var records []cloudflare.DNSRecordResponseEntry + records, err = cloudflareDNS.ListDNSRecord(context.Background(), recType, "", "", "", "", "") + if err != nil { + return + } + + for _, record := range records { + // Error if duplicates found + from := strings.ToLower(record.Name) + target := strings.ToLower(record.Content) + if existing, has := reverseMap[target]; has { + err = fmt.Errorf("duplicate NAME entries mapped to %s: (%s && %s)", target, from, existing) + return + } + reverseMap[target] = from + } + } + return +} + +func getSrvRecords(zoneID string, serviceName string) (srvEntries map[string]uint16, err error){ + srvEntries = make(map[string]uint16) + + cloudflareDNS := cloudflare.NewDNS(zoneID, cfEmail, cfAuthKey) + + var records []cloudflare.DNSRecordResponseEntry + records, err = cloudflareDNS.ListDNSRecord(context.Background(), "SRV", serviceName, "", "", "", "") + if err != nil { + return + } + + for _, record := range records { + // record.Content is "priority port dnsname" + contents := strings.Split(record.Content, "\t") + target := strings.ToLower(contents[2]) + target = strings.TrimRight(target, ".") + portString := contents[1] + var port64 uint64 + port64, err = strconv.ParseUint(portString, 10, 16) + if err != nil { + panic(fmt.Sprintf("Invalid SRV Port for %s: %s", target, portString)) + } + port := uint16(port64) + + // Error if duplicates found + if existing, has := srvEntries[target]; has { + err = fmt.Errorf("duplicate SRV entries mapped to %s: (%d && %d)", target, port, existing) + return + } + srvEntries[target] = port + } + return +} + +//var addCmd = &cobra.Command{ +// Use: "add", +// Short: "Add a DNS record", +// Long: "Adds a DNS record to map --from to --to, using A if to == IP or CNAME otherwise\n", +// Example: "algons dns add -f a.test.algodev.network -t r1.algodev.network\n" + +// "algons dns add -f a.test.algodev.network -t 192.168.100.10", +// Run: func(cmd *cobra.Command, args []string) { +// err := doAddDNS(addFromName, addToAddress) +// if err != nil { +// fmt.Fprintf(os.Stderr, "Error adding DNS entry: %v\n", err) +// os.Exit(1) +// } else { +// fmt.Printf("DNS Entry Added\n") +// } +// }, +//} +// +//var deleteCmd = &cobra.Command{ +// Use: "delete", +// Short: "Delete DNS and SRV records for a specified network", +// Run: func(cmd *cobra.Command, args []string) { +// if !doDeleteDNS(deleteNetwork, noPrompt, excludePattern) { +// os.Exit(1) +// } +// }, +//} +// +//func doAddDNS(from string, to string) (err error) { +// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() +// if err != nil { +// return fmt.Errorf("error getting DNS credentials: %v", err) +// } +// +// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) +// +// const priority = 1 +// const proxied = false +// +// // If we need to register anything, first register a DNS entry +// // to map our network DNS name to our public name (or IP) provided to nodecfg +// // Network HostName = eg r1.testnet.algorand.network +// isIP := net.ParseIP(to) != nil +// var recordType string +// if isIP { +// recordType = "A" +// } else { +// recordType = "CNAME" +// } +// cloudflareDNS.SetDNSRecord(context.Background(), recordType, from, to, cloudflare.AutomaticTTL, priority, proxied) +// +// return +//} + +//func checkDNSRecord(dnsName string) { +// fmt.Printf("------------------------\nDNS Lookup: %s\n", dnsName) +// ips, err := net.LookupIP(dnsName) +// if err != nil { +// fmt.Printf("Cannot resolve %s: %v\n", dnsName, err) +// } else { +// sort.Sort(byIP(ips)) +// for _, ip := range ips { +// fmt.Printf("-> %s\n", ip.String()) +// } +// } +//} +// +//func doDeleteDNS(network string, noPrompt bool, excludePattern string) bool { +// +// if network == "" || network == "testnet" || network == "devnet" || network == "mainnet" { +// fmt.Fprintf(os.Stderr, "Deletion of network '%s' using this tool is not allowed\n", network) +// return false +// } +// +// var excludeRegex *regexp.Regexp +// if excludePattern != "" { +// var err error +// excludeRegex, err = regexp.Compile(excludePattern) +// if err != nil { +// fmt.Fprintf(os.Stderr, "specified regular expression exclude pattern ('%s') is not a valid regular expression : %v", excludePattern, err) +// return false +// } +// } +// +// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() +// if err != nil { +// fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) +// return false +// } +// +// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) +// +// idsToDelete := make(map[string]string) // Maps record ID to Name +// +// for _, service := range []string{"_algobootstrap", "_metrics"} { +// records, err := cloudflareDNS.ListDNSRecord(context.Background(), "SRV", service+"._tcp."+network+".algodev.network", "", "", "", "") +// if err != nil { +// fmt.Fprintf(os.Stderr, "Error listing SRV '%s' entries: %v\n", service, err) +// os.Exit(1) +// } +// for _, r := range records { +// if excludeRegex != nil { +// if excludeRegex.MatchString(r.Name) { +// fmt.Printf("Excluding SRV '%s' record: %s\n", service, r.Name) +// continue +// } +// } +// fmt.Printf("Found SRV '%s' record: %s\n", service, r.Name) +// idsToDelete[r.ID] = r.Name +// } +// } +// +// networkSuffix := "." + network + ".algodev.network" +// +// for _, recordType := range []string{"A", "CNAME"} { +// records, err := cloudflareDNS.ListDNSRecord(context.Background(), recordType, "", "", "", "", "") +// if err != nil { +// fmt.Fprintf(os.Stderr, "Error listing DNS '%s' entries: %v\n", recordType, err) +// os.Exit(1) +// } +// for _, r := range records { +// if strings.Index(r.Name, networkSuffix) > 0 { +// if excludeRegex != nil { +// if excludeRegex.MatchString(r.Name) { +// fmt.Printf("Excluding DNS '%s' record: %s\n", recordType, r.Name) +// continue +// } +// } +// fmt.Printf("Found DNS '%s' record: %s\n", recordType, r.Name) +// idsToDelete[r.ID] = r.Name +// } +// } +// } +// +// if len(idsToDelete) == 0 { +// fmt.Printf("No DNS/SRV records found\n") +// return true +// } +// +// var text string +// if !noPrompt { +// reader := bufio.NewReader(os.Stdin) +// fmt.Printf("Delete these %d entries (type 'yes' to delete)? ", len(idsToDelete)) +// text, _ = reader.ReadString('\n') +// text = strings.Replace(text, "\n", "", -1) +// } else { +// text = "yes" +// } +// +// if text == "yes" { +// for id, name := range idsToDelete { +// fmt.Fprintf(os.Stdout, "Deleting %s\n", name) +// err = cloudflareDNS.DeleteDNSRecord(context.Background(), id) +// if err != nil { +// fmt.Fprintf(os.Stderr, " !! error deleting %s: %v\n", name, err) +// } +// } +// } +// return true +//} +// From e8640b2cc9c70c2776b25f24fee60f63eb9e296b Mon Sep 17 00:00:00 2001 From: David Shoots Date: Sat, 15 Jun 2019 17:44:02 -0400 Subject: [PATCH 22/54] Implemented update, fixed issues, ran it! --- cmd/algorelay/relayCmd.go | 281 +++++++++++++++++++++++++++++--------- 1 file changed, 217 insertions(+), 64 deletions(-) diff --git a/cmd/algorelay/relayCmd.go b/cmd/algorelay/relayCmd.go index cb8d439d20..b157be0879 100644 --- a/cmd/algorelay/relayCmd.go +++ b/cmd/algorelay/relayCmd.go @@ -3,14 +3,16 @@ package main import ( "context" "fmt" - "github.com/algorand/go-algorand/cmd/algorelay/eb" - "github.com/algorand/go-algorand/tools/network/cloudflare" - "github.com/algorand/go-algorand/util/codecs" - "github.com/spf13/cobra" "net" "os" "strconv" "strings" + + "github.com/spf13/cobra" + + "github.com/algorand/go-algorand/cmd/algorelay/eb" + "github.com/algorand/go-algorand/tools/network/cloudflare" + "github.com/algorand/go-algorand/util/codecs" ) var ( @@ -59,6 +61,25 @@ func init() { checkCmd.MarkFlagRequired("defaultport") checkCmd.Flags().StringVarP(&dnsBootstrapArg, "dnsbootstrap", "b", "", "Bootstrap name for SRV records (eg mainnet)") checkCmd.MarkFlagRequired("dnsbootstrap") + + + rootCmd.AddCommand(updateCmd) + + updateCmd.Flags().StringVarP(&inputFileArg, "inputfile", "i", "", "File containing Relay data") + updateCmd.MarkFlagRequired("inputfile") + + updateCmd.Flags().StringVarP(&outputFileArg, "outputfile", "o", "", "File to output results to, as JSON") + + updateCmd.Flags().Int64Var(&recordIDArg, "id", 0, "Specific Datastore record ID to check (all if not specified)") + + updateCmd.Flags().StringVarP(&srvDomainArg, "srvdomain", "s", "", "Domain name for SRV records") + updateCmd.MarkFlagRequired("srvdomain") + updateCmd.Flags().StringVarP(&nameDomainArg, "namedomain", "n", "", "Domain name for A/CNAME records") + updateCmd.MarkFlagRequired("namedomain") + updateCmd.Flags().Uint16VarP(&defaultPortArg, "defaultport", "p", 4160, "Default listening port (eg 4160)") + updateCmd.MarkFlagRequired("defaultport") + updateCmd.Flags().StringVarP(&dnsBootstrapArg, "dnsbootstrap", "b", "", "Bootstrap name for SRV records (eg mainnet)") + updateCmd.MarkFlagRequired("dnsbootstrap") } func loadRelays(file string) []eb.Relay { @@ -76,28 +97,58 @@ type checkResult struct { Error string `json:",omitempty"` } +type dnsContext struct { + nameEntries map[string]string + bootstrap srvService + metrics srvService +} + +type srvService struct { + serviceName string + entries map[string]uint16 + shortName string + networkName string +} + +func makeDNSContext() *dnsContext { + nameEntries, err := getReverseMappedEntries(cfNameZoneID, nameRecordTypes) + if err != nil { + panic(err) + } + + bootstrap, err := getSrvRecords("_algobootstrap", dnsBootstrapArg + "." + srvDomainArg, cfSrvZoneID) + if err != nil { + panic(err) + } + + metrics, err := getSrvRecords("_metrics", srvDomainArg, cfSrvZoneID) + if err != nil { + panic(err) + } + + return &dnsContext{ + nameEntries: nameEntries, + bootstrap: bootstrap, + metrics: metrics, + } +} + +func makeService(shortName, networkName string) srvService { + return srvService{ + serviceName: shortName + "._tcp." + networkName, + entries: make(map[string]uint16), + shortName: shortName, + networkName: networkName, + } +} + var checkCmd = &cobra.Command{ Use: "check", Short: "Check status of all relays", Run: func(cmd *cobra.Command, args []string) { relays := loadRelays(inputFileArg) - nameEntries, err := getReverseMappedEntries(cfNameZoneID, nameRecordTypes) - if err != nil { - panic(err) - } - - bootstrapSrvService := "_algobootstrap._tcp." + dnsBootstrapArg + "." + srvDomainArg - srvEntries, err := getSrvRecords(cfSrvZoneID, bootstrapSrvService) - if err != nil { - panic(err) - } - - metricsSrvService := "_metrics._tcp." + srvDomainArg - metricsEntries, err := getSrvRecords(cfSrvZoneID, metricsSrvService) - if err != nil { - panic(err) - } + context := makeDNSContext() checkOne := recordIDArg != 0 results := make([]checkResult,0) @@ -108,7 +159,8 @@ var checkCmd = &cobra.Command{ continue } - name, port, err := verifyRelayStatus(relay, nameDomainArg, srvDomainArg, defaultPortArg, nameEntries, srvEntries, metricsEntries) + const checkOnly = true + name, port, err := ensureRelayStatus(checkOnly, relay, nameDomainArg, srvDomainArg, defaultPortArg, context) if err != nil { fmt.Fprintf(os.Stderr, "[%d] ERROR: %s: %s\n", relay.ID, relay.IPOrDNSName, err) results = append(results, checkResult{ @@ -141,7 +193,63 @@ var checkCmd = &cobra.Command{ }, } -func verifyRelayStatus(relay eb.Relay, nameDomain string, srvDomain string, defaultPort uint16, nameEntries map[string]string, srvEntries map[string]uint16, metricsEntries map[string]uint16) (srvName string, srvPort uint16, err error) { +var updateCmd = &cobra.Command{ + Use: "update", + Short: "Updates configuration for all relays to match the expectations", + Run: func(cmd *cobra.Command, args []string) { + relays := loadRelays(inputFileArg) + + context := makeDNSContext() + + updateOne := recordIDArg != 0 + results := make([]checkResult,0) + anyUpdateError := false + + for _, relay := range relays { + if updateOne && relay.ID != recordIDArg { + continue + } + + if !relay.CheckSuccess { + fmt.Printf("[%d] OK: Skipping NotSuccessful %s\n", relay.ID, relay.IPOrDNSName) + // Don't output results if skipped + continue + } + const checkOnly = false + name, port, err := ensureRelayStatus(checkOnly, relay, nameDomainArg, srvDomainArg, defaultPortArg, context) + if err != nil { + fmt.Fprintf(os.Stderr, "[%d] ERROR: %s: %s\n", relay.ID, relay.IPOrDNSName, err) + results = append(results, checkResult{ + ID: relay.ID, + Success: false, + Error: err.Error(), + }) + anyUpdateError = true + } else { + fmt.Printf("[%d] OK: %s -> %s:%d\n", relay.ID, relay.IPOrDNSName, name, port) + results = append(results, checkResult{ + ID: relay.ID, + Success: true, + }) + } + + if updateOne { + break + } + } + + if outputFileArg != "" { + codecs.SaveObjectToFile(outputFileArg, &results, true) + } + + // Only return success if all checked out + if anyUpdateError { + os.Exit(-1) + } + }, +} + +func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDomain string, defaultPort uint16, ctx *dnsContext) (srvName string, srvPort uint16, err error) { var port uint16 target, portString, err := net.SplitHostPort(relay.IPOrDNSName) if err != nil { @@ -162,27 +270,48 @@ func verifyRelayStatus(relay eb.Relay, nameDomain string, srvDomain string, defa } // Error if target has another name entry - target should be relay provider's domain so shouldn't be possible - if mapsTo, has := nameEntries[target]; has { + if mapsTo, has := ctx.nameEntries[target]; has { err = fmt.Errorf("relay target has a DNS Name entry and should not (%s -> %s)", target, mapsTo) } - names, err := getTargetDNSChain(nameEntries, target) + names, err := getTargetDNSChain(ctx.nameEntries, target) if err != nil { return } // Error if no entries if len(names) == 1 { - err = fmt.Errorf("no DNS entries found mapping to %s in '%s'", target, nameDomain) - return + if checkOnly { + err = fmt.Errorf("no DNS entries found mapping to %s in '%s'", target, nameDomain) + return + } } topmost := names[len(names)-1] - if topmost != relay.DNSAlias+"."+nameDomain { - err = fmt.Errorf("topmost DNS name is not the assigned DNS Alias (wanted: %s, found %s)", - relay.DNSAlias, topmost) - return + if relay.DNSAlias == "" { + err = fmt.Errorf("missing DNSAlias name") + } + + targetDomainAlias := relay.DNSAlias + "." + nameDomain + if topmost != targetDomainAlias { + if checkOnly { + err = fmt.Errorf("topmost DNS name is not the assigned DNS Alias (wanted: %s, found %s)", + relay.DNSAlias, topmost) + return + } + + // Add A/CNAME for the DNSAlias assigned + err = addDNSRecord(targetDomainAlias, topmost, cfNameZoneID) + if err != nil { + return + } else { + fmt.Printf("[%d] Added DNS Record: %s -> %s\n", relay.ID, targetDomainAlias, topmost) + + // Update our state + names = append(names, targetDomainAlias) + topmost = targetDomainAlias + } } var ensureEntry = func(use string, entries map[string]uint16, port uint16) error { @@ -215,15 +344,35 @@ func verifyRelayStatus(relay eb.Relay, nameDomain string, srvDomain string, defa return nil } - err = ensureEntry("bootstrap", srvEntries, port) + err = ensureEntry("bootstrap", ctx.bootstrap.entries, port) if err != nil { - return + if checkOnly { + return + } + + // Add SRV entry to map to our DNSAlias + err = addSRVRecord(ctx.bootstrap.networkName, topmost, port, ctx.bootstrap.shortName, cfSrvZoneID) + if err != nil { + return + } else { + fmt.Printf("[%d] Added boostrap SRV Record: %s:%d\n", relay.ID, targetDomainAlias, port) + } } - err = ensureEntry("metrics", metricsEntries, metricsPort) + err = ensureEntry("metrics", ctx.metrics.entries, metricsPort) if relay.MetricsEnabled { if err != nil { - return + if checkOnly { + return + } + + // Add SRV entry for metrics + err = addSRVRecord(ctx.metrics.networkName, topmost, metricsPort, ctx.metrics.shortName, cfSrvZoneID) + if err != nil { + return + } else { + fmt.Printf("[%d] Added metrics SRV Record: %s:%d\n", relay.ID, targetDomainAlias, metricsPort) + } } } else if err == nil { err = fmt.Errorf("metrics should not be registered for %s but it is", target) @@ -278,13 +427,13 @@ func getReverseMappedEntries(zoneID string, recordTypes []string) (reverseMap ma return } -func getSrvRecords(zoneID string, serviceName string) (srvEntries map[string]uint16, err error){ - srvEntries = make(map[string]uint16) +func getSrvRecords(serviceName string, networkName, zoneID string) (service srvService, err error){ + service = makeService(serviceName, networkName) cloudflareDNS := cloudflare.NewDNS(zoneID, cfEmail, cfAuthKey) var records []cloudflare.DNSRecordResponseEntry - records, err = cloudflareDNS.ListDNSRecord(context.Background(), "SRV", serviceName, "", "", "", "") + records, err = cloudflareDNS.ListDNSRecord(context.Background(), "SRV", service.serviceName, "", "", "", "") if err != nil { return } @@ -303,15 +452,44 @@ func getSrvRecords(zoneID string, serviceName string) (srvEntries map[string]uin port := uint16(port64) // Error if duplicates found - if existing, has := srvEntries[target]; has { + if existing, has := service.entries[target]; has { err = fmt.Errorf("duplicate SRV entries mapped to %s: (%d && %d)", target, port, existing) return } - srvEntries[target] = port + service.entries[target] = port } return } +func addDNSRecord(from string, to string, cfZoneID string) error { + cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfAuthKey) + + const priority = 1 + const proxied = false + + // If we need to register anything, first register a DNS entry + // to map our network DNS name to our public name (or IP) provided to nodecfg + // Network HostName = eg r1.testnet.algorand.network + isIP := net.ParseIP(to) != nil + var recordType string + if isIP { + recordType = "A" + } else { + recordType = "CNAME" + } + return cloudflareDNS.SetDNSRecord(context.Background(), recordType, from, to, cloudflare.AutomaticTTL, priority, proxied) +} + +func addSRVRecord(srvNetwork string, target string, port uint16, serviceShortName string, cfZoneID string) error { + cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfAuthKey) + + const priority = 1 + const weight = 1 + + return cloudflareDNS.SetSRVRecord(context.Background(), srvNetwork, target, cloudflare.AutomaticTTL, priority, uint(port), serviceShortName, "_tcp", weight) +} + + //var addCmd = &cobra.Command{ // Use: "add", // Short: "Add a DNS record", @@ -339,31 +517,6 @@ func getSrvRecords(zoneID string, serviceName string) (srvEntries map[string]uin // }, //} // -//func doAddDNS(from string, to string) (err error) { -// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() -// if err != nil { -// return fmt.Errorf("error getting DNS credentials: %v", err) -// } -// -// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) -// -// const priority = 1 -// const proxied = false -// -// // If we need to register anything, first register a DNS entry -// // to map our network DNS name to our public name (or IP) provided to nodecfg -// // Network HostName = eg r1.testnet.algorand.network -// isIP := net.ParseIP(to) != nil -// var recordType string -// if isIP { -// recordType = "A" -// } else { -// recordType = "CNAME" -// } -// cloudflareDNS.SetDNSRecord(context.Background(), recordType, from, to, cloudflare.AutomaticTTL, priority, proxied) -// -// return -//} //func checkDNSRecord(dnsName string) { // fmt.Printf("------------------------\nDNS Lookup: %s\n", dnsName) From 7b2547b2858fc2398a1d98b06de48aff14609fbb Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Mon, 17 Jun 2019 11:03:24 -0400 Subject: [PATCH 23/54] Catchup: avoid retrying unless previous block was retrieved. (#31) Existing code would retry retrieving a block regardless of whether the previous block was retrieved successfully. While (functionally) it won't hurt to make the subsequent attempt, it generate excessive network traffic when the first block retrieval is being delayed. We want to keep the happy path retrieve blocks as fast as possible, while slowing down once we ran into network failures. --- catchup/service.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/catchup/service.go b/catchup/service.go index cdbce404ea..c73a06d75e 100644 --- a/catchup/service.go +++ b/catchup/service.go @@ -150,6 +150,21 @@ func (s *Service) fetchAndWrite(fetcher rpcs.Fetcher, r basics.Round, prevFetchC if err != nil { s.log.Debugf("fetchAndWrite(%v): Could not fetch: %v (attempt %d)", r, err, i) + // we've just failed to retrieve a block; wait until the previous block is fetched before trying again + // to avoid the usecase where the first block doesn't exists and we're making many requests down the chain + // for no reason. + if !hasLookback { + select { + case <-s.ctx.Done(): + s.log.Debugf("fetchAndWrite(%v): Aborted while waiting for lookback block to ledger after failing once", r) + return false + case hasLookback = <-lookbackComplete: + if !hasLookback { + s.log.Debugf("fetchAndWrite(%v): lookback block doesn't exist, won't try to retrieve block again", r) + return false + } + } + } continue // retry the fetch } else if block == nil || cert == nil { // someone already wrote the block to the ledger, we should stop syncing From 909b436bb19a2e6ac458f0ec8a0d0d2bc6823190 Mon Sep 17 00:00:00 2001 From: Evan Richard Date: Mon, 17 Jun 2019 21:21:57 -0400 Subject: [PATCH 24/54] GOAL account export. Clarify info messages for seed import/export. (#36) It is possible to go to/from a wallet recovery mnemonic, but previously this functionality was not mirrored in account seed mnemonics: one could only import an account's mnemonic (for example, from algokey), and not export it. This PR proposes to add a new command `goal account export`, which recovers a stored account's mnemonic. Additionally, the distinctions between wallet mnemonics and account mnemonics are clarified somewhat. Testing: Tested this by destroying and then recovering an account on TestNet. --- cmd/goal/account.go | 41 +++++++++++++++++++++++++++++++++++++++-- cmd/goal/messages.go | 3 +++ cmd/goal/wallet.go | 2 +- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/cmd/goal/account.go b/cmd/goal/account.go index 97f6bac9b2..23aa655771 100644 --- a/cmd/goal/account.go +++ b/cmd/goal/account.go @@ -71,6 +71,7 @@ func init() { accountCmd.AddCommand(addParticipationKeyCmd) accountCmd.AddCommand(listParticipationKeysCmd) accountCmd.AddCommand(importCmd) + accountCmd.AddCommand(exportCmd) accountCmd.AddCommand(importRootKeysCmd) accountCmd.AddCommand(accountMultisigCmd) @@ -140,7 +141,9 @@ func init() { // import flags importCmd.Flags().BoolVarP(&importDefault, "default", "f", false, "Set this account as the default one") importCmd.Flags().StringVarP(&mnemonic, "mnemonic", "m", "", "Mnemonic to import (will prompt otherwise)") - + // export flags + exportCmd.Flags().StringVarP(&accountAddress, "address", "a", "", "Address of account to export") + exportCmd.MarkFlagRequired("address") // importRootKeys flags importRootKeysCmd.Flags().BoolVarP(&unencryptedWallet, "unencrypted-wallet", "u", false, "Import into the default unencrypted wallet, potentially creating it") @@ -756,6 +759,7 @@ var listParticipationKeysCmd = &cobra.Command{ var importCmd = &cobra.Command{ Use: "import", Short: "Import an account key from mnemonic", + Long: "Import an account key from a mnemonic generated by the export command or by algokey (NOT a mnemonic from the goal wallet command). The imported account will be listed alongside your wallet-generated accounts, but will not be tied to your wallet.", Run: func(cmd *cobra.Command, args []string) { dataDir := ensureSingleDataDir() accountList := makeAccountsList(dataDir) @@ -810,10 +814,43 @@ var importCmd = &cobra.Command{ }, } +var exportCmd = &cobra.Command{ + Use: "export", + Short: "Export an account key for use with account import", + Long: "Export an account mnemonic seed, for use with account import. This exports the seed for a single account and should not be confused with the wallet mnemonic.", + Run: func(cmd *cobra.Command, args []string) { + dataDir := ensureSingleDataDir() + client := ensureKmdClient(dataDir) + + wh, pw := ensureWalletHandleMaybePassword(dataDir, walletName, true) + passwordString := string(pw) + + response, err := client.ExportKey(wh, passwordString, accountAddress) + + if err != nil { + reportErrorf(errorRequestFail, err) + } + + seed, err := crypto.SecretKeyToSeed(response.PrivateKey) + + if err != nil { + reportErrorf(errorSeedConversion, accountAddress, err) + } + + privKeyAsMnemonic, err := passphrase.KeyToMnemonic(seed[:]) + + if err != nil { + reportErrorf(errorMnemonicConversion, accountAddress, err) + } + + reportInfof(infoExportedKey, accountAddress, privKeyAsMnemonic) + }, +} + var importRootKeysCmd = &cobra.Command{ Use: "importrootkey", Short: "Import .rootkey files from the data directory into a kmd wallet", - Long: "Import .rootkey files from the data directory into a kmd wallet", + Long: "Import .rootkey files from the data directory into a kmd wallet. This is analogous to using the import command with an account seed mnemonic: the imported account will be displayed alongside your wallet-derived accounts, but will not be tied to your wallet mnemonic.", Args: validateNoPosArgsFn, Run: func(cmd *cobra.Command, args []string) { dataDir := ensureSingleDataDir() diff --git a/cmd/goal/messages.go b/cmd/goal/messages.go index 309084551c..1a3140cb3c 100644 --- a/cmd/goal/messages.go +++ b/cmd/goal/messages.go @@ -28,6 +28,7 @@ const ( infoNoAccounts = "Did not find any account. Please import or create a new one." infoRenamedAccount = "Renamed account '%s' to '%s'" infoImportedKey = "Imported %s" + infoExportedKey = "Exported key for account %s: \"%s\"" infoImportedNKeys = "Imported %d key%s" infoCreatedNewAccount = "Created new account with address %s" errorNameAlreadyTaken = "The account name '%s' is already taken, please choose another." @@ -40,6 +41,8 @@ const ( warnMultisigDuplicatesDetected = "Warning: one or more duplicate addresses detected in multisig account creation. This will effectively give the duplicated address(es) extra signature weight. Continuing multisig account creation." errLastRoundInvalid = "roundLastValid needs to be well after the current round (%d)" errExistingPartKey = "Account already has a participation key valid at least until roundLastValid (%d) - current is %d" + errorSeedConversion = "Got private key for account %s, but was unable to convert to seed: %s" + errorMnemonicConversion = "Got seed for account %s, but was unable to convert to mnemonic: %s" // KMD infoKMDStopped = "Stopped kmd" diff --git a/cmd/goal/wallet.go b/cmd/goal/wallet.go index 479d638d4d..ad70eff50e 100644 --- a/cmd/goal/wallet.go +++ b/cmd/goal/wallet.go @@ -43,7 +43,7 @@ func init() { walletCmd.Flags().StringVarP(&defaultWalletName, "default", "f", "", "Set the wallet with this name to be the default wallet") // Should we recover the wallet? - newWalletCmd.Flags().BoolVarP(&recoverWallet, "recover", "r", false, "Recover the wallet from a backup mnemonic. Regenerate accounts in the wallet with `goal account new`") + newWalletCmd.Flags().BoolVarP(&recoverWallet, "recover", "r", false, "Recover the wallet from the backup mnemonic provided at wallet creation (NOT the mnemonic provided by goal account export or by algokey). Regenerate accounts in the wallet with `goal account new`") } var walletCmd = &cobra.Command{ From 35dbd7d03f0ea134cc07392317ecaf6a91e9e8a0 Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 17 Jun 2019 21:24:52 -0400 Subject: [PATCH 25/54] Updated configure_dev to include libtool, autoconf, and automake. (#42) Required for building libsodium. --- scripts/configure_dev.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/configure_dev.sh b/scripts/configure_dev.sh index bbc6977328..2970ea2e9f 100755 --- a/scripts/configure_dev.sh +++ b/scripts/configure_dev.sh @@ -22,6 +22,9 @@ elif [ "${OS}" = "darwin" ]; then install_or_upgrade pkg-config install_or_upgrade boost install_or_upgrade jq + install_or_upgrade libtool + install_or_upgrade autoconf + install_or_upgrade automake fi ${SCRIPTPATH}/configure_dev-deps.sh From a1c864a9d5cb61ee6a9df46be7b3cc7a3d10ec83 Mon Sep 17 00:00:00 2001 From: jecassis Date: Mon, 17 Jun 2019 18:33:23 -0700 Subject: [PATCH 26/54] Enhance NodeExporterPath JSON option to pass additional node_exporter arguments (#30) Example from config.json: "NodeExporterPath": "./node_exporter --collector.systemd" This should result in the following process being launched when metrics are enabled: ./node_exporter --collector.systemd --web.listen-address=:9100 --web.telemetry-path=/metrics --- util/metrics/reporter.go | 30 +++++++++++++++++--- util/metrics/reporter_test.go | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) create mode 100755 util/metrics/reporter_test.go diff --git a/util/metrics/reporter.go b/util/metrics/reporter.go index 62a6bb26e4..78d9a67649 100644 --- a/util/metrics/reporter.go +++ b/util/metrics/reporter.go @@ -23,6 +23,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strings" "time" // logging imports metrics so that we can have metrics about logging, which is more important than the four Debug lines we had here logging about metrics. TODO: find a more clever cycle resolution @@ -177,6 +178,30 @@ func (reporter *MetricReporter) tryDetachNodeExporter() { } } +// parseNodeExporterArgs parses the NodeExporterPath configuration string to extract Node Exporter's arguments. +func parseNodeExporterArgs(nodeExporterPath string, nodeExporterListenAddress string, nodeExporterMetricsPath string) []string { + whitespaceRE := regexp.MustCompile(`\s+`) + listenAddressRE := regexp.MustCompile(`--web.listen-address=(.+)`) + telemetryPathRE := regexp.MustCompile(`--web.telemetry-path=(.+)`) + vargs := whitespaceRE.Split(nodeExporterPath, -1) + temp := vargs[:0] + for _, varg := range vargs { + if listenAddressRE.MatchString(varg) { + nodeExporterListenAddress = listenAddressRE.FindStringSubmatch(varg)[1] + } else if telemetryPathRE.MatchString(varg) { + nodeExporterMetricsPath = telemetryPathRE.FindStringSubmatch(varg)[1] + } else if varg == "" { + continue + } else { + temp = append(temp, varg) + } + } + vargs = append(vargs[:len(temp)], + "--web.listen-address="+nodeExporterListenAddress, + "--web.telemetry-path="+nodeExporterMetricsPath) + return vargs +} + func (reporter *MetricReporter) tryInvokeNodeExporter(ctx context.Context) { var err error if nil == reporter.neSync { @@ -205,10 +230,7 @@ func (reporter *MetricReporter) tryInvokeNodeExporter(ctx context.Context) { os.Stderr} } // prepare the vargs that the new process is going to have. - vargs := []string{ - reporter.serviceConfig.NodeExporterPath, - "--web.listen-address=" + reporter.serviceConfig.NodeExporterListenAddress, - "--web.telemetry-path=" + nodeExporterMetricsPath} + vargs := parseNodeExporterArgs(reporter.serviceConfig.NodeExporterPath, reporter.serviceConfig.NodeExporterListenAddress, nodeExporterMetricsPath) // launch the process proc, err := os.StartProcess(vargs[0], vargs, &neAttributes) if err != nil { diff --git a/util/metrics/reporter_test.go b/util/metrics/reporter_test.go new file mode 100755 index 0000000000..5e5031a8ed --- /dev/null +++ b/util/metrics/reporter_test.go @@ -0,0 +1,52 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseNodeExporterArgs(t *testing.T) { + passTestcases := map[string][]string{ + "./node_exporter": []string{"./node_exporter", "--web.listen-address=:9100", "--web.telemetry-path=/metrics"}, // simple case + "./node_exporter --collector.systemd": []string{"./node_exporter", "--collector.systemd", "--web.listen-address=:9100", "--web.telemetry-path=/metrics"}, // extended case with one argument + "./node_exporter random --collector.systemd": []string{"./node_exporter", "random", "--collector.systemd", "--web.listen-address=:9100", "--web.telemetry-path=/metrics"}, // extended case multiple arguments + "/usr/bin/local/node_exporter --collector.systemd random": []string{"/usr/bin/local/node_exporter", "--collector.systemd", "random", "--web.listen-address=:9100", "--web.telemetry-path=/metrics"}, // other executable path + " /usr/bin/local/node_exporter --collector.systemd random": []string{"/usr/bin/local/node_exporter", "--collector.systemd", "random", "--web.listen-address=:9100", "--web.telemetry-path=/metrics"}, // space at beginning of option + "./node_exporter --web.telemetry-path=/foobar --web.listen-address=:9090 ": []string{"./node_exporter", "--web.listen-address=:9090", "--web.telemetry-path=/foobar"}, // overriding defaults + "./node_exporter --web.listen-address=:8080 --web.telemetry-path=/barfoo": []string{"./node_exporter", "--web.listen-address=:8080", "--web.telemetry-path=/barfoo"}, // overriding defaults different order and multiple spaces + "./node_exporter --web.listen-address=:9090 --collector.proc --web.telemetry-path=/foobar": []string{"./node_exporter", "--collector.proc", "--web.listen-address=:9090", "--web.telemetry-path=/foobar"}, // argument in between the persistent ones + "./node_exporter --web.listen-address=:9090 --collector.test --collector.systemd ": []string{"./node_exporter", "--collector.test", "--collector.systemd", "--web.listen-address=:9090", "--web.telemetry-path=/metrics"}, // argument after persistent one + } + for test, expected := range passTestcases { + vargs := parseNodeExporterArgs(test, ":9100", "/metrics") + require.Equalf(t, vargs, expected, "Argument parsing did not result in expected value for: %v, got: %v, want: %v.", test, vargs, expected) + } + + failTestcases := map[string][]string{ + "./node_exporter": []string{"./node_exporter", "--web.listen-address=:9090", "--web.telemetry-path=/foobar"}, // default arguments not being passed + "./node_exporter --collector.systemd": []string{"./node_exporter", "--web.listen-address=:9100", "--web.telemetry-path=/metrics", "--collector.systemd"}, // incorrect order of persistent and added options + "./node_exporter random --collector.systemd": []string{"./node_exporter", "--collector.systemd", "random", "--web.listen-address=:9100", "--web.telemetry-path=/metrics"}, // reversed order of persistent options + " /usr/bin/local/node_exporter --collector.systemd random": []string{" /usr/bin/local/node_exporter", "--collector.systemd", "random", "--web.listen-address=:9100", "--web.telemetry-path=/metrics"}, // space at beginning of option preserved + } + for test, notexpected := range failTestcases { + vargs := parseNodeExporterArgs(test, ":9100", "/metrics") + require.NotEqualf(t, vargs, notexpected, "Argument parsing did result in expected value for: %v, got: %v, want: %v.", test, vargs, notexpected) + } +} From 7da2e2acd5c068ffdf0e9f02535a0691f7dfc8e7 Mon Sep 17 00:00:00 2001 From: jecassis Date: Mon, 17 Jun 2019 18:54:57 -0700 Subject: [PATCH 27/54] Add Origin: and Label: fields to Debian package releases (#39) * Add Origin: and Label: fields to Debian package releases * Add configuration file-handling directives for the Debian package --- installer/{50algorand-upgrades => 51algorand-upgrades} | 5 +++++ installer/debian/conffiles | 2 +- scripts/build_deb.sh | 2 +- scripts/release_deb.sh | 3 +-- 4 files changed, 8 insertions(+), 4 deletions(-) rename installer/{50algorand-upgrades => 51algorand-upgrades} (73%) diff --git a/installer/50algorand-upgrades b/installer/51algorand-upgrades similarity index 73% rename from installer/50algorand-upgrades rename to installer/51algorand-upgrades index f3a941250c..1cadc380d2 100644 --- a/installer/50algorand-upgrades +++ b/installer/51algorand-upgrades @@ -4,3 +4,8 @@ Unattended-Upgrade::Allowed-Origins { "Algorand:stable"; }; + +Dpkg::Options { + "--force-confdef"; + "--force-confold"; +}; diff --git a/installer/debian/conffiles b/installer/debian/conffiles index 4aa4aa1622..0b093dca22 100644 --- a/installer/debian/conffiles +++ b/installer/debian/conffiles @@ -1,2 +1,2 @@ -/etc/apt/apt.conf.d/50algorand-upgrades +/etc/apt/apt.conf.d/51algorand-upgrades /var/lib/algorand/genesis.json diff --git a/scripts/build_deb.sh b/scripts/build_deb.sh index a125b2b26e..c1cfd8089e 100755 --- a/scripts/build_deb.sh +++ b/scripts/build_deb.sh @@ -94,7 +94,7 @@ for svc in "${systemd_files[@]}"; do cp installer/${svc} ${PKG_ROOT}/lib/systemd/system done -unattended_upgrades_files=("50algorand-upgrades") +unattended_upgrades_files=("51algorand-upgrades") mkdir -p ${PKG_ROOT}/etc/apt/apt.conf.d for f in "${unattended_upgrades_files[@]}"; do cp installer/${f} ${PKG_ROOT}/etc/apt/apt.conf.d diff --git a/scripts/release_deb.sh b/scripts/release_deb.sh index ce16cd0065..fea211c68d 100755 --- a/scripts/release_deb.sh +++ b/scripts/release_deb.sh @@ -71,9 +71,8 @@ SNAPSHOT=algorand-$(date +%Y%m%d_%H%M%S) aptly snapshot create ${SNAPSHOT} from repo algorand if [ ! -z "${FIRSTTIME}" ]; then echo "first publish" - aptly publish snapshot ${SNAPSHOT} "s3:${APTLY_S3_NAME}:" + aptly publish snapshot -origin=Algorand -label=Algorand ${SNAPSHOT} "s3:${APTLY_S3_NAME}:" else echo "publish snapshot ${SNAPSHOT}" aptly publish switch stable "s3:${APTLY_S3_NAME}:" ${SNAPSHOT} fi - From 3c4897f634a48578678ba939211bc1b974983702 Mon Sep 17 00:00:00 2001 From: Evan Richard Date: Mon, 17 Jun 2019 22:19:15 -0400 Subject: [PATCH 28/54] sendReceive_tests move more money. (#25) The existing sendReceive_test family of tests is a ping-pong-like test, so this changes these tests to move 45x ("a lot") more money. The test also was restructured somewhat to make it run faster. --- .../features/transactions/sendReceive_test.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/test/e2e-go/features/transactions/sendReceive_test.go b/test/e2e-go/features/transactions/sendReceive_test.go index 8af501277d..6c67401f56 100644 --- a/test/e2e-go/features/transactions/sendReceive_test.go +++ b/test/e2e-go/features/transactions/sendReceive_test.go @@ -91,10 +91,10 @@ func testAccountsCanSendMoney(t *testing.T, templatePath string) { transactionFee := minTxnFee + 5 amountPongSendsPing := minAcctBalance amountPingSendsPong := minAcctBalance * 3 / 2 - const numberOfSends = 5 + const numberOfSends = 225 + txidsToAddresses := make(map[string]string) for i := 0; i < numberOfSends; i++ { - txidsToAddresses := make(map[string]string) pongTx, err := pongClient.SendPaymentFromUnencryptedWallet(pongAccount, pingAccount, transactionFee, amountPongSendsPing, GenerateRandomBytes(8)) txidsToAddresses[pongTx.ID().String()] = pongAccount a.NoError(err, "fixture should be able to send money (pong -> ping), error on send number %v", i) @@ -105,15 +105,14 @@ func testAccountsCanSendMoney(t *testing.T, templatePath string) { expectedPongBalance = expectedPongBalance - transactionFee - amountPongSendsPing + amountPingSendsPong curStatus, _ := pongClient.Status() curRound := curStatus.LastRound - fixture.WaitForAllTxnsToConfirm(curRound+uint64(5), txidsToAddresses) - curStatus, _ = pongClient.Status() - curRound = curStatus.LastRound err = fixture.WaitForRoundWithTimeout(curRound + uint64(1)) a.NoError(err) - pingBalance, err = c.GetBalance(pingAccount) - pongBalance, err = c.GetBalance(pongAccount) - - a.True(expectedPingBalance <= pingBalance, "ping balance is different than expected., payment = %d", i) - a.True(expectedPongBalance <= pongBalance, "pong balance is different than expected., payment = %d", i) } + curStatus, _ := pongClient.Status() + curRound := curStatus.LastRound + fixture.WaitForAllTxnsToConfirm(curRound+uint64(5), txidsToAddresses) + pingBalance, err = c.GetBalance(pingAccount) + pongBalance, err = c.GetBalance(pongAccount) + a.True(expectedPingBalance <= pingBalance, "ping balance is different than expected.") + a.True(expectedPongBalance <= pongBalance, "pong balance is different than expected.") } From c218e362a55de828fdbca3f84d039c706d5bfe9e Mon Sep 17 00:00:00 2001 From: David Shoots Date: Mon, 17 Jun 2019 20:47:52 -0700 Subject: [PATCH 29/54] Switch default genesis.json file to mainnet (#46) --- scripts/compute_branch_release_network.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/compute_branch_release_network.sh b/scripts/compute_branch_release_network.sh index 15882507f2..744d5a5eac 100755 --- a/scripts/compute_branch_release_network.sh +++ b/scripts/compute_branch_release_network.sh @@ -9,9 +9,9 @@ if [ -z "${NETWORK}" ]; then exit -1 fi -#if [ "${NETWORK}" = "testnet" ]; then -# echo "mainnet" -# exit 0 -#fi +if [ "${NETWORK}" = "testnet" ]; then + echo "mainnet" + exit 0 +fi echo "${NETWORK}" From 3f3b54bc28baced653ea46dbfef7955d0a530050 Mon Sep 17 00:00:00 2001 From: algobolson <45948765+algobolson@users.noreply.github.com> Date: Tue, 18 Jun 2019 06:55:29 -0400 Subject: [PATCH 30/54] build refinements 20190617 (#44) * stop making copies to obsolete -telem package names * only upload when s3 destination vars are set fixes to deb relases --- scripts/build_packages.sh | 8 -------- scripts/build_release.sh | 23 ++++++++++++++++------- scripts/build_release_local.sh | 20 +++++++++++++++++--- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/scripts/build_packages.sh b/scripts/build_packages.sh index adac93ab09..d3b504ff7e 100755 --- a/scripts/build_packages.sh +++ b/scripts/build_packages.sh @@ -131,13 +131,5 @@ for var in "${VARIATION_ARRAY[@]}"; do cp -p *.deb ${PKG_ROOT}/${GATE_PREFIX}algorand_${CHANNEL}_${PKG_NAME}_${FULLVERSION}.deb popd fi - - # For now, we only deploy packages with telemetry (so we can transition smoothly). - # Copy xxx_channel_yyy -> xxx_channel-telem_yyy - if [[ ("${var}" = "") && (! -z ${TRANSITION_TELEMETRY_BUILDS}) ]]; then - cp ${PKG_ROOT}/${GATE_PREFIX}node_${CHANNEL}_${PKG_NAME}_${FULLVERSION}.tar.gz ${PKG_ROOT}/${GATE_PREFIX}node_${CHANNEL}-telem_${PKG_NAME}_${FULLVERSION}.tar.gz - cp ${PKG_ROOT}/${GATE_PREFIX}install_${CHANNEL}_${PKG_NAME}_${FULLVERSION}.tar.gz ${PKG_ROOT}/${GATE_PREFIX}install_${CHANNEL}-telem_${PKG_NAME}_${FULLVERSION}.tar.gz - cp ${PKG_ROOT}/${GATE_PREFIX}tools_${CHANNEL}_${PKG_NAME}_${FULLVERSION}.tar.gz ${PKG_ROOT}/${GATE_PREFIX}tools_${CHANNEL}-telem_${PKG_NAME}_${FULLVERSION}.tar.gz - fi done done diff --git a/scripts/build_release.sh b/scripts/build_release.sh index ad2a4cf823..8cfb1a3d47 100755 --- a/scripts/build_release.sh +++ b/scripts/build_release.sh @@ -4,7 +4,8 @@ # be prompted for GPG key password at a couple points. # # Externally settable env vars: -# S3_PREFIX= where to upload build artifacts +# S3_PREFIX= where to upload build artifacts (no trailing /) +# S3_PREFIX_BUILDLOG= where upload build log (no trailing /) # AWS_EFS_MOUNT= NFS to mount for `aptly` persistent state and scratch storage # SIGNING_KEY_ADDR= dev@algorand.com or similar for GPG key # RSTAMP= `scripts/reverse_hex_timestamp` @@ -16,10 +17,6 @@ date "+build_release start %Y%m%d_%H%M%S" set -e set -x -if [ -z "${S3_PREFIX}" ]; then - S3_PREFIX=s3://algorand-builds -fi - # persistent storage of repo manager scratch space is on EFS if [ ! -z "${AWS_EFS_MOUNT}" ]; then if mount|grep -q /data; then @@ -37,6 +34,14 @@ fi export GOPATH=${HOME}/go export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} +# a previous docker centos build can leave junk owned by root. chown and clean +sudo chown -R ${USER} ${GOPATH} +if [ -f ${GOPATH}/src/github.com/algorand/go-algorand/crypto/libsodium-fork/Makefile ]; then + (cd ${GOPATH}/src/github.com/algorand/go-algorand/crypto/libsodium-fork && make distclean) +fi +rm -rf ${GOPATH}/src/github.com/algorand/go-algorand/crypto/lib + + cd ${GOPATH}/src/github.com/algorand/go-algorand export RELEASE_GENESIS_PROCESS=true export TRANSITION_TELEMETRY_BUILDS=true @@ -136,7 +141,9 @@ gpg --detach-sign "${HASHFILE}" gpg --clearsign "${HASHFILE}" echo RSTAMP=${RSTAMP} > "${HOME}/rstamp" -aws s3 sync --quiet --exclude dev\* --exclude master\* --exclude nightly\* --exclude stable\* --acl public-read ./ ${S3_PREFIX}/${CHANNEL}/${RSTAMP}_${FULLVERSION}/ +if [ ! -z "${S3_PREFIX}" ]; then + aws s3 sync --quiet --exclude dev\* --exclude master\* --exclude nightly\* --exclude stable\* --acl public-read ./ ${S3_PREFIX}/${CHANNEL}/${RSTAMP}_${FULLVERSION}/ +fi # copy .rpm file to intermediate yum repo scratch space, actual publish manually later if [ ! -d /data/yumrepo ]; then @@ -172,7 +179,9 @@ EOF dpkg -l >>"${STATUSFILE}" gpg --clearsign "${STATUSFILE}" gzip "${STATUSFILE}.asc" -aws s3 cp --quiet "${STATUSFILE}.asc.gz" "s3://algorand-devops-misc/buildlog/${RSTAMP}/${STATUSFILE}.asc.gz" +if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then + aws s3 cp --quiet "${STATUSFILE}.asc.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/${STATUSFILE}.asc.gz" +fi # use aptly to push .deb to its serving repo # Leave .deb publishing to manual step after we do more checks on the release artifacts. diff --git a/scripts/build_release_local.sh b/scripts/build_release_local.sh index e534ac57f4..6f7f81d2af 100644 --- a/scripts/build_release_local.sh +++ b/scripts/build_release_local.sh @@ -1,4 +1,10 @@ #!/bin/bash +# +# This is a file of commands to copy and paste to run build_release.sh on an AWS EC2 instance. +# Should work on Ubuntu 16.04 ro 18.04 +# +# Externally settable env vars: +# S3_PREFIX_BUILDLOG= where upload build log (no trailing /) echo "this is a file of commands to copy and paste to run build_release.sh on an AWS EC2 instance" exit 1 @@ -40,7 +46,7 @@ umask 0002 # this will require your key password, and export a private key file protected by the same password # warm up your local gpg-agent -gpg --clearsign +gpg -u dev@algorand.com --clearsign type some stuff ^D @@ -52,6 +58,12 @@ REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgbin/remote_gpg_socket) LOCAL_GPG_SOCKET=$(gpgconf --list-dir agent-extra-socket) ssh -A -R "${REMOTE_GPG_SOCKET}:${LOCAL_GPG_SOCKET}" ubuntu@${TARGET} +# check gpg agent connection +gpg -u dev@algorand.com --clearsign +blah blah +^D + + # set AWS credentials so we can upload to S3 and connect to EFS export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= @@ -74,5 +86,7 @@ if [ -z "${RSTAMP}" ]; then echo "could not figure out RSTAMP, script must have failed early" exit 1 fi -gzip buildlog_* -aws s3 cp buildlog_*.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/ +gzip "buildlog_${BUILDTIMESTAMP}" +if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then + aws s3 cp "buildlog_${BUILDTIMESTAMP}.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/buildlog_${BUILDTIMESTAMP}.gz" +fi From 0d9b7c172e6ebf5f0f678c70eea79bb505a64804 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Tue, 18 Jun 2019 09:01:27 -0400 Subject: [PATCH 31/54] check fewer rounds in ledger.TestArchival to speed it up (#28) --- ledger/archival_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/archival_test.go b/ledger/archival_test.go index 97fe5c02a6..113d29d607 100644 --- a/ledger/archival_test.go +++ b/ledger/archival_test.go @@ -106,7 +106,7 @@ func TestArchival(t *testing.T) { wl.l.AddBlock(blk, agreement.Certificate{}) // Don't bother checking the trackers every round -- it's too slow.. - if crypto.RandUint64()%10 > 0 { + if crypto.RandUint64()%23 > 0 { continue } From 0da21b4bd7dd2bb00e644199b46d3d29618500ec Mon Sep 17 00:00:00 2001 From: David Shoots Date: Tue, 18 Jun 2019 09:02:05 -0400 Subject: [PATCH 32/54] Skip relays that aren't validated --- cmd/algorelay/relayCmd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/algorelay/relayCmd.go b/cmd/algorelay/relayCmd.go index b157be0879..6f3f45d408 100644 --- a/cmd/algorelay/relayCmd.go +++ b/cmd/algorelay/relayCmd.go @@ -159,6 +159,10 @@ var checkCmd = &cobra.Command{ continue } + if !relay.CheckSuccess { + continue + } + const checkOnly = true name, port, err := ensureRelayStatus(checkOnly, relay, nameDomainArg, srvDomainArg, defaultPortArg, context) if err != nil { From f346d3fd2b565ac8235898da29df8279cef67d11 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Tue, 18 Jun 2019 11:34:03 -0400 Subject: [PATCH 33/54] Add GitHub templates. (#48) Add GitHub templates. --- .github/ISSUE_TEMPLATE/bug_report.md | 30 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 30 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/question.md | 17 +++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 23 +++++++++++++++++ CONTRIBUTING.md | 2 +- 5 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..c709eff1c4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: '🐜 Bug report' +about: 'Report a reproducible bug.' +title: '' +labels: 'new-bug' +--- + + +### Subject of the issue +Describe your issue here. + +### Your environment +* Software version: `algod -v` +* Node status if applicable: `goal node status` +* Operating System details. +* In many cases log files and cadaver files are also useful to include. Since these files may be large, an Algorand developer may request them later. These files may include public addresses that you're participating with. If that is a concern please be sure to scrub that data. + +### Steps to reproduce +Tell us how to reproduce this issue. + +### Expected behaviour +Tell us what should happen + +### Actual behaviour +Tell us what happens instead diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..3ad2519657 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,30 @@ +--- +name: '🔔 Feature Request' +about: 'Suggestions for how we can improve the algorand platform.' +title: '' +labels: 'new-feature-request' +--- + + + +**Is your feature request related to a problem? Please describe.** + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** + +A clear and concise description of what you want to happen. + +**Additional context** + +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000000..bab0d9df73 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,17 @@ +--- +name: '❓ Question' +about: 'General questions related to the algorand platform.' +title: '' +labels: 'question' +--- + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..f4c962c611 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,23 @@ + + +## Summary + +Explain the goal of this change and what problem it is solving. + +## Test Plan + +How did you test these changes? Please provide the exact scenarios you tested in as much detail as possible including commands, output and rationale. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18591da8b7..e9080e1056 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ Again, if you have a patch for a critical security vulnerability, please use our For Go code we use the [Golang guidelines defined here](https://golang.org/doc/effective_go.html). * Code must adhere to the official Go formatting guidelines (i.e. uses gofmt). -* We use **gofmt** and **golint**. Also make sure to run `make fix` and `make generate` before opening a pull request. +* We use **gofmt** and **golint**. Also make sure to run `make sanity` and `make generate` before opening a pull request. * Code must be documented adhering to the official Go commentary guidelines. For JavaScript code we use the [MDN formatting rules](https://developer.mozilla.org/en-US/docs/MDN/Contribute/Guidelines/Code_guidelines/JavaScript). From 72e90ace268a3b08d3fdc6e7de41b05a18076170 Mon Sep 17 00:00:00 2001 From: David Shoots Date: Tue, 18 Jun 2019 13:31:34 -0400 Subject: [PATCH 34/54] Don't report error if metrics not enabled and shouldn't be --- cmd/algorelay/relayCmd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/algorelay/relayCmd.go b/cmd/algorelay/relayCmd.go index 6f3f45d408..09b906fbce 100644 --- a/cmd/algorelay/relayCmd.go +++ b/cmd/algorelay/relayCmd.go @@ -380,6 +380,10 @@ func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDom } } else if err == nil { err = fmt.Errorf("metrics should not be registered for %s but it is", target) + } else { + // If metrics are not enabled, then we SHOULD get an error. + // Since this isn't actually an error, reset to nil + err = nil } srvName = topmost From a1adb693d1dc203ee7085a000f80ee3028aa8d8e Mon Sep 17 00:00:00 2001 From: algobolson <45948765+algobolson@users.noreply.github.com> Date: Tue, 18 Jun 2019 14:50:58 -0400 Subject: [PATCH 35/54] skip flaky test (#45) --- test/e2e-go/features/transactions/transactionPool_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e-go/features/transactions/transactionPool_test.go b/test/e2e-go/features/transactions/transactionPool_test.go index 2976b6c771..6f6da0323d 100644 --- a/test/e2e-go/features/transactions/transactionPool_test.go +++ b/test/e2e-go/features/transactions/transactionPool_test.go @@ -27,6 +27,7 @@ import ( ) func TestTransactionPoolOrderingAndClearing(t *testing.T) { + t.Skip("test is flaky as of 2019-06-18") t.Parallel() r := require.New(t) From 794d777445def9f3e11af61e933d65a2a134f018 Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 18 Jun 2019 17:56:35 -0400 Subject: [PATCH 36/54] [GOAL2-698] support configurable telemetry credentials. (#27) * goal2-698 support configurable telemetry credentials. * Adjusted username and password to use actual values per comment from David. --- logging/telemetryCommon.go | 2 + logging/telemetryConfig.go | 6 +- logging/telemetryConfig_test.go | 92 +++++++++++++++++++ logging/telemetryhook.go | 2 +- .../configs/logging/logging.config.example | 10 ++ 5 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 logging/telemetryConfig_test.go create mode 100644 test/testdata/configs/logging/logging.config.example diff --git a/logging/telemetryCommon.go b/logging/telemetryCommon.go index 969b4ae239..c8c10538b4 100644 --- a/logging/telemetryCommon.go +++ b/logging/telemetryCommon.go @@ -55,6 +55,8 @@ type TelemetryConfig struct { FilePath string // Path to file on disk, if any ChainID string `json:"-"` SessionGUID string `json:"-"` + UserName string + Password string } type asyncTelemetryHook struct { diff --git a/logging/telemetryConfig.go b/logging/telemetryConfig.go index 0b5f1323e6..f232aa77e9 100644 --- a/logging/telemetryConfig.go +++ b/logging/telemetryConfig.go @@ -32,10 +32,6 @@ import ( var loggingFilename = "logging.config" -// these credentials have the minimum privilege set required to write to elasticsearch -var userName = "telemetry-v9" -var password = "oq%$FA1TOJ!yYeMEcJ7D688eEOE#MGCu" - func elasticsearchEndpoint() string { return "https://1ae9f9654b25441090fe5c48c833b95a.us-east-1.aws.found.io:9243" } @@ -70,6 +66,8 @@ func createTelemetryConfig() TelemetryConfig { MinLogLevel: logrus.WarnLevel, ReportHistoryLevel: logrus.WarnLevel, LogHistoryDepth: 100, + UserName: "telemetry-v9", + Password: "oq%$FA1TOJ!yYeMEcJ7D688eEOE#MGCu", } } diff --git a/logging/telemetryConfig_test.go b/logging/telemetryConfig_test.go new file mode 100644 index 0000000000..24da62fb1b --- /dev/null +++ b/logging/telemetryConfig_test.go @@ -0,0 +1,92 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package logging + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_loadTelemetryConfig(t *testing.T) { + + sample := TelemetryConfig{ + Enable: true, + GUID: "guid", + URI: "elastic.algorand.com", + MinLogLevel: 4, + ReportHistoryLevel: 4, + LogHistoryDepth: 100, + UserName: "telemetry-v9", + Password: "oq%$FA1TOJ!yYeMEcJ7D688eEOE#MGCu", + } + + a := require.New(t) + ourPath, err := os.Getwd() + a.NoError(err) + configsPath := filepath.Join(ourPath, "../test/testdata/configs/logging/logging.config.example") + + config, err := loadTelemetryConfig(configsPath) + a.NoError(err) + + a.Equal(sample.Enable, config.Enable) + a.Equal(sample.GUID, config.GUID) + a.Equal(sample.URI, config.URI) + a.Equal(sample.MinLogLevel, config.MinLogLevel) + a.Equal(sample.ReportHistoryLevel, config.ReportHistoryLevel) + a.Equal(sample.UserName, config.UserName) + a.Equal(sample.Password, config.Password) + +} + +func Test_CreateSaveLoadTelemetryConfig(t *testing.T) { + + testDir := os.Getenv("TESTDIR") + + if testDir == "" { + testDir, _ = ioutil.TempDir("", "tmp") + } + + a := require.New(t) + + configsPath := filepath.Join(testDir, "logging.config") + config1 := createTelemetryConfig() + + err := config1.Save(configsPath) + a.NoError(err) + + config2, err := loadTelemetryConfig(configsPath) + a.NoError(err) + + a.Equal(config1.Enable, config2.Enable) + a.Equal(config1.URI, config2.URI) + a.Equal(config1.Name, config2.Name) + a.Equal(config1.GUID, config2.GUID) + a.Equal(config1.MinLogLevel, config2.MinLogLevel) + a.Equal(config1.ReportHistoryLevel, config2.ReportHistoryLevel) + a.Equal(config1.LogHistoryDepth, config2.LogHistoryDepth) + a.Equal(config1.FilePath, "") + a.Equal(configsPath, config2.FilePath) + a.Equal(config1.ChainID, config2.ChainID) + a.Equal(config1.SessionGUID, config2.SessionGUID) + a.Equal(config1.UserName, config2.UserName) + a.Equal(config1.Password, config2.Password) + +} diff --git a/logging/telemetryhook.go b/logging/telemetryhook.go index 4e2ef3974b..017835cdec 100644 --- a/logging/telemetryhook.go +++ b/logging/telemetryhook.go @@ -123,7 +123,7 @@ func (hook *asyncTelemetryHook) Flush() { func createElasticHook(cfg TelemetryConfig) (hook logrus.Hook, err error) { client, err := elastic.NewClient(elastic.SetURL(cfg.URI), - elastic.SetBasicAuth(userName, password), + elastic.SetBasicAuth(cfg.UserName, cfg.Password), elastic.SetSniff(false), elastic.SetGzip(true)) if err != nil { diff --git a/test/testdata/configs/logging/logging.config.example b/test/testdata/configs/logging/logging.config.example new file mode 100644 index 0000000000..ba5642e8c0 --- /dev/null +++ b/test/testdata/configs/logging/logging.config.example @@ -0,0 +1,10 @@ +{ + "Enable": true, + "GUID": "guid", + "URI": "elastic.algorand.com", + "MinLogLevel": 4, + "ReportHistoryLevel": 4, + "LogHistoryDepth": 100, + "UserName": "telemetry-v9", + "Password": "oq%$FA1TOJ!yYeMEcJ7D688eEOE#MGCu" +} From c84729e061e19182b9693354f2ddeb1f6a0b1baf Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 19 Jun 2019 12:15:23 -0400 Subject: [PATCH 37/54] Fix copy/paste error in promote_stable script. (#52) --- scripts/promote_stable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/promote_stable.sh b/scripts/promote_stable.sh index bacaf26814..2c82e794ad 100755 --- a/scripts/promote_stable.sh +++ b/scripts/promote_stable.sh @@ -39,5 +39,5 @@ CHANNEL="stable" ${S3CMD} ls s3://${S3_UPLOAD_BUCKET}/pending_ | grep _${CHANNEL}[_-] | awk '{ print $4 }' | while read line; do NEW_ARTIFACT_NAME=$(echo "$line" | sed -e 's/pending_//') echo "Rename ${line} => ${NEW_ARTIFACT_NAME}" - ${S3CMD} mv ${line} ${RENAMED} + ${S3CMD} mv ${line} ${NEW_ARTIFACT_NAME} done From ea855418b0b928435c0f883e3847456b938ab293 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Wed, 19 Jun 2019 12:37:17 -0400 Subject: [PATCH 38/54] [GOAL2-731] Better slow peers disconnection logic (#15) Disconnect slow peers based on their network activity rather than pending buffer size. --- network/ping.go | 2 +- network/wsNetwork.go | 112 ++++++++++++++++++++++++++++---------- network/wsNetwork_test.go | 110 +++++++++++++++++++++++++++++++++++++ network/wsPeer.go | 42 ++++++++++---- network/wsPeer_test.go | 39 +++++++++++++ 5 files changed, 264 insertions(+), 41 deletions(-) create mode 100644 network/wsPeer_test.go diff --git a/network/ping.go b/network/ping.go index 53309cf633..84d00f4837 100644 --- a/network/ping.go +++ b/network/ping.go @@ -35,7 +35,7 @@ func pingHandler(message IncomingMessage) OutgoingMessage { copy(mbytes, tbytes) copy(mbytes[len(tbytes):], message.Data) var digest crypto.Digest // leave blank, ping message too short - peer.writeNonBlock(mbytes, false, digest) + peer.writeNonBlock(mbytes, false, digest, time.Now()) return OutgoingMessage{} } diff --git a/network/wsNetwork.go b/network/wsNetwork.go index efd483f0db..849a628eb7 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -83,12 +83,21 @@ const MaxInt = int((^uint(0)) >> 1) // connectionActivityMonitorInterval is the interval at which we check // if any of the connected peers have been idle for a long while and // need to be disconnected. -const connectionActivityMonitorInterval = time.Minute * 3 +const connectionActivityMonitorInterval = 3 * time.Minute // maxPeerInactivityDuration is the maximum allowed duration for a // peer to remain completly idle (i.e. no inbound or outbound communication), before // we discard the connection. -const maxPeerInactivityDuration = time.Minute * 5 +const maxPeerInactivityDuration = 5 * time.Minute + +// maxMessageQueueDuration is the maximum amount of time a message is allowed to be waiting +// in the various queues before being sent. Once that deadline has reached, sending the message +// is pointless, as it's too stale to be of any value +const maxMessageQueueDuration = 25 * time.Second + +// slowWritingPeerMonitorInterval is the interval at which we peek on the connected peers to +// verify that their current outgoing message is not being blocked for too long. +const slowWritingPeerMonitorInterval = 5 * time.Second var networkIncomingConnections = metrics.MakeGauge(metrics.NetworkIncomingConnections) var networkOutgoingConnections = metrics.MakeGauge(metrics.NetworkOutgoingConnections) @@ -99,10 +108,12 @@ var networkHandleMicros = metrics.MakeCounter(metrics.MetricName{Name: "algod_ne var networkBroadcasts = metrics.MakeCounter(metrics.MetricName{Name: "algod_network_broadcasts_total", Description: "number of broadcast operations"}) var networkBroadcastQueueMicros = metrics.MakeCounter(metrics.MetricName{Name: "algod_network_broadcast_queue_micros_total", Description: "microseconds broadcast requests sit on queue"}) var networkBroadcastSendMicros = metrics.MakeCounter(metrics.MetricName{Name: "algod_network_broadcast_send_micros_total", Description: "microseconds spent broadcasting"}) -var networkBroadcastsDropped = metrics.MakeCounter(metrics.MetricName{Name: "algod_broadcasts_dropped_total", Description: "number of broadcast messages not sent to some peer"}) +var networkBroadcastsDropped = metrics.MakeCounter(metrics.MetricName{Name: "algod_broadcasts_dropped_total", Description: "number of broadcast messages not sent to any peer"}) +var networkPeerBroadcastDropped = metrics.MakeCounter(metrics.MetricName{Name: "algod_peer_broadcast_dropped_total", Description: "number of broadcast messages not sent to some peer"}) var networkSlowPeerDrops = metrics.MakeCounter(metrics.MetricName{Name: "algod_network_slow_drops_total", Description: "number of peers dropped for being slow to send to"}) var networkIdlePeerDrops = metrics.MakeCounter(metrics.MetricName{Name: "algod_network_idle_drops_total", Description: "number of peers dropped due to idle connection"}) +var networkBroadcastQueueFull = metrics.MakeCounter(metrics.MetricName{Name: "algod_network_broadcast_queue_full_total", Description: "number of messages that were drops due to full broadcast queue"}) var minPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_min_ping_seconds", Description: "Network round trip time to fastest peer in seconds."}) var meanPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_mean_ping_seconds", Description: "Network round trip time to average peer in seconds."}) @@ -294,14 +305,20 @@ type WebsocketNetwork struct { // once we detect that we have a misconfigured UseForwardedForAddress, we set this and write an warning message. misconfiguredUseForwardedForAddress bool + + // outgoingMessagesBufferSize is the size used for outgoing messages. + outgoingMessagesBufferSize int + + // slowWritingPeerMonitorInterval defines the interval between two consecutive tests for slow peer writing + slowWritingPeerMonitorInterval time.Duration } type broadcastRequest struct { - tag Tag - data []byte - except *wsPeer - done chan struct{} - start time.Time + tag Tag + data []byte + except *wsPeer + done chan struct{} + enqueueTime time.Time } // Address returns a string and whether that is a 'final' address or guessed. @@ -335,7 +352,7 @@ func (wn *WebsocketNetwork) PublicAddress() string { // if wait is true then the call blocks until the packet has actually been sent to all neighbors. // TODO: add `priority` argument so that we don't have to guess it based on tag func (wn *WebsocketNetwork) Broadcast(ctx context.Context, tag protocol.Tag, data []byte, wait bool, except Peer) error { - request := broadcastRequest{tag: tag, data: data, start: time.Now()} + request := broadcastRequest{tag: tag, data: data, enqueueTime: time.Now()} if except != nil { request.except = except.(*wsPeer) } @@ -373,6 +390,7 @@ func (wn *WebsocketNetwork) Broadcast(ctx context.Context, tag protocol.Tag, dat default: wn.log.Debugf("broadcast queue full") // broadcastQueue full, and we're not going to wait for it. + networkBroadcastQueueFull.Inc(nil) return errBcastQFull } } @@ -499,13 +517,25 @@ func (wn *WebsocketNetwork) setup() { wn.server.IdleTimeout = httpServerIdleTimeout wn.server.MaxHeaderBytes = httpServerMaxHeaderBytes wn.ctx, wn.ctxCancel = context.WithCancel(context.Background()) - wn.broadcastQueueHighPrio = make(chan broadcastRequest, 1000) + // roughly estimate the number of messages that could be sent over the lifespan of a single round. + wn.outgoingMessagesBufferSize = int(config.Consensus[protocol.ConsensusCurrentVersion].NumProposers*2 + + config.Consensus[protocol.ConsensusCurrentVersion].SoftCommitteeSize + + config.Consensus[protocol.ConsensusCurrentVersion].CertCommitteeSize + + config.Consensus[protocol.ConsensusCurrentVersion].NextCommitteeSize + + config.Consensus[protocol.ConsensusCurrentVersion].LateCommitteeSize + + config.Consensus[protocol.ConsensusCurrentVersion].RedoCommitteeSize + + config.Consensus[protocol.ConsensusCurrentVersion].DownCommitteeSize) + + wn.broadcastQueueHighPrio = make(chan broadcastRequest, wn.outgoingMessagesBufferSize) wn.broadcastQueueBulk = make(chan broadcastRequest, 100) wn.meshUpdateRequests = make(chan meshRequest, 5) wn.readyChan = make(chan struct{}) wn.tryConnectAddrs = make(map[string]int64) wn.eventualReadyDelay = time.Minute wn.prioTracker = newPrioTracker(wn) + if wn.slowWritingPeerMonitorInterval == 0 { + wn.slowWritingPeerMonitorInterval = slowWritingPeerMonitorInterval + } readBufferLen := wn.config.IncomingConnectionsLimit + wn.config.GossipFanout if readBufferLen < 100 { @@ -838,7 +868,7 @@ func (wn *WebsocketNetwork) ServeHTTP(response http.ResponseWriter, request *htt prioChallenge: challenge, } peer.TelemetryGUID = otherTelemetryGUID - peer.init(wn.config) + peer.init(wn.config, wn.outgoingMessagesBufferSize) wn.addPeer(peer) localAddr, _ := wn.Address() wn.log.With("event", "ConnectedIn").With("remote", otherPublicAddr).With("local", localAddr).Infof("Accepted incoming connection from peer %s", otherPublicAddr) @@ -913,6 +943,23 @@ func (wn *WebsocketNetwork) checkPeersConnectivity() { } } +// checkSlowWritingPeers tests each of the peer's current message timestamp. +// if that timestamp is too old, it means that the transmission of that message +// takes longer than desired. In that case, it will disconnect the peer, allowing it to reconnect +// to a faster network endpoint. +func (wn *WebsocketNetwork) checkSlowWritingPeers() { + wn.peersLock.Lock() + defer wn.peersLock.Unlock() + currentTime := time.Now() + for _, peer := range wn.peers { + if peer.CheckSlowWritingPeer(currentTime) { + wn.wg.Add(1) + go wn.disconnectThread(peer, disconnectSlowConn) + networkSlowPeerDrops.Inc(nil) + } + } +} + func (wn *WebsocketNetwork) sendFilterMessage(msg IncomingMessage) { digest := generateMessageDigest(msg.Tag, msg.Data) //wn.log.Debugf("send filter %s(%d) %v", msg.Tag, len(msg.Data), digest) @@ -922,8 +969,12 @@ func (wn *WebsocketNetwork) sendFilterMessage(msg IncomingMessage) { func (wn *WebsocketNetwork) broadcastThread() { defer wn.wg.Done() var peers []*wsPeer + slowWritingPeerCheckTicker := time.NewTicker(wn.slowWritingPeerMonitorInterval) + defer slowWritingPeerCheckTicker.Stop() for { // broadcast from high prio channel as long as we can + // we want to try and keep this as a single case select with a default, since go compiles a single-case + // select with a default into a more efficient non-blocking receive, instead of compiling it to the general-purpose selectgo select { case request := <-wn.broadcastQueueHighPrio: wn.innerBroadcast(request, true, &peers) @@ -935,6 +986,9 @@ func (wn *WebsocketNetwork) broadcastThread() { select { case request := <-wn.broadcastQueueHighPrio: wn.innerBroadcast(request, true, &peers) + case <-slowWritingPeerCheckTicker.C: + wn.checkSlowWritingPeers() + continue case request := <-wn.broadcastQueueBulk: wn.innerBroadcast(request, false, &peers) case <-wn.ctx.Done(): @@ -957,8 +1011,16 @@ func (wn *WebsocketNetwork) peerSnapshot(dest []*wsPeer) []*wsPeer { // prio is set if the broadcast is a high-priority broadcast. func (wn *WebsocketNetwork) innerBroadcast(request broadcastRequest, prio bool, ppeers *[]*wsPeer) { - broadcastQueueTime := time.Now().Sub(request.start) - networkBroadcastQueueMicros.AddUint64(uint64(broadcastQueueTime.Nanoseconds()/1000), nil) + if request.done != nil { + defer close(request.done) + } + + broadcastQueueDuration := time.Now().Sub(request.enqueueTime) + networkBroadcastQueueMicros.AddUint64(uint64(broadcastQueueDuration.Nanoseconds()/1000), nil) + if broadcastQueueDuration > maxMessageQueueDuration { + networkBroadcastsDropped.Inc(nil) + return + } start := time.Now() tbytes := []byte(request.tag) @@ -975,37 +1037,27 @@ func (wn *WebsocketNetwork) innerBroadcast(request broadcastRequest, prio bool, peers := *ppeers // first send to all the easy outbound peers who don't block, get them started. + sentMessageCount := 0 for pi, peer := range peers { - if wn.config.BroadcastConnectionsLimit >= 0 && pi >= wn.config.BroadcastConnectionsLimit { + if wn.config.BroadcastConnectionsLimit >= 0 && sentMessageCount >= wn.config.BroadcastConnectionsLimit { break } if peer == request.except { peers[pi] = nil continue } - ok := peer.writeNonBlock(mbytes, prio, digest) + ok := peer.writeNonBlock(mbytes, prio, digest, request.enqueueTime) if ok { peers[pi] = nil + sentMessageCount++ continue } - if prio { - // couldn't send a high prio message; give up - wn.log.Infof("dropping peer for being too slow to send to: %s, %d enqueued", peer.rootURL, len(peer.sendBufferHighPrio)) - wn.removePeer(peer, disconnectTooSlow) - peer.Close() - networkSlowPeerDrops.Inc(nil) - } else { - networkBroadcastsDropped.Inc(nil) - } + networkPeerBroadcastDropped.Inc(nil) } dt := time.Now().Sub(start) networkBroadcasts.Inc(nil) networkBroadcastSendMicros.AddUint64(uint64(dt.Nanoseconds()/1000), nil) - - if request.done != nil { - close(request.done) - } } // NumPeers returns number of peers we connect to (all peers incoming and outbound). @@ -1434,7 +1486,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { } peer := &wsPeer{wsPeerCore: wsPeerCore{net: wn, rootURL: addr}, conn: conn, outgoing: true, incomingMsgFilter: wn.incomingMsgFilter} peer.TelemetryGUID = otherTelemetryGUID - peer.init(wn.config) + peer.init(wn.config, wn.outgoingMessagesBufferSize) wn.addPeer(peer) localAddr, _ := wn.Address() wn.log.With("event", "ConnectedOut").With("remote", addr).With("local", localAddr).Infof("Made outgoing connection to peer %v", addr) @@ -1452,7 +1504,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { resp := wn.prioScheme.MakePrioResponse(challenge) if resp != nil { mbytes := append([]byte(protocol.NetPrioResponseTag), resp...) - sent := peer.writeNonBlock(mbytes, true, crypto.Digest{}) + sent := peer.writeNonBlock(mbytes, true, crypto.Digest{}, time.Now()) if !sent { wn.log.With("remote", addr).With("local", localAddr).Warnf("could not send priority response to %v", addr) } diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go index e5870502b3..b9dd648fd1 100644 --- a/network/wsNetwork_test.go +++ b/network/wsNetwork_test.go @@ -48,6 +48,8 @@ import ( "github.com/algorand/go-algorand/util/metrics" ) +const sendBufferLength = 1000 + func TestMain(m *testing.M) { logging.Base().SetLevel(logging.Debug) os.Exit(m.Run()) @@ -583,6 +585,7 @@ func avgSendBufferHighPrioLength(wn *WebsocketNetwork) float64 { // // This is a deeply invasive test that reaches into the guts of WebsocketNetwork and wsPeer. If the implementation chainges consider throwing away or totally reimplementing this test. func TestSlowOutboundPeer(t *testing.T) { + t.Skip() // todo - update this test to reflect the new implementation. xtag := protocol.ProposalPayloadTag node := makeTestWebsocketNode(t) destPeers := make([]wsPeer, 5) @@ -1354,3 +1357,110 @@ func TestWebsocketNetwork_checkHeaders(t *testing.T) { }) } } + +func (wn *WebsocketNetwork) broadcastWithTimestamp(tag protocol.Tag, data []byte, when time.Time) error { + request := broadcastRequest{tag: tag, data: data, enqueueTime: when} + + broadcastQueue := wn.broadcastQueueBulk + if highPriorityTag(tag) { + broadcastQueue = wn.broadcastQueueHighPrio + } + // no wait + select { + case broadcastQueue <- request: + return nil + default: + return errBcastQFull + } +} + +func TestDelayedMessageDrop(t *testing.T) { + netA := makeTestWebsocketNode(t) + netA.config.GossipFanout = 1 + netA.Start() + defer func() { t.Log("stopping A"); netA.Stop(); t.Log("A done") }() + + noAddressConfig := defaultConfig + noAddressConfig.NetAddress = "" + netB := makeTestWebsocketNodeWithConfig(t, noAddressConfig) + netB.config.GossipFanout = 1 + addrA, postListen := netA.Address() + require.True(t, postListen) + t.Log(addrA) + netB.phonebook = &oneEntryPhonebook{addrA} + netB.Start() + defer func() { t.Log("stopping B"); netB.Stop(); t.Log("B done") }() + counter := newMessageCounter(t, 5) + counterDone := counter.done + netB.RegisterHandlers([]TaggedMessageHandler{TaggedMessageHandler{Tag: debugTag, MessageHandler: counter}}) + + readyTimeout := time.NewTimer(2 * time.Second) + waitReady(t, netA, readyTimeout.C) + waitReady(t, netB, readyTimeout.C) + + currentTime := time.Now() + for i := 0; i < 10; i++ { + err := netA.broadcastWithTimestamp(debugTag, []byte("foo"), currentTime.Add(time.Hour*time.Duration(i-5))) + require.NoErrorf(t, err, "No error was expected") + } + + select { + case <-counterDone: + case <-time.After(maxMessageQueueDuration): + require.Equalf(t, 5, counter.count, "One or more messages failed to reach destination network") + } +} + +func TestSlowPeerDisconnection(t *testing.T) { + log := logging.TestingLog(t) + log.SetLevel(logging.Level(defaultConfig.BaseLoggerDebugLevel)) + wn := &WebsocketNetwork{ + log: log, + config: defaultConfig, + phonebook: emptyPhonebookSingleton, + GenesisID: "go-test-network-genesis", + NetworkID: config.Devtestnet, + slowWritingPeerMonitorInterval: time.Millisecond * 50, + } + wn.setup() + wn.eventualReadyDelay = time.Second + + netA := wn + netA.config.GossipFanout = 1 + netA.Start() + defer func() { t.Log("stopping A"); netA.Stop(); t.Log("A done") }() + + noAddressConfig := defaultConfig + noAddressConfig.NetAddress = "" + netB := makeTestWebsocketNodeWithConfig(t, noAddressConfig) + netB.config.GossipFanout = 1 + addrA, postListen := netA.Address() + require.True(t, postListen) + t.Log(addrA) + netB.phonebook = &oneEntryPhonebook{addrA} + netB.Start() + defer func() { t.Log("stopping B"); netB.Stop(); t.Log("B done") }() + + readyTimeout := time.NewTimer(2 * time.Second) + waitReady(t, netA, readyTimeout.C) + waitReady(t, netB, readyTimeout.C) + + var peers []*wsPeer + peers = netA.peerSnapshot(peers) + require.Equalf(t, len(peers), 1, "Expected number of peers should be 1") + peer := peers[0] + // modify the peer on netA and + atomic.StoreInt64(&peer.intermittentOutgoingMessageEnqueueTime, time.Now().Add(-maxMessageQueueDuration).Add(-time.Second).UnixNano()) + // wait up to 2*slowWritingPeerMonitorInterval for the monitor to figure out it needs to disconnect. + expire := time.Now().Add(maxMessageQueueDuration * time.Duration(2)) + for { + peers = netA.peerSnapshot(peers) + if len(peers) == 0 || peers[0] != peer { + break + } + if time.Now().After(expire) { + require.Fail(t, "Slow peer was not disconnected") + } + time.Sleep(time.Millisecond * 5) + } +} diff --git a/network/wsPeer.go b/network/wsPeer.go index 148d66a488..062d1c1e1a 100644 --- a/network/wsPeer.go +++ b/network/wsPeer.go @@ -47,8 +47,6 @@ const maxMessageLength = 4 * 1024 * 1024 // Currently the biggest message is VB // buffer and starve messages from other peers. const msgsInReadBufferPerPeer = 10 -const sendBufferLength = 1000 - var networkSentBytesTotal = metrics.MakeCounter(metrics.NetworkSentBytesTotal) var networkReceivedBytesTotal = metrics.MakeCounter(metrics.NetworkReceivedBytesTotal) @@ -72,8 +70,9 @@ type wsPeerWebsocketConn interface { } type sendMessage struct { - data []byte - enqueued time.Time + data []byte + enqueued time.Time // the time at which the message was first generated + peerEnqueued time.Time // the time at which the peer was attempting to enqueue the message } // wsPeerCore also works for non-connected peers we want to do HTTP GET from @@ -91,6 +90,7 @@ const disconnectTooSlow disconnectReason = "TooSlow" const disconnectReadError disconnectReason = "ReadError" const disconnectWriteError disconnectReason = "WriteError" const disconnectIdleConn disconnectReason = "IdleConnection" +const disconnectSlowConn disconnectReason = "SlowConnection" type wsPeer struct { // lastPacketTime contains the UnixNano at the last time a successfull communication was made with the peer. @@ -99,6 +99,10 @@ type wsPeer struct { // we want this to be a 64-bit aligned for atomics. lastPacketTime int64 + // intermittentOutgoingMessageEnqueueTime contains the UnixNano of the message's enqueue time that is currently being written to the + // peer, or zero if no message is being written. + intermittentOutgoingMessageEnqueueTime int64 + wsPeerCore // conn will be *websocket.Conn (except in testing) @@ -192,7 +196,7 @@ func (wp *wsPeer) Unicast(ctx context.Context, msg []byte, tag protocol.Tag) err digest = crypto.Hash(mbytes) } - ok := wp.writeNonBlock(mbytes, false, digest) + ok := wp.writeNonBlock(mbytes, false, digest, time.Now()) if !ok { networkBroadcastsDropped.Inc(nil) err = fmt.Errorf("wsPeer failed to unicast: %v", wp.GetAddress()) @@ -202,7 +206,7 @@ func (wp *wsPeer) Unicast(ctx context.Context, msg []byte, tag protocol.Tag) err } // setup values not trivially assigned -func (wp *wsPeer) init(config config.Local) { +func (wp *wsPeer) init(config config.Local, sendBufferLength int) { wp.net.log.Debugf("wsPeer init outgoing=%v %#v", wp.outgoing, wp.rootURL) wp.closing = make(chan struct{}) wp.sendBufferHighPrio = make(chan sendMessage, sendBufferLength) @@ -343,6 +347,15 @@ func (wp *wsPeer) writeLoopSend(msg sendMessage) (exit bool) { // just drop it, don't break the connection return false } + // check if this message was waiting in the queue for too long. If this is the case, return "true" to indicate that we want to close the connection. + msgWaitDuration := time.Now().Sub(msg.enqueued) + if msgWaitDuration > maxMessageQueueDuration { + wp.net.log.Warnf("peer stale enqueued message %dms", msgWaitDuration.Nanoseconds()/1000000) + networkConnectionsDroppedTotal.Inc(map[string]string{"reason": "stale message"}) + return true + } + atomic.StoreInt64(&wp.intermittentOutgoingMessageEnqueueTime, msg.enqueued.UnixNano()) + defer atomic.StoreInt64(&wp.intermittentOutgoingMessageEnqueueTime, 0) err := wp.conn.WriteMessage(websocket.BinaryMessage, msg.data) if err != nil { if atomic.LoadInt32(&wp.didInnerClose) == 0 { @@ -354,7 +367,7 @@ func (wp *wsPeer) writeLoopSend(msg sendMessage) (exit bool) { atomic.StoreInt64(&wp.lastPacketTime, time.Now().UnixNano()) networkSentBytesTotal.AddUint64(uint64(len(msg.data)), nil) networkMessageSentTotal.AddUint64(1, nil) - networkMessageQueueMicrosTotal.AddUint64(uint64(time.Now().Sub(msg.enqueued).Nanoseconds()/1000), nil) + networkMessageQueueMicrosTotal.AddUint64(uint64(time.Now().Sub(msg.peerEnqueued).Nanoseconds()/1000), nil) return false } @@ -391,7 +404,7 @@ func (wp *wsPeer) writeLoopCleanup() { } // return true if enqueued/sent -func (wp *wsPeer) writeNonBlock(data []byte, highPrio bool, digest crypto.Digest) bool { +func (wp *wsPeer) writeNonBlock(data []byte, highPrio bool, digest crypto.Digest, msgEnqueueTime time.Time) bool { if wp.outgoingMsgFilter != nil && len(data) > messageFilterSize && wp.outgoingMsgFilter.CheckDigest(digest, false, false) { //wp.net.log.Debugf("msg drop as outbound dup %s(%d) %v", string(data[:2]), len(data)-2, digest) // peer has notified us it doesn't need this message @@ -407,7 +420,7 @@ func (wp *wsPeer) writeNonBlock(data []byte, highPrio bool, digest crypto.Digest outchan = wp.sendBufferBulk } select { - case outchan <- sendMessage{data, time.Now()}: + case outchan <- sendMessage{data: data, enqueued: msgEnqueueTime, peerEnqueued: time.Now()}: return true default: } @@ -432,7 +445,7 @@ func (wp *wsPeer) sendPing() bool { copy(mbytes, tagBytes) rand.Read(mbytes[len(tagBytes):]) wp.pingData = mbytes[len(tagBytes):] - sent := wp.writeNonBlock(mbytes, false, crypto.Digest{}) + sent := wp.writeNonBlock(mbytes, false, crypto.Digest{}, time.Now()) if sent { wp.pingInFlight = true @@ -476,3 +489,12 @@ func (wp *wsPeer) CloseAndWait() { func (wp *wsPeer) GetLastPacketTime() int64 { return atomic.LoadInt64(&wp.lastPacketTime) } + +func (wp *wsPeer) CheckSlowWritingPeer(now time.Time) bool { + ongoingMessageTime := atomic.LoadInt64(&wp.intermittentOutgoingMessageEnqueueTime) + if ongoingMessageTime == 0 { + return false + } + timeSinceMessageCreated := now.Sub(time.Unix(0, ongoingMessageTime)) + return timeSinceMessageCreated > maxMessageQueueDuration +} diff --git a/network/wsPeer_test.go b/network/wsPeer_test.go new file mode 100644 index 0000000000..7084b0aa44 --- /dev/null +++ b/network/wsPeer_test.go @@ -0,0 +1,39 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package network + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCheckSlowWritingPeer(t *testing.T) { + now := time.Now() + peer := wsPeer{ + intermittentOutgoingMessageEnqueueTime: 0, + } + require.Equal(t, peer.CheckSlowWritingPeer(now), false) + + peer.intermittentOutgoingMessageEnqueueTime = now.UnixNano() + require.Equal(t, peer.CheckSlowWritingPeer(now), false) + + peer.intermittentOutgoingMessageEnqueueTime = now.Add(-maxMessageQueueDuration * 2).UnixNano() + require.Equal(t, peer.CheckSlowWritingPeer(now), true) + +} From 1a60fafa867eef4f70a3a1396cb81818dec9d308 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Wed, 19 Jun 2019 16:39:02 -0400 Subject: [PATCH 39/54] fix max truncation bug in auction use of TransactionsByAddr() (#61) --- auction/tracker.go | 3 ++- cmd/auctionminion/main.go | 3 ++- daemon/algod/api/client/restClient.go | 5 +++-- test/e2e-go/restAPI/restClient_test.go | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/auction/tracker.go b/auction/tracker.go index effc445a3e..a71101abd4 100644 --- a/auction/tracker.go +++ b/auction/tracker.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "math" "sync" "github.com/algorand/go-deadlock" @@ -289,7 +290,7 @@ func (am *Tracker) LiveUpdateWithContext(ctx context.Context, wg *sync.WaitGroup log.Debugf("Getting transactions for %d-%d", am.LastRound+1, status.LastRound) - transactions, err := rc.TransactionsByAddr(am.AuctionKey.GetChecksumAddress().String(), am.LastRound+1, status.LastRound) + transactions, err := rc.TransactionsByAddr(am.AuctionKey.GetChecksumAddress().String(), am.LastRound+1, status.LastRound, math.MaxUint64) if err != nil { log.Error(err) fmt.Println(err) diff --git a/cmd/auctionminion/main.go b/cmd/auctionminion/main.go index 941fe315a3..e8128c9db3 100644 --- a/cmd/auctionminion/main.go +++ b/cmd/auctionminion/main.go @@ -21,6 +21,7 @@ import ( "flag" "fmt" "io/ioutil" + "math" "net/url" "os" @@ -143,7 +144,7 @@ func main() { fmt.Printf("Checking round %d..\n", curRound) } - txns, err := restClient.TransactionsByAddr(auctionChecksumAddr.String(), curRound, curRound) + txns, err := restClient.TransactionsByAddr(auctionChecksumAddr.String(), curRound, curRound, math.MaxUint64) if err != nil { fmt.Fprintf(os.Stderr, "Cannot fetch transactions from %d: %v\n", curRound, err) os.Exit(1) diff --git a/daemon/algod/api/client/restClient.go b/daemon/algod/api/client/restClient.go index 12222b0b16..ede9be3e78 100644 --- a/daemon/algod/api/client/restClient.go +++ b/daemon/algod/api/client/restClient.go @@ -210,12 +210,13 @@ func (client RestClient) LedgerSupply() (response models.Supply, err error) { type transactionsByAddrParams struct { FirstRound uint64 `url:"firstRound"` LastRound uint64 `url:"lastRound"` + Max uint64 `url:"max"` } // TransactionsByAddr returns all transactions for a PK [addr] in the [first, // last] rounds range. -func (client RestClient) TransactionsByAddr(addr string, first, last uint64) (response models.TransactionList, err error) { - err = client.get(&response, fmt.Sprintf("/account/%s/transactions", addr), transactionsByAddrParams{first, last}) +func (client RestClient) TransactionsByAddr(addr string, first, last, max uint64) (response models.TransactionList, err error) { + err = client.get(&response, fmt.Sprintf("/account/%s/transactions", addr), transactionsByAddrParams{first, last, max}) return } diff --git a/test/e2e-go/restAPI/restClient_test.go b/test/e2e-go/restAPI/restClient_test.go index 4ee5a479aa..39ac93b0c4 100644 --- a/test/e2e-go/restAPI/restClient_test.go +++ b/test/e2e-go/restAPI/restClient_test.go @@ -211,7 +211,7 @@ func TestTransactionsByAddr(t *testing.T) { restClient, err := localFixture.NC.AlgodClient() require.NoError(t, err) - res, err := restClient.TransactionsByAddr(toAddress, 0, rnd.LastRound) + res, err := restClient.TransactionsByAddr(toAddress, 0, rnd.LastRound, 100) require.NoError(t, err) require.Equal(t, 1, len(res.Transactions)) From 6cbbcda734906f3f64a63356f199d6189964b782 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Wed, 19 Jun 2019 16:39:26 -0400 Subject: [PATCH 40/54] print expected outcomes hash in auctionminion (#64) --- cmd/auctionminion/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/auctionminion/main.go b/cmd/auctionminion/main.go index e8128c9db3..9063ba067b 100644 --- a/cmd/auctionminion/main.go +++ b/cmd/auctionminion/main.go @@ -17,6 +17,7 @@ package main import ( + "encoding/base64" "encoding/json" "flag" "fmt" @@ -249,4 +250,8 @@ func main() { cfg.StartRound = ra.LastRound() + 1 writeConfig(cfg) fmt.Printf("Wrote updated state to %s\n", *stateFile) + + outcomes := ra.Settle(false) + outcomesHash := crypto.HashObj(outcomes) + fmt.Printf("Expected outcomes hash (if settled without cancelling): %v\n", base64.StdEncoding.EncodeToString(outcomesHash[:])) } From ece2bbb1d735cf085607286318d3efa1099db38a Mon Sep 17 00:00:00 2001 From: Evan Richard Date: Wed, 19 Jun 2019 21:01:17 -0400 Subject: [PATCH 41/54] [GOAL2-796] re-enable firstvalid and lastvalid for goal clerk (#55) --- cmd/goal/clerk.go | 9 ++--- libgoal/libgoal.go | 28 +++++++++++---- libgoal/unencryptedWallet.go | 2 +- test/e2e-go/cli/perf/payment_test.go | 4 +-- .../e2e-go/features/multisig/multisig_test.go | 6 ++-- .../onlineOfflineParticipation_test.go | 2 +- .../transactions/close_account_test.go | 2 +- test/e2e-go/restAPI/restClient_test.go | 34 +++++++++---------- .../createManyAndGoOnline_test.go | 2 +- test/framework/fixtures/restClientFixture.go | 2 +- 10 files changed, 54 insertions(+), 37 deletions(-) diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index e324ad6d93..b9de36a627 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -24,6 +24,7 @@ import ( "os" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/protocol" @@ -61,8 +62,8 @@ func init() { sendCmd.Flags().StringVarP(&toAddress, "to", "t", "", "Address to send to money to (required)") sendCmd.Flags().Uint64VarP(&amount, "amount", "a", 0, "The amount to be transferred (required), in microAlgos") sendCmd.Flags().Uint64Var(&fee, "fee", 0, "The transaction fee (automatically determined by default), in microAlgos") - sendCmd.Flags().Uint64Var(&firstValid, "firstvalid", 0, "The first round where the transaction may be committed to the ledger (currently ignored)") - sendCmd.Flags().Uint64Var(&lastValid, "lastvalid", 0, "The last round where the transaction may be committed to the ledger (currently ignored)") + sendCmd.Flags().Uint64Var(&firstValid, "firstvalid", 0, "The first round where the transaction may be committed to the ledger") + sendCmd.Flags().Uint64Var(&lastValid, "lastvalid", 0, "The last round where the transaction may be committed to the ledger") sendCmd.Flags().StringVar(¬eBase64, "noteb64", "", "Note (URL-base64 encoded)") sendCmd.Flags().StringVarP(¬eText, "note", "n", "", "Note text (ignored if --noteb64 used also)") sendCmd.Flags().StringVarP(&txFilename, "out", "o", "", "Dump an unsigned tx to the given file. In order to dump a signed transaction, pass -s") @@ -145,7 +146,7 @@ var sendCmd = &cobra.Command{ if txFilename == "" { // Sign and broadcast the tx wh, pw := ensureWalletHandleMaybePassword(dataDir, walletName, true) - tx, err := client.SendPaymentFromWallet(wh, pw, fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved) + tx, err := client.SendPaymentFromWallet(wh, pw, fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved, basics.Round(firstValid), basics.Round(lastValid)) // update information from Transaction txid := tx.ID().String() @@ -191,7 +192,7 @@ var sendCmd = &cobra.Command{ } } } else { - payment, err := client.ConstructPayment(fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved) + payment, err := client.ConstructPayment(fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved, basics.Round(firstValid), basics.Round(lastValid)) if err != nil { reportErrorf(errorConstructingTX, err) } diff --git a/libgoal/libgoal.go b/libgoal/libgoal.go index 97ca13e569..f4d8f6fa52 100644 --- a/libgoal/libgoal.go +++ b/libgoal/libgoal.go @@ -428,13 +428,17 @@ type MultisigInfo struct { } // SendPaymentFromWallet signs a transaction using the given wallet and returns the resulted transaction id -func (c *Client) SendPaymentFromWallet(walletHandle, pw []byte, from, to string, fee, amount uint64, note []byte, closeTo string) (transactions.Transaction, error) { +func (c *Client) SendPaymentFromWallet(walletHandle, pw []byte, from, to string, fee, amount uint64, note []byte, closeTo string, firstValid, lastValid basics.Round) (transactions.Transaction, error) { // Build the transaction - tx, err := c.ConstructPayment(from, to, fee, amount, note, closeTo) + tx, err := c.ConstructPayment(from, to, fee, amount, note, closeTo, firstValid, lastValid) if err != nil { return transactions.Transaction{}, err } + return c.signAndBroadcastTransactionWithWallet(walletHandle, pw, tx) +} + +func (c *Client) signAndBroadcastTransactionWithWallet(walletHandle, pw []byte, tx transactions.Transaction) (transactions.Transaction, error) { // Sign the transaction kmd, err := c.ensureKmdClient() if err != nil { @@ -467,7 +471,9 @@ func (c *Client) SendPaymentFromWallet(walletHandle, pw []byte, from, to string, // ConstructPayment builds a payment transaction to be signed // If the fee is 0, the function will use the suggested one form the network -func (c *Client) ConstructPayment(from, to string, fee, amount uint64, note []byte, closeTo string) (transactions.Transaction, error) { +// if the lastValid is 0, firstValid + maxTxnLifetime will be used +// if the firstValid is 0, lastRound + 1 will be used +func (c *Client) ConstructPayment(from, to string, fee, amount uint64, note []byte, closeTo string, firstValid, lastValid basics.Round) (transactions.Transaction, error) { fromAddr, err := basics.UnmarshalChecksumAddress(from) if err != nil { return transactions.Transaction{}, err @@ -484,15 +490,25 @@ func (c *Client) ConstructPayment(from, to string, fee, amount uint64, note []by return transactions.Transaction{}, err } - round := params.LastRound cp := config.Consensus[protocol.ConsensusVersion(params.ConsensusVersion)] + if firstValid == 0 && lastValid == 0 { + firstValid = basics.Round(params.LastRound + 1) + lastValid = firstValid + basics.Round(cp.MaxTxnLife) + } else if firstValid != 0 && lastValid == 0 { + lastValid = firstValid + basics.Round(cp.MaxTxnLife) + } else if firstValid > lastValid { + return transactions.Transaction{}, fmt.Errorf("cannot construct payment: txn would first be valid on round %d which is after last valid round %d", firstValid, lastValid) + } else if lastValid-firstValid > basics.Round(cp.MaxTxnLife) { + return transactions.Transaction{}, fmt.Errorf("cannot construct payment: txn validity period ( %d to %d ) is greater than protocol max txn lifetime %d ", firstValid, lastValid, cp.MaxTxnLife) + } + tx := transactions.Transaction{ Type: protocol.PaymentTx, Header: transactions.Header{ Sender: fromAddr, Fee: basics.MicroAlgos{Raw: fee}, - FirstValid: basics.Round(round), - LastValid: basics.Round(round) + basics.Round(cp.MaxTxnLife), + FirstValid: firstValid, + LastValid: lastValid, Note: note, }, PaymentTxnFields: transactions.PaymentTxnFields{ diff --git a/libgoal/unencryptedWallet.go b/libgoal/unencryptedWallet.go index 943e0c20cb..2fce2396ec 100644 --- a/libgoal/unencryptedWallet.go +++ b/libgoal/unencryptedWallet.go @@ -35,7 +35,7 @@ func (c *Client) SendPaymentFromUnencryptedWallet(from, to string, fee, amount u return transactions.Transaction{}, err } - return c.SendPaymentFromWallet(wh, nil, from, to, fee, amount, note, "") + return c.SendPaymentFromWallet(wh, nil, from, to, fee, amount, note, "", 0, 0) } // GetUnencryptedWalletHandle returns the unencrypted wallet handle. If there diff --git a/test/e2e-go/cli/perf/payment_test.go b/test/e2e-go/cli/perf/payment_test.go index 0aeb23c96e..c16293430c 100644 --- a/test/e2e-go/cli/perf/payment_test.go +++ b/test/e2e-go/cli/perf/payment_test.go @@ -58,7 +58,7 @@ func BenchmarkSendPayment(b *testing.B) { for i := 0; i < b.N; i++ { var nonce [8]byte crypto.RandBytes(nonce[:]) - tx, err = c.ConstructPayment(addr, addr, 1, 1, nonce[:], "") + tx, err = c.ConstructPayment(addr, addr, 1, 1, nonce[:], "", 0, 0) require.NoError(b, err) } }) @@ -74,7 +74,7 @@ func BenchmarkSendPayment(b *testing.B) { for i := 0; i < b.N; i++ { var nonce [8]byte crypto.RandBytes(nonce[:]) - _, err := c.SendPaymentFromWallet(wallet, nil, addr, addr, 1, 1, nonce[:], "") + _, err := c.SendPaymentFromWallet(wallet, nil, addr, addr, 1, 1, nonce[:], "", 0, 0) require.NoError(b, err) } }) diff --git a/test/e2e-go/features/multisig/multisig_test.go b/test/e2e-go/features/multisig/multisig_test.go index b13ca6d791..c09f8564ee 100644 --- a/test/e2e-go/features/multisig/multisig_test.go +++ b/test/e2e-go/features/multisig/multisig_test.go @@ -71,7 +71,7 @@ func TestBasicMultisig(t *testing.T) { fixture.SendMoneyAndWait(curStatus.LastRound, amountToFund, minTxnFee, fundingAddr, multisigAddr) // try to transact with 1 of 3 amountToSend := minAcctBalance - unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, nil, "") + unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, nil, "", 0, 0) r.NoError(err, "Unexpected error when constructing payment transaction") emptyPartial := crypto.MultisigSig{} emptySignature := crypto.Signature{} @@ -90,7 +90,7 @@ func TestBasicMultisig(t *testing.T) { r.True(fixture.WaitForTxnConfirmation(curStatus.LastRound+uint64(5), multisigAddr, txid)) // Need a new txid to avoid dup detection - unsignedTransaction, err = client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, []byte("foobar"), "") + unsignedTransaction, err = client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, []byte("foobar"), "", 0, 0) r.NoError(err, "Unexpected error when constructing payment transaction") signatureWithOne, err = client.UnencryptedMultisigSignTransaction(unsignedTransaction, addrs[0], emptyPartial) r.NoError(err, "first signing returned error") @@ -196,7 +196,7 @@ func TestDuplicateKeys(t *testing.T) { fixture.SendMoneyAndWait(curStatus.LastRound, amountToFund, txnFee, fundingAddr, multisigAddr) // try to transact with "1" signature (though, this is a signature from "every" member of the multisig) amountToSend := minAcctBalance - unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], txnFee, amountToSend, nil, "") + unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], txnFee, amountToSend, nil, "", 0, 0) r.NoError(err, "Unexpected error when constructing payment transaction") emptyPartial := crypto.MultisigSig{} emptySignature := crypto.Signature{} diff --git a/test/e2e-go/features/participation/onlineOfflineParticipation_test.go b/test/e2e-go/features/participation/onlineOfflineParticipation_test.go index 35a9144c6b..cf7956deef 100644 --- a/test/e2e-go/features/participation/onlineOfflineParticipation_test.go +++ b/test/e2e-go/features/participation/onlineOfflineParticipation_test.go @@ -59,7 +59,7 @@ func TestParticipationKeyOnlyAccountParticipatesCorrectly(t *testing.T) { amountToSend := uint64(10000) // arbitrary wh, err := client.GetUnencryptedWalletHandle() a.NoError(err, "should get unencrypted wallet handle") - _, err = client.SendPaymentFromWallet(wh, nil, partkeyOnlyAccount, richAccount, amountToSend, transactionFee, nil, "") + _, err = client.SendPaymentFromWallet(wh, nil, partkeyOnlyAccount, richAccount, amountToSend, transactionFee, nil, "", basics.Round(0), basics.Round(0)) a.Error(err, "attempt to send money from partkey-only account should be treated as though wallet is not controlled") // partkeyonly_account attempts to go offline, should fail (no rootkey to sign txn with) goOfflineUTx, err := client.MakeUnsignedGoOfflineTx(partkeyOnlyAccount, 0, 0, transactionFee) diff --git a/test/e2e-go/features/transactions/close_account_test.go b/test/e2e-go/features/transactions/close_account_test.go index 0b24f26fca..74656076c2 100644 --- a/test/e2e-go/features/transactions/close_account_test.go +++ b/test/e2e-go/features/transactions/close_account_test.go @@ -62,7 +62,7 @@ func TestAccountsCanClose(t *testing.T) { a.NoError(err) fixture.WaitForConfirmedTxn(status.LastRound+10, baseAcct, tx.ID().String()) - tx, err = client.SendPaymentFromWallet(walletHandle, nil, acct0, acct1, 1, 100000, nil, acct2) + tx, err = client.SendPaymentFromWallet(walletHandle, nil, acct0, acct1, 1, 100000, nil, acct2, 0, 0) a.NoError(err) fixture.WaitForConfirmedTxn(status.LastRound+10, acct0, tx.ID().String()) diff --git a/test/e2e-go/restAPI/restClient_test.go b/test/e2e-go/restAPI/restClient_test.go index 39ac93b0c4..e4972710f3 100644 --- a/test/e2e-go/restAPI/restClient_test.go +++ b/test/e2e-go/restAPI/restClient_test.go @@ -193,7 +193,7 @@ func TestTransactionsByAddr(t *testing.T) { t.Error("no addr with funds") } toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) - tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "") + tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "", 0, 0) require.NoError(t, err) txID := tx.ID() rnd, err := testClient.Status() @@ -256,7 +256,7 @@ func TestClientRejectsBadFromAddressWhenSending(t *testing.T) { require.NoError(t, err) badAccountAddress := "This is absolutely not a valid account address." goodAccountAddress := addresses[0] - _, err = testClient.SendPaymentFromWallet(wh, nil, badAccountAddress, goodAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, badAccountAddress, goodAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -269,7 +269,7 @@ func TestClientRejectsBadToAddressWhenSending(t *testing.T) { require.NoError(t, err) badAccountAddress := "This is absolutely not a valid account address." goodAccountAddress := addresses[0] - _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, badAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, badAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -289,7 +289,7 @@ func TestClientRejectsMutatedFromAddressWhenSending(t *testing.T) { require.NoError(t, err) } mutatedAccountAddress := mutateStringAtIndex(unmutatedAccountAddress, 0) - _, err = testClient.SendPaymentFromWallet(wh, nil, mutatedAccountAddress, goodAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, mutatedAccountAddress, goodAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -309,7 +309,7 @@ func TestClientRejectsMutatedToAddressWhenSending(t *testing.T) { require.NoError(t, err) } mutatedAccountAddress := mutateStringAtIndex(unmutatedAccountAddress, 0) - _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, mutatedAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, mutatedAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -322,7 +322,7 @@ func TestClientRejectsSendingMoneyFromAccountForWhichItHasNoKey(t *testing.T) { require.NoError(t, err) goodAccountAddress := addresses[0] nodeDoesNotHaveKeyForThisAddress := "NJY27OQ2ZXK6OWBN44LE4K43TA2AV3DPILPYTHAJAMKIVZDWTEJKZJKO4A" - _, err = testClient.SendPaymentFromWallet(wh, nil, nodeDoesNotHaveKeyForThisAddress, goodAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, nodeDoesNotHaveKeyForThisAddress, goodAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -344,7 +344,7 @@ func TestClientOversizedNote(t *testing.T) { } maxTxnNoteBytes := config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnNoteBytes note := make([]byte, maxTxnNoteBytes+1) - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, note, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, note, "", 0, 0) require.Error(t, err) } @@ -363,7 +363,7 @@ func TestClientCanSendAndGetNote(t *testing.T) { toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) maxTxnNoteBytes := config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnNoteBytes note := make([]byte, maxTxnNoteBytes) - tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, note, "") + tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, note, "", 0, 0) require.NoError(t, err) txStatus, err := waitForTransaction(t, testClient, someAddress, tx.ID().String(), 15*time.Second) require.NoError(t, err) @@ -383,7 +383,7 @@ func TestClientCanGetTransactionStatus(t *testing.T) { t.Error("no addr with funds") } toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) - tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "") + tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "", 0, 0) t.Log(string(protocol.EncodeJSON(tx))) require.NoError(t, err) t.Log(tx.ID().String()) @@ -430,22 +430,22 @@ func TestSendingTooMuchFails(t *testing.T) { fromBalance, err := testClient.GetBalance(fromAddress) require.NoError(t, err) // too much amount - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, fromBalance+100, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, fromBalance+100, nil, "", 0, 0) t.Log(err) require.Error(t, err) // waaaay too much amount - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, math.MaxUint64, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, math.MaxUint64, nil, "", 0, 0) t.Log(err) require.Error(t, err) // too much fee - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, fromBalance+100, 10000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, fromBalance+100, 10000, nil, "", 0, 0) t.Log(err) require.Error(t, err) // waaaay too much fee - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, math.MaxUint64, 10000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, math.MaxUint64, 10000, nil, "", 0, 0) t.Log(err) require.Error(t, err) } @@ -482,7 +482,7 @@ func TestSendingFromEmptyAccountFails(t *testing.T) { toAddress, err = testClient.GenerateAddress(wh) require.NoError(t, err) } - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -511,7 +511,7 @@ func TestSendingTooLittleToEmptyAccountFails(t *testing.T) { if someAddress == "" { t.Error("no addr with funds") } - _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, 1, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, 1, nil, "", 0, 0) require.Error(t, err) } @@ -531,7 +531,7 @@ func TestSendingLowFeeFails(t *testing.T) { t.Errorf("balance too low %d < %d", someBal, sendAmount) } toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) - utx, err := testClient.ConstructPayment(someAddress, toAddress, 1, sendAmount, nil, "") + utx, err := testClient.ConstructPayment(someAddress, toAddress, 1, sendAmount, nil, "", 0, 0) require.NoError(t, err) utx.Fee.Raw = 1 stx, err := testClient.SignTransactionWithWallet(wh, nil, utx) @@ -586,7 +586,7 @@ func TestSendingNotClosingAccountFails(t *testing.T) { t.Error("no addr with funds") } amt := someBal - 10000 - 1 - _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, amt, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, amt, nil, "", 0, 0) require.Error(t, err) } diff --git a/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go b/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go index 8ace23ae54..acd2666735 100644 --- a/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go +++ b/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go @@ -36,7 +36,7 @@ func cascadeCreateAndFundAccounts(amountToSend, transactionFee uint64, fundingAc a.NoError(err, "should be able to get unencrypted wallet handle") newAddress, err := client.GenerateAddress(wh) a.NoError(err, "should be able to generate new address") - tx, err := client.SendPaymentFromWallet(wh, nil, fundingAccount, newAddress, transactionFee, amountToSend, nil, "") + tx, err := client.SendPaymentFromWallet(wh, nil, fundingAccount, newAddress, transactionFee, amountToSend, nil, "", 0, 0) a.NoError(err, "should be no errors when funding new accounts, send number %v", i) i++ outputTxidsToAccounts[tx.ID().String()] = newAddress diff --git a/test/framework/fixtures/restClientFixture.go b/test/framework/fixtures/restClientFixture.go index 8be8d6d1fa..de7ab81131 100644 --- a/test/framework/fixtures/restClientFixture.go +++ b/test/framework/fixtures/restClientFixture.go @@ -276,7 +276,7 @@ func (f *RestClientFixture) SendMoneyAndWait(curRound, amountToSend, transaction // SendMoneyAndWaitFromWallet is as above, but for a specific wallet func (f *RestClientFixture) SendMoneyAndWaitFromWallet(walletHandle, walletPassword []byte, curRound, amountToSend, transactionFee uint64, fromAccount, toAccount string) (fundingTxid string) { client := f.LibGoalClient - fundingTx, err := client.SendPaymentFromWallet(walletHandle, walletPassword, fromAccount, toAccount, transactionFee, amountToSend, nil, "") + fundingTx, err := client.SendPaymentFromWallet(walletHandle, walletPassword, fromAccount, toAccount, transactionFee, amountToSend, nil, "", 0, 0) require.NoError(f.t, err, "client should be able to send money from rich to poor account") require.NotEmpty(f.t, fundingTx.ID().String(), "transaction ID should not be empty") waitingDeadline := curRound + uint64(5) From 0e9aa9e62c6ece9034062a7135bb1fc31558589e Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Wed, 19 Jun 2019 22:40:51 -0400 Subject: [PATCH 42/54] [GOAL2-782] Export DNS configuration (#43) * Add support for zone exporting. * Add space between functions. --- cmd/algons/dnsCmd.go | 130 +++++++++++++++++++++++-- tools/network/cloudflare/cloudflare.go | 84 ++++++++++++++-- tools/network/cloudflare/zones.go | 97 ++++++++++++++++++ 3 files changed, 297 insertions(+), 14 deletions(-) create mode 100644 tools/network/cloudflare/zones.go diff --git a/cmd/algons/dnsCmd.go b/cmd/algons/dnsCmd.go index 8083c60e1b..c9023faf96 100644 --- a/cmd/algons/dnsCmd.go +++ b/cmd/algons/dnsCmd.go @@ -20,6 +20,7 @@ import ( "bufio" "context" "fmt" + "io/ioutil" "net" "os" "regexp" @@ -40,6 +41,8 @@ var ( recordType string noPrompt bool excludePattern string + exportNetwork string + outputFilename string ) func init() { @@ -47,6 +50,10 @@ func init() { dnsCmd.AddCommand(addCmd) dnsCmd.AddCommand(deleteCmd) dnsCmd.AddCommand(listCmd) + dnsCmd.AddCommand(exportCmd) + + listCmd.AddCommand(listRecordsCmd) + listCmd.AddCommand(listZonesCmd) addCmd.Flags().StringVarP(&addFromName, "from", "f", "", "From name to add new DNS entry") addCmd.MarkFlagRequired("from") @@ -58,9 +65,13 @@ func init() { deleteCmd.Flags().BoolVarP(&noPrompt, "no-prompt", "y", false, "No prompting for records deletion") deleteCmd.Flags().StringVarP(&excludePattern, "exclude", "e", "", "name records exclude pattern") - listCmd.Flags().StringVarP(&listNetwork, "network", "n", "", "Domain name for records to list") - listCmd.Flags().StringVarP(&recordType, "recordType", "t", "", "DNS record type to list (A, CNAME, SRV)") - listCmd.MarkFlagRequired("network") + listRecordsCmd.Flags().StringVarP(&listNetwork, "network", "n", "", "Domain name for records to list") + listRecordsCmd.Flags().StringVarP(&recordType, "recordType", "t", "", "DNS record type to list (A, CNAME, SRV)") + listRecordsCmd.MarkFlagRequired("network") + + exportCmd.Flags().StringVarP(&exportNetwork, "network", "n", "", "Domain name to export") + exportCmd.MarkFlagRequired("network") + exportCmd.Flags().StringVarP(&outputFilename, "zonefile", "z", "", "Output file for backup ( intead of outputing it to stdout ) ") } type byIP []net.IP @@ -81,8 +92,17 @@ var dnsCmd = &cobra.Command{ var listCmd = &cobra.Command{ Use: "list", - Short: "List the DNS/SRV entries of the given network", - Long: "List the DNS/SRV entries of the given network", + Short: "List the A/SRV/Zones entries of the given network", + Long: "List the A/SRV/Zones entries of the given network", + Run: func(cmd *cobra.Command, args []string) { + cmd.HelpFunc()(cmd, args) + }, +} + +var listRecordsCmd = &cobra.Command{ + Use: "records", + Short: "List the A/SRV entries of the given network", + Long: "List the A/SRV entries of the given network", Run: func(cmd *cobra.Command, args []string) { recordType = strings.ToUpper(recordType) if recordType == "" || recordType == "A" || recordType == "CNAME" || recordType == "SRV" { @@ -94,6 +114,17 @@ var listCmd = &cobra.Command{ }, } +var listZonesCmd = &cobra.Command{ + Use: "zones", + Short: "List the zones", + Long: "List the zones", + Run: func(cmd *cobra.Command, args []string) { + if !doListZones() { + os.Exit(1) + } + }, +} + var checkCmd = &cobra.Command{ Use: "check", Short: "Check the status", @@ -140,6 +171,16 @@ var deleteCmd = &cobra.Command{ }, } +var exportCmd = &cobra.Command{ + Use: "export", + Short: "Export DNS record entries for a specified network", + Run: func(cmd *cobra.Command, args []string) { + if !doExportZone(exportNetwork, outputFilename) { + os.Exit(1) + } + }, +} + func doAddDNS(from string, to string) (err error) { cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() if err != nil { @@ -166,11 +207,23 @@ func doAddDNS(from string, to string) (err error) { return } -func getClouldflareCredentials() (zoneID string, email string, authKey string, err error) { - zoneID = os.Getenv("CLOUDFLARE_ZONE_ID") +func getClouldflareAuthCredentials() (email string, authKey string, err error) { email = os.Getenv("CLOUDFLARE_EMAIL") authKey = os.Getenv("CLOUDFLARE_AUTH_KEY") - if zoneID == "" || email == "" || authKey == "" { + if email == "" || authKey == "" { + err = fmt.Errorf("one or more credentials missing from ENV") + } + return +} + +func getClouldflareCredentials() (zoneID string, email string, authKey string, err error) { + email, authKey, err = getClouldflareAuthCredentials() + if err != nil { + return + } + + zoneID = os.Getenv("CLOUDFLARE_ZONE_ID") + if zoneID == "" { err = fmt.Errorf("one or more credentials missing from ENV") } return @@ -333,3 +386,64 @@ func listEntries(listNetwork string, recordType string) { } } } + +func doExportZone(network string, outputFilename string) bool { + cfEmail, cfKey, err := getClouldflareAuthCredentials() + if err != nil { + fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) + return false + } + cloudflareCred := cloudflare.NewCred(cfEmail, cfKey) + zones, err := cloudflareCred.GetZones(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error retrieving zones entries: %v\n", err) + return false + } + zoneID := "" + // find a zone that matches the requested network name. + for _, z := range zones { + if z.DomainName == network { + zoneID = z.ZoneID + break + } + fmt.Printf("%s : %s\n", z.DomainName, z.ZoneID) + } + if zoneID == "" { + fmt.Fprintf(os.Stderr, "No matching zoneID was found for %s\n", network) + return false + } + cloudflareDNS := cloudflare.NewDNS(zoneID, cfEmail, cfKey) + exportedZone, err := cloudflareDNS.ExportZone(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to export zone : %v\n", err) + return false + } + if outputFilename != "" { + err = ioutil.WriteFile(outputFilename, exportedZone, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to write exported zone file : %v\n", err) + return false + } + } else { + fmt.Fprint(os.Stdout, string(exportedZone)) + } + return true +} + +func doListZones() bool { + cfEmail, cfKey, err := getClouldflareAuthCredentials() + if err != nil { + fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) + return false + } + cloudflareCred := cloudflare.NewCred(cfEmail, cfKey) + zones, err := cloudflareCred.GetZones(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error listing zones entries: %v\n", err) + return false + } + for _, z := range zones { + fmt.Printf("%s : %s\n", z.DomainName, z.ZoneID) + } + return true +} diff --git a/tools/network/cloudflare/cloudflare.go b/tools/network/cloudflare/cloudflare.go index f62193706c..a4be998911 100644 --- a/tools/network/cloudflare/cloudflare.go +++ b/tools/network/cloudflare/cloudflare.go @@ -19,6 +19,7 @@ package cloudflare import ( "context" "fmt" + "io/ioutil" "net/http" "strings" ) @@ -29,19 +30,34 @@ const ( AutomaticTTL = 1 ) -// DNS is the cloudflare package main access class. Initiate an instance of this class to access the clouldflare APIs. -type DNS struct { - zoneID string +// Cred contains the credentials used to authenticate with the cloudflare API. +type Cred struct { authEmail string authKey string } +// DNS is the cloudflare package main access class. Initiate an instance of this class to access the clouldflare APIs. +type DNS struct { + zoneID string + Cred +} + +// NewCred creates a new credential structure used to authenticate with the cloudflare service. +func NewCred(authEmail string, authKey string) *Cred { + return &Cred{ + authEmail: authEmail, + authKey: authKey, + } +} + // NewDNS create a new instance of clouldflare DNS services class func NewDNS(zoneID string, authEmail string, authKey string) *DNS { return &DNS{ - zoneID: zoneID, - authEmail: authEmail, - authKey: authKey, + zoneID: zoneID, + Cred: Cred{ + authEmail: authEmail, + authKey: authKey, + }, } } @@ -241,3 +257,59 @@ func (d *DNS) UpdateSRVRecord(ctx context.Context, recordID string, name string, } return nil } + +// Zone represent a single zone on the cloudflare API. +type Zone struct { + DomainName string + ZoneID string +} + +// GetZones returns a list of zones that are associated with cloudflare. +func (c *Cred) GetZones(ctx context.Context) (zones []Zone, err error) { + request, err := getZonesRequest(c.authEmail, c.authKey) + if err != nil { + return nil, err + } + client := &http.Client{} + response, err := client.Do(request.WithContext(ctx)) + if err != nil { + return nil, err + } + + parsedResponse, err := parseGetZonesResponse(response) + if err != nil { + return nil, err + } + if parsedResponse.Success == false { + return nil, fmt.Errorf("failed to retrieve zone records : %v", parsedResponse) + } + + for _, z := range parsedResponse.Result { + zones = append(zones, + Zone{ + DomainName: z.Name, + ZoneID: z.ID, + }, + ) + } + return zones, err +} + +// ExportZone exports the zone into a BIND config bytes array +func (d *DNS) ExportZone(ctx context.Context) (exportedZoneBytes []byte, err error) { + request, err := exportZoneRequest(d.zoneID, d.authEmail, d.authKey) + if err != nil { + return nil, err + } + client := &http.Client{} + response, err := client.Do(request.WithContext(ctx)) + if err != nil { + return nil, err + } + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + return body, nil +} diff --git a/tools/network/cloudflare/zones.go b/tools/network/cloudflare/zones.go new file mode 100644 index 0000000000..944e0ed546 --- /dev/null +++ b/tools/network/cloudflare/zones.go @@ -0,0 +1,97 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package cloudflare + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/url" +) + +func getZonesRequest(authEmail, authKey string) (*http.Request, error) { + // construct the query + requestURI, err := url.Parse(cloudFlareURI) + if err != nil { + return nil, err + } + requestURI.Path = requestURI.Path + "zones" + request, err := http.NewRequest("GET", requestURI.String(), nil) + if err != nil { + return nil, err + } + addHeaders(request, authEmail, authKey) + return request, nil +} + +// GetZonesResult is the JSON response for a DNS create request +type GetZonesResult struct { + Success bool `json:"success"` + Errors []interface{} `json:"errors"` + Messages []interface{} `json:"messages"` + Result []GetZonesResultItem `json:"result"` + ResultInfo GetZonesResultPage `json:"result_info"` +} + +// GetZonesResultPage is the result of the response for the DNS create request +type GetZonesResultPage struct { + Page int `json:"page"` + PerPage int `json:"per_page"` + TotalPages int `json:"total_pages"` + Count int `json:"count"` + TotalCount int `json:"total_count"` +} + +// GetZonesResultItem is the result of the response for the DNS create request +type GetZonesResultItem struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Paused bool `json:"paused"` + Type string `json:"type"` + DevelopmentMode int `json:"development_mode"` + NameServers []string `json:"name_servers"` + OriginalNameServers []string `json:"original_name_servers"` +} + +func parseGetZonesResponse(response *http.Response) (*GetZonesResult, error) { + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + var parsedReponse GetZonesResult + if err := json.Unmarshal(body, &parsedReponse); err != nil { + return nil, err + } + return &parsedReponse, nil +} + +func exportZoneRequest(zoneID, authEmail, authKey string) (*http.Request, error) { + // construct the query + requestURI, err := url.Parse(cloudFlareURI) + if err != nil { + return nil, err + } + requestURI.Path = requestURI.Path + "zones/" + zoneID + "/dns_records/export" + request, err := http.NewRequest("GET", requestURI.String(), nil) + if err != nil { + return nil, err + } + addHeaders(request, authEmail, authKey) + return request, nil +} From d2da8a6cd3cc809292da1e135db6ce6477d68abc Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 19 Jun 2019 22:41:22 -0400 Subject: [PATCH 43/54] Add metrics to heartbeat event. (#7) --- cmd/algod/main.go | 22 ++++++++++++-- logging/telemetryspec/event.go | 5 ++++ network/wsNetwork.go | 14 +++++++++ util/metrics/counter.go | 20 +++++++++++++ util/metrics/gauge.go | 16 +++++++++++ util/metrics/registry.go | 9 ++++++ util/metrics/registryCommon.go | 1 + util/metrics/registry_test.go | 46 +++++++++++++++++++++++++++++ util/metrics/stringGauge.go | 48 +++++++++++++++++++++++++++++++ util/metrics/stringGaugeCommon.go | 11 +++++++ util/metrics/stringGauge_test.go | 36 +++++++++++++++++++++++ 11 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 util/metrics/registry_test.go create mode 100644 util/metrics/stringGauge.go create mode 100644 util/metrics/stringGaugeCommon.go create mode 100644 util/metrics/stringGauge_test.go diff --git a/cmd/algod/main.go b/cmd/algod/main.go index 3141559da8..c2fa2535a4 100644 --- a/cmd/algod/main.go +++ b/cmd/algod/main.go @@ -36,6 +36,7 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/metrics" "github.com/algorand/go-algorand/util/tokens" ) @@ -71,13 +72,20 @@ func main() { rand.Seed(time.Now().UnixNano()) } + version := config.GetCurrentVersion() if *versionCheck { - version := config.GetCurrentVersion() fmt.Printf("%d\n%s.%s [%s] (commit #%s)\n%s\n", version.AsUInt64(), version.String(), version.Channel, version.Branch, version.GetCommitHash(), config.GetLicenseInfo()) return } + heartbeatGauge := metrics.MakeStringGauge() + heartbeatGauge.Set("version", version.String()) + heartbeatGauge.Set("version-num", strconv.FormatUint(version.AsUInt64(), 10)) + heartbeatGauge.Set("channel", version.Channel) + heartbeatGauge.Set("branch", version.Branch) + heartbeatGauge.Set("commit-hash", version.GetCommitHash()) + if *branchCheck { fmt.Println(config.Branch) return @@ -183,10 +191,18 @@ func main() { log.EventWithDetails(telemetryspec.ApplicationState, telemetryspec.StartupEvent, startupDetails) // Send a heartbeat event every 10 minutes as a sign of life + ticker := time.NewTicker(10 * time.Minute) go func() { + values := make(map[string]string) for { - log.Event(telemetryspec.ApplicationState, telemetryspec.HeartbeatEvent) - <-time.After(10 * time.Minute) + metrics.DefaultRegistry().AddMetrics(values) + + heartbeatDetails := telemetryspec.HeartbeatEventDetails{ + Metrics: values, + } + + log.EventWithDetails(telemetryspec.ApplicationState, telemetryspec.HeartbeatEvent, heartbeatDetails) + <-ticker.C } }() } diff --git a/logging/telemetryspec/event.go b/logging/telemetryspec/event.go index d652f216ef..8395dd98fc 100644 --- a/logging/telemetryspec/event.go +++ b/logging/telemetryspec/event.go @@ -41,6 +41,11 @@ type StartupEventDetails struct { // HeartbeatEvent is sent periodically to indicate node is running const HeartbeatEvent Event = "Heartbeat" +// HeartbeatEventDetails contains details for the StartupEvent +type HeartbeatEventDetails struct { + Metrics map[string]string +} + // CatchupStartEvent event const CatchupStartEvent Event = "CatchupStart" diff --git a/network/wsNetwork.go b/network/wsNetwork.go index 849a628eb7..05c9ee159a 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -120,6 +120,10 @@ var meanPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_me var medianPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_median_ping_seconds", Description: "Network round trip time to median peer in seconds."}) var maxPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_max_ping_seconds", Description: "Network round trip time to slowest peer in seconds."}) +var peers = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peers", Description: "Number of active peers."}) +var incomingPeers = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_incoming_peers", Description: "Number of active incoming peers."}) +var outgoingPeers = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_outgoing_peers", Description: "Number of active outgoing peers."}) + // Peer opaque interface for referring to a neighbor in the network type Peer interface{} @@ -879,6 +883,9 @@ func (wn *WebsocketNetwork) ServeHTTP(response http.ResponseWriter, request *htt Incoming: true, InstanceName: otherInstanceName, }) + + peers.Set(float64(wn.NumPeers()), nil) + incomingPeers.Set(float64(wn.numIncomingPeers()), nil) } func (wn *WebsocketNetwork) messageHandlerThread() { @@ -1498,6 +1505,9 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { InstanceName: myInstanceName, }) + peers.Set(float64(wn.NumPeers()), nil) + outgoingPeers.Set(float64(wn.numOutgoingPeers()), nil) + if wn.prioScheme != nil { challenge := response.Header.Get(PriorityChallengeHeader) if challenge != "" { @@ -1573,6 +1583,10 @@ func (wn *WebsocketNetwork) removePeer(peer *wsPeer, reason disconnectReason) { Reason: string(reason), }) + peers.Set(float64(wn.NumPeers()), nil) + incomingPeers.Set(float64(wn.numIncomingPeers()), nil) + outgoingPeers.Set(float64(wn.numOutgoingPeers()), nil) + wn.peersLock.Lock() defer wn.peersLock.Unlock() if peer.peerIndex < len(wn.peers) && wn.peers[peer.peerIndex] == peer { diff --git a/util/metrics/counter.go b/util/metrics/counter.go index fde3d4fbd6..850be3a630 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -172,3 +172,23 @@ func (counter *Counter) WriteMetric(buf *strings.Builder, parentLabels string) { buf.WriteString("\n") } } + +// AddMetric adds the metric into the map +func (counter *Counter) AddMetric(values map[string]string) { + counter.Lock() + defer counter.Unlock() + + if len(counter.values) < 1 { + return + } + + for _, l := range counter.values { + sum := l.counter + if len(l.labels) == 0 { + sum += float64(atomic.LoadUint64(&counter.intValue)) + } + + values[counter.name] = strconv.FormatFloat(sum, 'f', -1, 32) + } + +} diff --git a/util/metrics/gauge.go b/util/metrics/gauge.go index f9aec394d4..b0925baefe 100644 --- a/util/metrics/gauge.go +++ b/util/metrics/gauge.go @@ -184,3 +184,19 @@ func (gauge *Gauge) WriteMetric(buf *strings.Builder, parentLabels string) { buf.WriteString("\n") } } + +// AddMetric adds the metric into the map +func (gauge *Gauge) AddMetric(values map[string]string) { + gauge.Lock() + defer gauge.Unlock() + + gauge.filterExpiredMetrics() + + if len(gauge.valuesIndices) < 1 { + return + } + + for _, l := range gauge.valuesIndices { + values[gauge.name] = strconv.FormatFloat(l.gauge, 'f', -1, 32) + } +} diff --git a/util/metrics/registry.go b/util/metrics/registry.go index 66018266c4..24d57a7094 100644 --- a/util/metrics/registry.go +++ b/util/metrics/registry.go @@ -66,3 +66,12 @@ func (r *Registry) WriteMetrics(buf *strings.Builder, parentLabels string) { m.WriteMetric(buf, parentLabels) } } + +// AddMetrics will add all the metrics that were registered to this registry +func (r *Registry) AddMetrics(values map[string]string) { + r.metricsMu.Lock() + defer r.metricsMu.Unlock() + for _, m := range r.metrics { + m.AddMetric(values) + } +} diff --git a/util/metrics/registryCommon.go b/util/metrics/registryCommon.go index bcf3fe9897..49db57f131 100644 --- a/util/metrics/registryCommon.go +++ b/util/metrics/registryCommon.go @@ -25,6 +25,7 @@ import ( // Metric represent any collectable metric type Metric interface { WriteMetric(buf *strings.Builder, parentLabels string) + AddMetric(values map[string]string) } // Registry represents a single set of metrics registry diff --git a/util/metrics/registry_test.go b/util/metrics/registry_test.go new file mode 100644 index 0000000000..b9c6c2d7d9 --- /dev/null +++ b/util/metrics/registry_test.go @@ -0,0 +1,46 @@ +// +build telemetry + +package metrics + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWriteAdd(t *testing.T) { + // Test AddMetrics and WriteMetrics with a counter + counter := MakeCounter(MetricName{Name: "gauge-name", Description: "gauge description"}) + counter.Add(12.34, nil) + + results := make(map[string]string) + DefaultRegistry().AddMetrics(results) + + require.Equal(t, 1, len(results)) + require.True(t, hasKey(results, "gauge-name")) + require.Equal(t, "12.34", results["gauge-name"]) + + bufBefore := strings.Builder{} + DefaultRegistry().WriteMetrics(&bufBefore, "label") + require.True(t, bufBefore.Len() > 0) + + // Test that WriteMetrics does not change after adding a StringGauge + stringGauge := MakeStringGauge() + stringGauge.Set("string-key", "value") + + DefaultRegistry().AddMetrics(results) + + require.True(t, hasKey(results, "string-key")) + require.Equal(t, "value", results["string-key"]) + require.True(t, hasKey(results, "gauge-name")) + require.Equal(t, "12.34", results["gauge-name"]) + + // not included in string builder + bufAfter := strings.Builder{} + DefaultRegistry().WriteMetrics(&bufAfter, "label") + require.Equal(t, bufBefore.String(), bufAfter.String()) + + stringGauge.Deregister(nil) + counter.Deregister(nil) +} diff --git a/util/metrics/stringGauge.go b/util/metrics/stringGauge.go new file mode 100644 index 0000000000..accd2b1618 --- /dev/null +++ b/util/metrics/stringGauge.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "strings" +) + +// MakeStringGauge create a new StringGauge. +func MakeStringGauge() *StringGauge { + c := &StringGauge{ + values: make(map[string]string), + } + c.Register(nil) + return c +} + +// Register registers the StringGauge with the default/specific registry +func (stringGauge *StringGauge) Register(reg *Registry) { + if reg == nil { + DefaultRegistry().Register(stringGauge) + } else { + reg.Register(stringGauge) + } +} + +// Deregister deregisters the StringGauge with the default/specific registry +func (stringGauge *StringGauge) Deregister(reg *Registry) { + if reg == nil { + DefaultRegistry().Deregister(stringGauge) + } else { + reg.Deregister(stringGauge) + } +} + +// Set updates a key with a value. +func (stringGauge *StringGauge) Set(key string, value string) { + stringGauge.values[key] = value +} + +// WriteMetric omit string gauges from the metrics report, not sure how they act with prometheus +func (stringGauge *StringGauge) WriteMetric(buf *strings.Builder, parentLabels string) { +} + +// AddMetric sets all the key value pairs in the provided map. +func (stringGauge *StringGauge) AddMetric(values map[string]string) { + for k, v := range stringGauge.values { + values[k] = v + } +} diff --git a/util/metrics/stringGaugeCommon.go b/util/metrics/stringGaugeCommon.go new file mode 100644 index 0000000000..c2c1e57f76 --- /dev/null +++ b/util/metrics/stringGaugeCommon.go @@ -0,0 +1,11 @@ +package metrics + +import ( + "github.com/algorand/go-deadlock" +) + +// StringGauge represents a map of key value pairs available to be written with the AddMetric +type StringGauge struct { + deadlock.Mutex + values map[string]string +} diff --git a/util/metrics/stringGauge_test.go b/util/metrics/stringGauge_test.go new file mode 100644 index 0000000000..6d2ae89482 --- /dev/null +++ b/util/metrics/stringGauge_test.go @@ -0,0 +1,36 @@ +package metrics + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func hasKey(data map[string]string, key string) bool { + _, ok := data[key] + return ok +} + +func TestMetricStringGauge(t *testing.T) { + stringGauge := MakeStringGauge() + stringGauge.Set("number-key", "1") + stringGauge.Set("string-key", "value") + + results := make(map[string]string) + DefaultRegistry().AddMetrics(results) + + // values are populated + require.Equal(t, 2, len(results)) + require.True(t, hasKey(results, "number-key")) + require.Equal(t, "1", results["number-key"]) + require.True(t, hasKey(results, "string-key")) + require.Equal(t, "value", results["string-key"]) + + // not included in string builder + buf := strings.Builder{} + DefaultRegistry().WriteMetrics(&buf, "not used") + require.Equal(t, "", buf.String()) + + stringGauge.Deregister(nil) +} From 9e7f2b7f057a5bd39bd562644389263a6b24764d Mon Sep 17 00:00:00 2001 From: David Shoots Date: Thu, 20 Jun 2019 01:43:35 -0400 Subject: [PATCH 44/54] Clean up code Remove extra fields from Relay, dead code, and add copyright headers. --- cmd/algorelay/eb/eb.go | 41 ++++------ cmd/algorelay/relayCmd.go | 153 ++++---------------------------------- 2 files changed, 32 insertions(+), 162 deletions(-) diff --git a/cmd/algorelay/eb/eb.go b/cmd/algorelay/eb/eb.go index 413b6b64a3..02024c4152 100644 --- a/cmd/algorelay/eb/eb.go +++ b/cmd/algorelay/eb/eb.go @@ -1,34 +1,25 @@ -package eb - -import "time" - -const NodeRunnerKind = "NodeRunner" +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . -// NodeRunner entities are synchronized directly from db1 using `afdb1 syncebnoderunners` -// They should not be modified manually -type NodeRunner struct { - CompanyName string - InvestorID string - NbRequiredRelays int // number of relays required relays the investor needs to run in the node agreement - NodeRunnerToken string // token used by eb.algorand.foundation -} - -const RelayKind = "Relay" +package eb type Relay struct { ID int64 // db key injected when loaded - InvestorID string - ContactEmail string // comma separated list of emails - NodeProvider string IPOrDNSName string - Specs string - Notes string - SubmissionTime time.Time - SRVRecordCreationTime time.Time - Telemetry string // GUID[:name] MetricsEnabled bool - CheckTime time.Time // time the check was done CheckSuccess bool // true if check was successful - CheckError string // non-empty if check error, contains the error DNSAlias string // DNS Alias name used } diff --git a/cmd/algorelay/relayCmd.go b/cmd/algorelay/relayCmd.go index 09b906fbce..1b1f2ea1d8 100644 --- a/cmd/algorelay/relayCmd.go +++ b/cmd/algorelay/relayCmd.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package main import ( @@ -496,140 +512,3 @@ func addSRVRecord(srvNetwork string, target string, port uint16, serviceShortNam return cloudflareDNS.SetSRVRecord(context.Background(), srvNetwork, target, cloudflare.AutomaticTTL, priority, uint(port), serviceShortName, "_tcp", weight) } - - -//var addCmd = &cobra.Command{ -// Use: "add", -// Short: "Add a DNS record", -// Long: "Adds a DNS record to map --from to --to, using A if to == IP or CNAME otherwise\n", -// Example: "algons dns add -f a.test.algodev.network -t r1.algodev.network\n" + -// "algons dns add -f a.test.algodev.network -t 192.168.100.10", -// Run: func(cmd *cobra.Command, args []string) { -// err := doAddDNS(addFromName, addToAddress) -// if err != nil { -// fmt.Fprintf(os.Stderr, "Error adding DNS entry: %v\n", err) -// os.Exit(1) -// } else { -// fmt.Printf("DNS Entry Added\n") -// } -// }, -//} -// -//var deleteCmd = &cobra.Command{ -// Use: "delete", -// Short: "Delete DNS and SRV records for a specified network", -// Run: func(cmd *cobra.Command, args []string) { -// if !doDeleteDNS(deleteNetwork, noPrompt, excludePattern) { -// os.Exit(1) -// } -// }, -//} -// - -//func checkDNSRecord(dnsName string) { -// fmt.Printf("------------------------\nDNS Lookup: %s\n", dnsName) -// ips, err := net.LookupIP(dnsName) -// if err != nil { -// fmt.Printf("Cannot resolve %s: %v\n", dnsName, err) -// } else { -// sort.Sort(byIP(ips)) -// for _, ip := range ips { -// fmt.Printf("-> %s\n", ip.String()) -// } -// } -//} -// -//func doDeleteDNS(network string, noPrompt bool, excludePattern string) bool { -// -// if network == "" || network == "testnet" || network == "devnet" || network == "mainnet" { -// fmt.Fprintf(os.Stderr, "Deletion of network '%s' using this tool is not allowed\n", network) -// return false -// } -// -// var excludeRegex *regexp.Regexp -// if excludePattern != "" { -// var err error -// excludeRegex, err = regexp.Compile(excludePattern) -// if err != nil { -// fmt.Fprintf(os.Stderr, "specified regular expression exclude pattern ('%s') is not a valid regular expression : %v", excludePattern, err) -// return false -// } -// } -// -// cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() -// if err != nil { -// fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) -// return false -// } -// -// cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfKey) -// -// idsToDelete := make(map[string]string) // Maps record ID to Name -// -// for _, service := range []string{"_algobootstrap", "_metrics"} { -// records, err := cloudflareDNS.ListDNSRecord(context.Background(), "SRV", service+"._tcp."+network+".algodev.network", "", "", "", "") -// if err != nil { -// fmt.Fprintf(os.Stderr, "Error listing SRV '%s' entries: %v\n", service, err) -// os.Exit(1) -// } -// for _, r := range records { -// if excludeRegex != nil { -// if excludeRegex.MatchString(r.Name) { -// fmt.Printf("Excluding SRV '%s' record: %s\n", service, r.Name) -// continue -// } -// } -// fmt.Printf("Found SRV '%s' record: %s\n", service, r.Name) -// idsToDelete[r.ID] = r.Name -// } -// } -// -// networkSuffix := "." + network + ".algodev.network" -// -// for _, recordType := range []string{"A", "CNAME"} { -// records, err := cloudflareDNS.ListDNSRecord(context.Background(), recordType, "", "", "", "", "") -// if err != nil { -// fmt.Fprintf(os.Stderr, "Error listing DNS '%s' entries: %v\n", recordType, err) -// os.Exit(1) -// } -// for _, r := range records { -// if strings.Index(r.Name, networkSuffix) > 0 { -// if excludeRegex != nil { -// if excludeRegex.MatchString(r.Name) { -// fmt.Printf("Excluding DNS '%s' record: %s\n", recordType, r.Name) -// continue -// } -// } -// fmt.Printf("Found DNS '%s' record: %s\n", recordType, r.Name) -// idsToDelete[r.ID] = r.Name -// } -// } -// } -// -// if len(idsToDelete) == 0 { -// fmt.Printf("No DNS/SRV records found\n") -// return true -// } -// -// var text string -// if !noPrompt { -// reader := bufio.NewReader(os.Stdin) -// fmt.Printf("Delete these %d entries (type 'yes' to delete)? ", len(idsToDelete)) -// text, _ = reader.ReadString('\n') -// text = strings.Replace(text, "\n", "", -1) -// } else { -// text = "yes" -// } -// -// if text == "yes" { -// for id, name := range idsToDelete { -// fmt.Fprintf(os.Stdout, "Deleting %s\n", name) -// err = cloudflareDNS.DeleteDNSRecord(context.Background(), id) -// if err != nil { -// fmt.Fprintf(os.Stderr, " !! error deleting %s: %v\n", name, err) -// } -// } -// } -// return true -//} -// From 1b642f4a15aeeea233b21ac12729a727be828a1e Mon Sep 17 00:00:00 2001 From: Zachary Estep Date: Thu, 20 Jun 2019 07:01:02 -0400 Subject: [PATCH 45/54] Updating the docker file to work (#53) * Updating the docker file to work * Updating dockerfile after review * Slight tweak to dockerfile apt-get line(s) --- docker/Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2bf3908670..15d4a70057 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,15 +1,18 @@ FROM ubuntu:18.04 ENV GOLANG_VERSION 1.12 +ENV DEBIAN_FRONTEND noninteractive + +RUN apt update && apt-get install -y git libboost-all-dev wget sqlite3 autoconf sudo tzdata bsdmainutils -RUN apt update && apt install -y git libboost-all-dev wget sqlite3 autoconf WORKDIR /root RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local ENV GOROOT /usr/local/go -ENV GOPATH $HOME/go +ENV GOPATH /go +ENV GOBIN /go/bin ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH RUN mkdir -p $GOPATH/src/github.com/algorand WORKDIR $GOPATH/src/github.com/algorand RUN git clone https://github.com/algorand/go-algorand WORKDIR $GOPATH/src/github.com/algorand/go-algorand -RUN git checkout master && scripts/configure_dev.sh && make +RUN git checkout master && ./scripts/configure_dev.sh && make install ENTRYPOINT ["/bin/bash"] From 14fdf0b46ab4007636a8db8547be269bbcc584ab Mon Sep 17 00:00:00 2001 From: algobolson <45948765+algobolson@users.noreply.github.com> Date: Thu, 20 Jun 2019 09:43:07 -0400 Subject: [PATCH 46/54] yum-cron runs hourly, don't random sleep more than 60 minutes (#68) --- installer/rpm/yum-cron-algorand.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/installer/rpm/yum-cron-algorand.conf b/installer/rpm/yum-cron-algorand.conf index d48b177347..b35acbeaf8 100644 --- a/installer/rpm/yum-cron-algorand.conf +++ b/installer/rpm/yum-cron-algorand.conf @@ -24,8 +24,7 @@ apply_updates = yes # minutes before running. This is useful for e.g. staggering the # times that multiple systems will access update servers. If # random_sleep is 0 or negative, the program will run immediately. -# 6*60 = 360 -random_sleep = 360 +random_sleep = 57 [emitters] From 1e099cfbe70fec89bed739d49ef6f231d580b2d4 Mon Sep 17 00:00:00 2001 From: egieseke Date: Thu, 20 Jun 2019 10:06:58 -0400 Subject: [PATCH 47/54] [GOAL2-614] Updated algoh to capture algod logs on early termination (#40) * GOAL2-614 Updated algoh so that logs will be captured if algod terminates before block watcher is initialized. Fix code analysis warnings. * Consolidated error collection into captureErrorLogs(). Removed blank identifiers. --- cmd/algoh/main.go | 75 +++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/cmd/algoh/main.go b/cmd/algoh/main.go index a9d9e498ca..8905133b9c 100644 --- a/cmd/algoh/main.go +++ b/cmd/algoh/main.go @@ -60,6 +60,7 @@ func (c *stdCollector) Write(p []byte) (n int, err error) { } func main() { + blockWatcherInitialized := false flag.Parse() nc := getNodeController() @@ -95,11 +96,11 @@ func main() { reportErrorf("Data directory %s does not appear to be valid\n", dataDir) } - config, err := algoh.LoadConfigFromFile(filepath.Join(dataDir, algoh.ConfigFilename)) + algohConfig, err := algoh.LoadConfigFromFile(filepath.Join(dataDir, algoh.ConfigFilename)) if err != nil && !os.IsNotExist(err) { reportErrorf("Error loading configuration, %v\n", err) } - validateConfig(config) + validateConfig(algohConfig) log := logging.Base() configureLogging(genesisID, log, absolutePath) @@ -126,35 +127,27 @@ func main() { err = cmd.Start() if err != nil { - reportErrorf("Error starting algod: %v", err) + reportErrorf("error starting algod: %v", err) + } + err = cmd.Wait() + if err != nil { + reportErrorf("error waiting for algod: %v", err) } - cmd.Wait() close(done) + // capture logs if algod terminated prior to blockWatcher starting + if !blockWatcherInitialized { + captureErrorLogs(algohConfig, errorOutput, output, absolutePath, true) + } + log.Infoln("++++++++++++++++++++++++++++++++++++++++") log.Infoln("algod exited. Exiting...") log.Infoln("++++++++++++++++++++++++++++++++++++++++") }() - // Set up error capturing in case algod exits before we can get REST client + // Set up error capturing defer func() { - if errorOutput.output != "" { - fmt.Fprintf(os.Stderr, errorOutput.output) - details := telemetryspec.ErrorOutputEventDetails{ - Error: errorOutput.output, - Output: output.output, - } - log.EventWithDetails(telemetryspec.HostApplicationState, telemetryspec.ErrorOutputEvent, details) - - // Write stdout & stderr streams to disk - ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdOutFilename), []byte(output.output), os.ModePerm) - ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdErrFilename), []byte(errorOutput.output), os.ModePerm) - - if config.UploadOnError { - fmt.Fprintf(os.Stdout, "Uploading logs...\n") - sendLogs() - } - } + captureErrorLogs(algohConfig, errorOutput, output, absolutePath, false) }() // Handle signals cleanly @@ -167,28 +160,30 @@ func main() { os.Exit(0) }() - client, err := waitForClient(nc, done) + algodClient, err := waitForClient(nc, done) if err != nil { reportErrorf("error creating Rest Client: %v\n", err) } var wg sync.WaitGroup - deadMan := makeDeadManWatcher(config.DeadManTimeSec, client, config.UploadOnError, done, &wg) + deadMan := makeDeadManWatcher(algohConfig.DeadManTimeSec, algodClient, algohConfig.UploadOnError, done, &wg) wg.Add(1) listeners := []blockListener{deadMan} - if config.SendBlockStats { + if algohConfig.SendBlockStats { // Note: Resume can be implemented here. Store blockListener state and set curBlock based on latestBlock/lastBlock. listeners = append(listeners, &blockstats{log: logging.Base()}) } - delayBetweenStatusChecks := time.Duration(config.StatusDelayMS) * time.Millisecond - stallDetectionDelay := time.Duration(config.StallDelayMS) * time.Millisecond + delayBetweenStatusChecks := time.Duration(algohConfig.StatusDelayMS) * time.Millisecond + stallDetectionDelay := time.Duration(algohConfig.StallDelayMS) * time.Millisecond - runBlockWatcher(listeners, client, done, &wg, delayBetweenStatusChecks, stallDetectionDelay) + runBlockWatcher(listeners, algodClient, done, &wg, delayBetweenStatusChecks, stallDetectionDelay) wg.Add(1) + blockWatcherInitialized = true + wg.Wait() fmt.Println("Exiting algoh normally...") } @@ -202,7 +197,7 @@ func waitForClient(nc nodecontrol.NodeController, abort chan struct{}) (client c select { case <-abort: - err = fmt.Errorf("Aborted waiting for client") + err = fmt.Errorf("aborted waiting for client") return case <-time.After(100 * time.Millisecond): } @@ -323,6 +318,28 @@ func initTelemetry(genesisID string, log logging.Logger, dataDirectory string) { } } +// capture algod error output and optionally upload logs +func captureErrorLogs(algohConfig algoh.HostConfig, errorOutput stdCollector, output stdCollector, absolutePath string, errorCondition bool) { + if errorOutput.output != "" { + fmt.Fprintf(os.Stdout, "errorOutput.output: `%s`\n", errorOutput.output) + errorCondition = true + fmt.Fprintf(os.Stderr, errorOutput.output) + details := telemetryspec.ErrorOutputEventDetails{ + Error: errorOutput.output, + Output: output.output, + } + log.EventWithDetails(telemetryspec.HostApplicationState, telemetryspec.ErrorOutputEvent, details) + + // Write stdout & stderr streams to disk + _ = ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdOutFilename), []byte(output.output), os.ModePerm) + _ = ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdErrFilename), []byte(errorOutput.output), os.ModePerm) + } + if errorCondition && algohConfig.UploadOnError { + fmt.Fprintf(os.Stdout, "Uploading logs...\n") + sendLogs() + } +} + func reportErrorf(format string, args ...interface{}) { fmt.Fprintf(os.Stderr, format, args...) logging.Base().Fatalf(format, args...) From 1cb4f71a622e7cbf00e58a02e7d7296fc34dd7a3 Mon Sep 17 00:00:00 2001 From: Benjamin Chan Date: Thu, 20 Jun 2019 13:59:52 -0400 Subject: [PATCH 48/54] Call voteStepFresh to add FPR check. --- agreement/voteAggregator.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/agreement/voteAggregator.go b/agreement/voteAggregator.go index ced4cb600b..dbbc83c800 100644 --- a/agreement/voteAggregator.go +++ b/agreement/voteAggregator.go @@ -221,7 +221,8 @@ func (agg *voteAggregator) filterBundle(ub unauthenticatedBundle, freshData fres // voteStepFresh is a helper function for vote relay rules. Votes from steps // [soft, next] are always propagated, as are votes from [s-1, s+1] where s is -// the current/last concluding step. +// the current/last concluding step. Set mine to 0 to effectively disable allowing +// votes adjacent to the current/last concluding step. func voteStepFresh(descr string, proto protocol.ConsensusVersion, mine, vote step) error { if vote <= next { // always propagate first recovery vote to ensure synchronous block of periods after partition @@ -249,11 +250,11 @@ func voteFresh(proto protocol.ConsensusVersion, freshData freshnessData, vote un } if freshData.PlayerRound+1 == vote.R.Round { - if (vote.R.Period > 0 || vote.R.Step > next) { - return fmt.Errorf("filtered future vote from bad period or step: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) + if (vote.R.Period > 0) { + return fmt.Errorf("filtered future vote from bad period: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) } // pipeline votes from next round period 0 - return nil + return voteStepFresh("from next round", proto, 0, vote.R.Step) } switch vote.R.Period { From 5588d0b78529dcea07d649f93130b44620ff4ddc Mon Sep 17 00:00:00 2001 From: David Shoots Date: Thu, 20 Jun 2019 17:12:03 -0400 Subject: [PATCH 49/54] Fix TestDBConcurrencyRW test to run on Macs TestDBConcurrencyRW assumes `/dev/shm` device exists. It doesn't on Mac, so use tmp folder in that case. --- util/db/dbutil_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/util/db/dbutil_test.go b/util/db/dbutil_test.go index 60362bb071..50b96990b8 100644 --- a/util/db/dbutil_test.go +++ b/util/db/dbutil_test.go @@ -20,6 +20,9 @@ import ( "database/sql" "errors" "fmt" + "io/ioutil" + "path/filepath" + "runtime" "sync" "sync/atomic" "testing" @@ -206,7 +209,18 @@ func TestDBConcurrency(t *testing.T) { } func TestDBConcurrencyRW(t *testing.T) { - fn := fmt.Sprintf("/dev/shm/%s.%d.sqlite3", t.Name(), crypto.RandUint64()) + dbFolder := "/dev/shm" + os := runtime.GOOS + if os == "darwin" { + var err error + dbFolder, err = ioutil.TempDir("", "TestDBConcurrencyRW") + if err != nil { + panic(err) + } + } + + fn := fmt.Sprintf("/%s.%d.sqlite3", t.Name(), crypto.RandUint64()) + fn = filepath.Join(dbFolder, fn) acc, err := MakeAccessor(fn, false, false) require.NoError(t, err) From ccd3a772ec0340b3964b2dfe35120bb724165478 Mon Sep 17 00:00:00 2001 From: David Shoots Date: Thu, 20 Jun 2019 18:27:24 -0400 Subject: [PATCH 50/54] Fixed lint --- cmd/algorelay/eb/eb.go | 1 + cmd/algorelay/relayCmd.go | 17 +++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cmd/algorelay/eb/eb.go b/cmd/algorelay/eb/eb.go index 02024c4152..283ebc38ea 100644 --- a/cmd/algorelay/eb/eb.go +++ b/cmd/algorelay/eb/eb.go @@ -16,6 +16,7 @@ package eb +// Relay represents the configuration data necessary for a single Relay type Relay struct { ID int64 // db key injected when loaded IPOrDNSName string diff --git a/cmd/algorelay/relayCmd.go b/cmd/algorelay/relayCmd.go index 1b1f2ea1d8..6e6fa6eaf9 100644 --- a/cmd/algorelay/relayCmd.go +++ b/cmd/algorelay/relayCmd.go @@ -325,13 +325,12 @@ func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDom err = addDNSRecord(targetDomainAlias, topmost, cfNameZoneID) if err != nil { return - } else { - fmt.Printf("[%d] Added DNS Record: %s -> %s\n", relay.ID, targetDomainAlias, topmost) - - // Update our state - names = append(names, targetDomainAlias) - topmost = targetDomainAlias } + fmt.Printf("[%d] Added DNS Record: %s -> %s\n", relay.ID, targetDomainAlias, topmost) + + // Update our state + names = append(names, targetDomainAlias) + topmost = targetDomainAlias } var ensureEntry = func(use string, entries map[string]uint16, port uint16) error { @@ -374,9 +373,8 @@ func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDom err = addSRVRecord(ctx.bootstrap.networkName, topmost, port, ctx.bootstrap.shortName, cfSrvZoneID) if err != nil { return - } else { - fmt.Printf("[%d] Added boostrap SRV Record: %s:%d\n", relay.ID, targetDomainAlias, port) } + fmt.Printf("[%d] Added boostrap SRV Record: %s:%d\n", relay.ID, targetDomainAlias, port) } err = ensureEntry("metrics", ctx.metrics.entries, metricsPort) @@ -390,9 +388,8 @@ func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDom err = addSRVRecord(ctx.metrics.networkName, topmost, metricsPort, ctx.metrics.shortName, cfSrvZoneID) if err != nil { return - } else { - fmt.Printf("[%d] Added metrics SRV Record: %s:%d\n", relay.ID, targetDomainAlias, metricsPort) } + fmt.Printf("[%d] Added metrics SRV Record: %s:%d\n", relay.ID, targetDomainAlias, metricsPort) } } else if err == nil { err = fmt.Errorf("metrics should not be registered for %s but it is", target) From 6ab212af83eb8af85ab0fdc38a1001afe9171b5a Mon Sep 17 00:00:00 2001 From: Rotem Hemo Date: Fri, 21 Jun 2019 08:12:50 -0400 Subject: [PATCH 51/54] -- Changing encoding of expected hash value to Base32 --- cmd/auctionminion/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/auctionminion/main.go b/cmd/auctionminion/main.go index 941fe315a3..5f1ff6874b 100644 --- a/cmd/auctionminion/main.go +++ b/cmd/auctionminion/main.go @@ -248,4 +248,8 @@ func main() { cfg.StartRound = ra.LastRound() + 1 writeConfig(cfg) fmt.Printf("Wrote updated state to %s\n", *stateFile) + + outcomes := ra.Settle(false) + outcomesHash := crypto.HashObj(outcomes) + fmt.Printf("Expected outcomes hash (if settled without cancelling): %v\n", outcomesHash.String()) } From f41aa9b654bb9df474a209e67081a733f1d39ac6 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 21 Jun 2019 12:31:30 -0400 Subject: [PATCH 52/54] Skip unit test TestFirstListenerSetupGetsPort8080WhenPassedPortZero (#73) * Skip test for taking port 8080 when this port is already in use. * fix typo. --- daemon/algod/server_test.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/daemon/algod/server_test.go b/daemon/algod/server_test.go index 99a48ae28e..4848e833a5 100644 --- a/daemon/algod/server_test.go +++ b/daemon/algod/server_test.go @@ -19,20 +19,35 @@ package algod // this should make dummy requests against the API and check the results for consistency import ( + "fmt" + "net" "testing" "github.com/stretchr/testify/require" ) +func isTCPPortAvailable(host string, port int) bool { + l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", host, port)) + if l != nil { + l.Close() + } + return err == nil +} func TestFirstListenerSetupGetsPort8080WhenPassedPortZero(t *testing.T) { // this test will fail if there is already a listener on the testing machine's port 8080 - // (ex if a dev has a node running on port 8080 and runs the test, it will fail) - defaultAddr := "127.0.0.1:0" - expectedAddr := "127.0.0.1:8080" + // (except if a dev has a node running on port 8080 and runs the test; in that case, we can't run this test.) + targetPort := 8080 + host := "127.0.0.1" + // check if port 8080 is busy : + if !isTCPPortAvailable(host, targetPort) { + t.Skipf("Cannot run this test since port 8080 is already in use.") + } + defaultAddr := host + ":0" + expectedAddr := fmt.Sprintf("%s:%d", host, targetPort) listener, err := makeListener(defaultAddr) require.NoError(t, err) actualAddr := listener.Addr().String() - require.Equal(t, expectedAddr, actualAddr, "if port 8080 is occupied when this test runs, it will fail") + require.Equalf(t, expectedAddr, actualAddr, "if port %d is occupied when this test runs, it will fail", targetPort) } func TestSecondListenerSetupGetsAnotherPortWhen8080IsBusy(t *testing.T) { From b0ffe820983369de5f38deaf68c169b96372b1bd Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 21 Jun 2019 14:38:13 -0400 Subject: [PATCH 53/54] Minor cleanup (#72) Fix nit from #7 and run 'make sanity' --- agreement/voteAggregator.go | 2 +- util/metrics/counter.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/agreement/voteAggregator.go b/agreement/voteAggregator.go index dbbc83c800..30c51f7e06 100644 --- a/agreement/voteAggregator.go +++ b/agreement/voteAggregator.go @@ -250,7 +250,7 @@ func voteFresh(proto protocol.ConsensusVersion, freshData freshnessData, vote un } if freshData.PlayerRound+1 == vote.R.Round { - if (vote.R.Period > 0) { + if vote.R.Period > 0 { return fmt.Errorf("filtered future vote from bad period: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) } // pipeline votes from next round period 0 diff --git a/util/metrics/counter.go b/util/metrics/counter.go index 850be3a630..0b9d18c009 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -190,5 +190,4 @@ func (counter *Counter) AddMetric(values map[string]string) { values[counter.name] = strconv.FormatFloat(sum, 'f', -1, 32) } - } From f105c21953cfd1e5dcd26ac28018f0a66b4b7930 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 21 Jun 2019 15:34:11 -0400 Subject: [PATCH 54/54] Fix flushThread starting bug. (#75) --- vendor/github.com/algorand/websocket/conn.go | 25 ++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/vendor/github.com/algorand/websocket/conn.go b/vendor/github.com/algorand/websocket/conn.go index 90125afb3e..244d2983d2 100644 --- a/vendor/github.com/algorand/websocket/conn.go +++ b/vendor/github.com/algorand/websocket/conn.go @@ -469,6 +469,19 @@ func (c *Conn) flushThread() { defer c.bwLock.Unlock() for true { + if c.bw == nil { + return + } + c.conn.SetWriteDeadline(c.writeDeadline) + err := c.bw.Flush() + if err != nil { + c.writeErrMu.Lock() + if c.writeErr != nil { + c.writeErr = err + } + c.writeErrMu.Unlock() + } + c.bwCond.Wait() if c.bw == nil { return @@ -482,18 +495,6 @@ func (c *Conn) flushThread() { } c.bwLock.Lock() - if c.bw == nil { - return - } - c.conn.SetWriteDeadline(c.writeDeadline) - err := c.bw.Flush() - if err != nil { - c.writeErrMu.Lock() - if c.writeErr != nil { - c.writeErr = err - } - c.writeErrMu.Unlock() - } } }