diff --git a/agreement/voteAggregator.go b/agreement/voteAggregator.go index c27775d655..30c51f7e06 100644 --- a/agreement/voteAggregator.go +++ b/agreement/voteAggregator.go @@ -221,7 +221,8 @@ func (agg *voteAggregator) filterBundle(ub unauthenticatedBundle, freshData fres // voteStepFresh is a helper function for vote relay rules. Votes from steps // [soft, next] are always propagated, as are votes from [s-1, s+1] where s is -// the current/last concluding step. +// the current/last concluding step. Set mine to 0 to effectively disable allowing +// votes adjacent to the current/last concluding step. func voteStepFresh(descr string, proto protocol.ConsensusVersion, mine, vote step) error { if vote <= next { // always propagate first recovery vote to ensure synchronous block of periods after partition @@ -248,8 +249,12 @@ func voteFresh(proto protocol.ConsensusVersion, freshData freshnessData, vote un return fmt.Errorf("filtered vote from bad round: player.Round=%v; vote.Round=%v", freshData.PlayerRound, vote.R.Round) } - if freshData.PlayerRound+1 == vote.R.Round && (vote.R.Period > 0 || vote.R.Step > next) { - return fmt.Errorf("filtered future vote from bad period or step: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) + if freshData.PlayerRound+1 == vote.R.Round { + if vote.R.Period > 0 { + return fmt.Errorf("filtered future vote from bad period: player.Round=%v; vote.(Round,Period,Step)=(%v,%v,%v)", freshData.PlayerRound, vote.R.Round, vote.R.Period, vote.R.Step) + } + // pipeline votes from next round period 0 + return voteStepFresh("from next round", proto, 0, vote.R.Step) } switch vote.R.Period { diff --git a/agreement/voteAggregator_test.go b/agreement/voteAggregator_test.go index a2ca0b822d..841769afe7 100644 --- a/agreement/voteAggregator_test.go +++ b/agreement/voteAggregator_test.go @@ -777,3 +777,65 @@ func TestVoteAggregatorFiltersVotePresentPeriod(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, res, "VotePresent not correctly filtered") } + +func TestVoteAggregatorFiltersVoteNextRound(t *testing.T) { + // Set up a composed test machine + rRouter := new(rootRouter) + rRouter.update(player{}, 0, false) + voteM := &ioAutomataConcrete{ + listener: rRouter.voteRoot, + routerCtx: rRouter, + } + helper := voteMakerHelper{} + helper.Setup() + b := testCaseBuilder{} + + // define a current player state for freshness testing + lastConcludingStep := next + msgTemplate := filterableMessageEvent{ + FreshnessData: freshnessData{ + PlayerRound: round(10), + PlayerPeriod: period(10), + PlayerStep: next + 5, + PlayerLastConcluding: lastConcludingStep, + }, + } + // generate old next vote in next round, period 0, step 1; make sure it is accepted + pV := helper.MakeRandomProposalValue() + uv := helper.MakeUnauthenticatedVote(t, 0, round(11), period(0), soft, *pV) + inMsg := msgTemplate // copy + inMsg.messageEvent = messageEvent{ + T: votePresent, + Input: message{ + UnauthenticatedVote: uv, + }, + } + b.AddInOutPair(inMsg, emptyEvent{}) + + // next round, period 0, step > next should be rejected + uv = helper.MakeUnauthenticatedVote(t, 1, round(11), period(0), next+1, *pV) + inMsg = msgTemplate // copy + inMsg.messageEvent = messageEvent{ + T: votePresent, + Input: message{ + UnauthenticatedVote: uv, + }, + } + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + + // next round, period 1 should be rejected + uv = helper.MakeUnauthenticatedVote(t, 1, round(11), period(1), soft, *pV) + inMsg = msgTemplate // copy + inMsg.messageEvent = messageEvent{ + T: votePresent, + Input: message{ + UnauthenticatedVote: uv, + }, + } + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + + // finalize + res, err := b.Build().Validate(voteM) + require.NoError(t, err) + require.NoErrorf(t, res, "Votes from next round not correctly filtered") +} diff --git a/auction/tracker.go b/auction/tracker.go index effc445a3e..a71101abd4 100644 --- a/auction/tracker.go +++ b/auction/tracker.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "math" "sync" "github.com/algorand/go-deadlock" @@ -289,7 +290,7 @@ func (am *Tracker) LiveUpdateWithContext(ctx context.Context, wg *sync.WaitGroup log.Debugf("Getting transactions for %d-%d", am.LastRound+1, status.LastRound) - transactions, err := rc.TransactionsByAddr(am.AuctionKey.GetChecksumAddress().String(), am.LastRound+1, status.LastRound) + transactions, err := rc.TransactionsByAddr(am.AuctionKey.GetChecksumAddress().String(), am.LastRound+1, status.LastRound, math.MaxUint64) if err != nil { log.Error(err) fmt.Println(err) diff --git a/cmd/algod/main.go b/cmd/algod/main.go index 3141559da8..c2fa2535a4 100644 --- a/cmd/algod/main.go +++ b/cmd/algod/main.go @@ -36,6 +36,7 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/metrics" "github.com/algorand/go-algorand/util/tokens" ) @@ -71,13 +72,20 @@ func main() { rand.Seed(time.Now().UnixNano()) } + version := config.GetCurrentVersion() if *versionCheck { - version := config.GetCurrentVersion() fmt.Printf("%d\n%s.%s [%s] (commit #%s)\n%s\n", version.AsUInt64(), version.String(), version.Channel, version.Branch, version.GetCommitHash(), config.GetLicenseInfo()) return } + heartbeatGauge := metrics.MakeStringGauge() + heartbeatGauge.Set("version", version.String()) + heartbeatGauge.Set("version-num", strconv.FormatUint(version.AsUInt64(), 10)) + heartbeatGauge.Set("channel", version.Channel) + heartbeatGauge.Set("branch", version.Branch) + heartbeatGauge.Set("commit-hash", version.GetCommitHash()) + if *branchCheck { fmt.Println(config.Branch) return @@ -183,10 +191,18 @@ func main() { log.EventWithDetails(telemetryspec.ApplicationState, telemetryspec.StartupEvent, startupDetails) // Send a heartbeat event every 10 minutes as a sign of life + ticker := time.NewTicker(10 * time.Minute) go func() { + values := make(map[string]string) for { - log.Event(telemetryspec.ApplicationState, telemetryspec.HeartbeatEvent) - <-time.After(10 * time.Minute) + metrics.DefaultRegistry().AddMetrics(values) + + heartbeatDetails := telemetryspec.HeartbeatEventDetails{ + Metrics: values, + } + + log.EventWithDetails(telemetryspec.ApplicationState, telemetryspec.HeartbeatEvent, heartbeatDetails) + <-ticker.C } }() } diff --git a/cmd/algoh/main.go b/cmd/algoh/main.go index a9d9e498ca..8905133b9c 100644 --- a/cmd/algoh/main.go +++ b/cmd/algoh/main.go @@ -60,6 +60,7 @@ func (c *stdCollector) Write(p []byte) (n int, err error) { } func main() { + blockWatcherInitialized := false flag.Parse() nc := getNodeController() @@ -95,11 +96,11 @@ func main() { reportErrorf("Data directory %s does not appear to be valid\n", dataDir) } - config, err := algoh.LoadConfigFromFile(filepath.Join(dataDir, algoh.ConfigFilename)) + algohConfig, err := algoh.LoadConfigFromFile(filepath.Join(dataDir, algoh.ConfigFilename)) if err != nil && !os.IsNotExist(err) { reportErrorf("Error loading configuration, %v\n", err) } - validateConfig(config) + validateConfig(algohConfig) log := logging.Base() configureLogging(genesisID, log, absolutePath) @@ -126,35 +127,27 @@ func main() { err = cmd.Start() if err != nil { - reportErrorf("Error starting algod: %v", err) + reportErrorf("error starting algod: %v", err) + } + err = cmd.Wait() + if err != nil { + reportErrorf("error waiting for algod: %v", err) } - cmd.Wait() close(done) + // capture logs if algod terminated prior to blockWatcher starting + if !blockWatcherInitialized { + captureErrorLogs(algohConfig, errorOutput, output, absolutePath, true) + } + log.Infoln("++++++++++++++++++++++++++++++++++++++++") log.Infoln("algod exited. Exiting...") log.Infoln("++++++++++++++++++++++++++++++++++++++++") }() - // Set up error capturing in case algod exits before we can get REST client + // Set up error capturing defer func() { - if errorOutput.output != "" { - fmt.Fprintf(os.Stderr, errorOutput.output) - details := telemetryspec.ErrorOutputEventDetails{ - Error: errorOutput.output, - Output: output.output, - } - log.EventWithDetails(telemetryspec.HostApplicationState, telemetryspec.ErrorOutputEvent, details) - - // Write stdout & stderr streams to disk - ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdOutFilename), []byte(output.output), os.ModePerm) - ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdErrFilename), []byte(errorOutput.output), os.ModePerm) - - if config.UploadOnError { - fmt.Fprintf(os.Stdout, "Uploading logs...\n") - sendLogs() - } - } + captureErrorLogs(algohConfig, errorOutput, output, absolutePath, false) }() // Handle signals cleanly @@ -167,28 +160,30 @@ func main() { os.Exit(0) }() - client, err := waitForClient(nc, done) + algodClient, err := waitForClient(nc, done) if err != nil { reportErrorf("error creating Rest Client: %v\n", err) } var wg sync.WaitGroup - deadMan := makeDeadManWatcher(config.DeadManTimeSec, client, config.UploadOnError, done, &wg) + deadMan := makeDeadManWatcher(algohConfig.DeadManTimeSec, algodClient, algohConfig.UploadOnError, done, &wg) wg.Add(1) listeners := []blockListener{deadMan} - if config.SendBlockStats { + if algohConfig.SendBlockStats { // Note: Resume can be implemented here. Store blockListener state and set curBlock based on latestBlock/lastBlock. listeners = append(listeners, &blockstats{log: logging.Base()}) } - delayBetweenStatusChecks := time.Duration(config.StatusDelayMS) * time.Millisecond - stallDetectionDelay := time.Duration(config.StallDelayMS) * time.Millisecond + delayBetweenStatusChecks := time.Duration(algohConfig.StatusDelayMS) * time.Millisecond + stallDetectionDelay := time.Duration(algohConfig.StallDelayMS) * time.Millisecond - runBlockWatcher(listeners, client, done, &wg, delayBetweenStatusChecks, stallDetectionDelay) + runBlockWatcher(listeners, algodClient, done, &wg, delayBetweenStatusChecks, stallDetectionDelay) wg.Add(1) + blockWatcherInitialized = true + wg.Wait() fmt.Println("Exiting algoh normally...") } @@ -202,7 +197,7 @@ func waitForClient(nc nodecontrol.NodeController, abort chan struct{}) (client c select { case <-abort: - err = fmt.Errorf("Aborted waiting for client") + err = fmt.Errorf("aborted waiting for client") return case <-time.After(100 * time.Millisecond): } @@ -323,6 +318,28 @@ func initTelemetry(genesisID string, log logging.Logger, dataDirectory string) { } } +// capture algod error output and optionally upload logs +func captureErrorLogs(algohConfig algoh.HostConfig, errorOutput stdCollector, output stdCollector, absolutePath string, errorCondition bool) { + if errorOutput.output != "" { + fmt.Fprintf(os.Stdout, "errorOutput.output: `%s`\n", errorOutput.output) + errorCondition = true + fmt.Fprintf(os.Stderr, errorOutput.output) + details := telemetryspec.ErrorOutputEventDetails{ + Error: errorOutput.output, + Output: output.output, + } + log.EventWithDetails(telemetryspec.HostApplicationState, telemetryspec.ErrorOutputEvent, details) + + // Write stdout & stderr streams to disk + _ = ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdOutFilename), []byte(output.output), os.ModePerm) + _ = ioutil.WriteFile(filepath.Join(absolutePath, nodecontrol.StdErrFilename), []byte(errorOutput.output), os.ModePerm) + } + if errorCondition && algohConfig.UploadOnError { + fmt.Fprintf(os.Stdout, "Uploading logs...\n") + sendLogs() + } +} + func reportErrorf(format string, args ...interface{}) { fmt.Fprintf(os.Stderr, format, args...) logging.Base().Fatalf(format, args...) diff --git a/cmd/algons/dnsCmd.go b/cmd/algons/dnsCmd.go index 8083c60e1b..c9023faf96 100644 --- a/cmd/algons/dnsCmd.go +++ b/cmd/algons/dnsCmd.go @@ -20,6 +20,7 @@ import ( "bufio" "context" "fmt" + "io/ioutil" "net" "os" "regexp" @@ -40,6 +41,8 @@ var ( recordType string noPrompt bool excludePattern string + exportNetwork string + outputFilename string ) func init() { @@ -47,6 +50,10 @@ func init() { dnsCmd.AddCommand(addCmd) dnsCmd.AddCommand(deleteCmd) dnsCmd.AddCommand(listCmd) + dnsCmd.AddCommand(exportCmd) + + listCmd.AddCommand(listRecordsCmd) + listCmd.AddCommand(listZonesCmd) addCmd.Flags().StringVarP(&addFromName, "from", "f", "", "From name to add new DNS entry") addCmd.MarkFlagRequired("from") @@ -58,9 +65,13 @@ func init() { deleteCmd.Flags().BoolVarP(&noPrompt, "no-prompt", "y", false, "No prompting for records deletion") deleteCmd.Flags().StringVarP(&excludePattern, "exclude", "e", "", "name records exclude pattern") - listCmd.Flags().StringVarP(&listNetwork, "network", "n", "", "Domain name for records to list") - listCmd.Flags().StringVarP(&recordType, "recordType", "t", "", "DNS record type to list (A, CNAME, SRV)") - listCmd.MarkFlagRequired("network") + listRecordsCmd.Flags().StringVarP(&listNetwork, "network", "n", "", "Domain name for records to list") + listRecordsCmd.Flags().StringVarP(&recordType, "recordType", "t", "", "DNS record type to list (A, CNAME, SRV)") + listRecordsCmd.MarkFlagRequired("network") + + exportCmd.Flags().StringVarP(&exportNetwork, "network", "n", "", "Domain name to export") + exportCmd.MarkFlagRequired("network") + exportCmd.Flags().StringVarP(&outputFilename, "zonefile", "z", "", "Output file for backup ( intead of outputing it to stdout ) ") } type byIP []net.IP @@ -81,8 +92,17 @@ var dnsCmd = &cobra.Command{ var listCmd = &cobra.Command{ Use: "list", - Short: "List the DNS/SRV entries of the given network", - Long: "List the DNS/SRV entries of the given network", + Short: "List the A/SRV/Zones entries of the given network", + Long: "List the A/SRV/Zones entries of the given network", + Run: func(cmd *cobra.Command, args []string) { + cmd.HelpFunc()(cmd, args) + }, +} + +var listRecordsCmd = &cobra.Command{ + Use: "records", + Short: "List the A/SRV entries of the given network", + Long: "List the A/SRV entries of the given network", Run: func(cmd *cobra.Command, args []string) { recordType = strings.ToUpper(recordType) if recordType == "" || recordType == "A" || recordType == "CNAME" || recordType == "SRV" { @@ -94,6 +114,17 @@ var listCmd = &cobra.Command{ }, } +var listZonesCmd = &cobra.Command{ + Use: "zones", + Short: "List the zones", + Long: "List the zones", + Run: func(cmd *cobra.Command, args []string) { + if !doListZones() { + os.Exit(1) + } + }, +} + var checkCmd = &cobra.Command{ Use: "check", Short: "Check the status", @@ -140,6 +171,16 @@ var deleteCmd = &cobra.Command{ }, } +var exportCmd = &cobra.Command{ + Use: "export", + Short: "Export DNS record entries for a specified network", + Run: func(cmd *cobra.Command, args []string) { + if !doExportZone(exportNetwork, outputFilename) { + os.Exit(1) + } + }, +} + func doAddDNS(from string, to string) (err error) { cfZoneID, cfEmail, cfKey, err := getClouldflareCredentials() if err != nil { @@ -166,11 +207,23 @@ func doAddDNS(from string, to string) (err error) { return } -func getClouldflareCredentials() (zoneID string, email string, authKey string, err error) { - zoneID = os.Getenv("CLOUDFLARE_ZONE_ID") +func getClouldflareAuthCredentials() (email string, authKey string, err error) { email = os.Getenv("CLOUDFLARE_EMAIL") authKey = os.Getenv("CLOUDFLARE_AUTH_KEY") - if zoneID == "" || email == "" || authKey == "" { + if email == "" || authKey == "" { + err = fmt.Errorf("one or more credentials missing from ENV") + } + return +} + +func getClouldflareCredentials() (zoneID string, email string, authKey string, err error) { + email, authKey, err = getClouldflareAuthCredentials() + if err != nil { + return + } + + zoneID = os.Getenv("CLOUDFLARE_ZONE_ID") + if zoneID == "" { err = fmt.Errorf("one or more credentials missing from ENV") } return @@ -333,3 +386,64 @@ func listEntries(listNetwork string, recordType string) { } } } + +func doExportZone(network string, outputFilename string) bool { + cfEmail, cfKey, err := getClouldflareAuthCredentials() + if err != nil { + fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) + return false + } + cloudflareCred := cloudflare.NewCred(cfEmail, cfKey) + zones, err := cloudflareCred.GetZones(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error retrieving zones entries: %v\n", err) + return false + } + zoneID := "" + // find a zone that matches the requested network name. + for _, z := range zones { + if z.DomainName == network { + zoneID = z.ZoneID + break + } + fmt.Printf("%s : %s\n", z.DomainName, z.ZoneID) + } + if zoneID == "" { + fmt.Fprintf(os.Stderr, "No matching zoneID was found for %s\n", network) + return false + } + cloudflareDNS := cloudflare.NewDNS(zoneID, cfEmail, cfKey) + exportedZone, err := cloudflareDNS.ExportZone(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to export zone : %v\n", err) + return false + } + if outputFilename != "" { + err = ioutil.WriteFile(outputFilename, exportedZone, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to write exported zone file : %v\n", err) + return false + } + } else { + fmt.Fprint(os.Stdout, string(exportedZone)) + } + return true +} + +func doListZones() bool { + cfEmail, cfKey, err := getClouldflareAuthCredentials() + if err != nil { + fmt.Fprintf(os.Stderr, "error getting DNS credentials: %v", err) + return false + } + cloudflareCred := cloudflare.NewCred(cfEmail, cfKey) + zones, err := cloudflareCred.GetZones(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error listing zones entries: %v\n", err) + return false + } + for _, z := range zones { + fmt.Printf("%s : %s\n", z.DomainName, z.ZoneID) + } + return true +} diff --git a/cmd/auctionminion/main.go b/cmd/auctionminion/main.go index 941fe315a3..477423a326 100644 --- a/cmd/auctionminion/main.go +++ b/cmd/auctionminion/main.go @@ -21,6 +21,7 @@ import ( "flag" "fmt" "io/ioutil" + "math" "net/url" "os" @@ -143,7 +144,7 @@ func main() { fmt.Printf("Checking round %d..\n", curRound) } - txns, err := restClient.TransactionsByAddr(auctionChecksumAddr.String(), curRound, curRound) + txns, err := restClient.TransactionsByAddr(auctionChecksumAddr.String(), curRound, curRound, math.MaxUint64) if err != nil { fmt.Fprintf(os.Stderr, "Cannot fetch transactions from %d: %v\n", curRound, err) os.Exit(1) @@ -248,4 +249,8 @@ func main() { cfg.StartRound = ra.LastRound() + 1 writeConfig(cfg) fmt.Printf("Wrote updated state to %s\n", *stateFile) + + outcomes := ra.Settle(false) + outcomesHash := crypto.HashObj(outcomes) + fmt.Printf("Expected outcomes hash (if settled without cancelling): %v\n", outcomesHash.String()) } diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index e324ad6d93..b9de36a627 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -24,6 +24,7 @@ import ( "os" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/protocol" @@ -61,8 +62,8 @@ func init() { sendCmd.Flags().StringVarP(&toAddress, "to", "t", "", "Address to send to money to (required)") sendCmd.Flags().Uint64VarP(&amount, "amount", "a", 0, "The amount to be transferred (required), in microAlgos") sendCmd.Flags().Uint64Var(&fee, "fee", 0, "The transaction fee (automatically determined by default), in microAlgos") - sendCmd.Flags().Uint64Var(&firstValid, "firstvalid", 0, "The first round where the transaction may be committed to the ledger (currently ignored)") - sendCmd.Flags().Uint64Var(&lastValid, "lastvalid", 0, "The last round where the transaction may be committed to the ledger (currently ignored)") + sendCmd.Flags().Uint64Var(&firstValid, "firstvalid", 0, "The first round where the transaction may be committed to the ledger") + sendCmd.Flags().Uint64Var(&lastValid, "lastvalid", 0, "The last round where the transaction may be committed to the ledger") sendCmd.Flags().StringVar(¬eBase64, "noteb64", "", "Note (URL-base64 encoded)") sendCmd.Flags().StringVarP(¬eText, "note", "n", "", "Note text (ignored if --noteb64 used also)") sendCmd.Flags().StringVarP(&txFilename, "out", "o", "", "Dump an unsigned tx to the given file. In order to dump a signed transaction, pass -s") @@ -145,7 +146,7 @@ var sendCmd = &cobra.Command{ if txFilename == "" { // Sign and broadcast the tx wh, pw := ensureWalletHandleMaybePassword(dataDir, walletName, true) - tx, err := client.SendPaymentFromWallet(wh, pw, fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved) + tx, err := client.SendPaymentFromWallet(wh, pw, fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved, basics.Round(firstValid), basics.Round(lastValid)) // update information from Transaction txid := tx.ID().String() @@ -191,7 +192,7 @@ var sendCmd = &cobra.Command{ } } } else { - payment, err := client.ConstructPayment(fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved) + payment, err := client.ConstructPayment(fromAddressResolved, toAddressResolved, fee, amount, noteBytes, closeToAddressResolved, basics.Round(firstValid), basics.Round(lastValid)) if err != nil { reportErrorf(errorConstructingTX, err) } diff --git a/daemon/algod/api/client/restClient.go b/daemon/algod/api/client/restClient.go index 12222b0b16..ede9be3e78 100644 --- a/daemon/algod/api/client/restClient.go +++ b/daemon/algod/api/client/restClient.go @@ -210,12 +210,13 @@ func (client RestClient) LedgerSupply() (response models.Supply, err error) { type transactionsByAddrParams struct { FirstRound uint64 `url:"firstRound"` LastRound uint64 `url:"lastRound"` + Max uint64 `url:"max"` } // TransactionsByAddr returns all transactions for a PK [addr] in the [first, // last] rounds range. -func (client RestClient) TransactionsByAddr(addr string, first, last uint64) (response models.TransactionList, err error) { - err = client.get(&response, fmt.Sprintf("/account/%s/transactions", addr), transactionsByAddrParams{first, last}) +func (client RestClient) TransactionsByAddr(addr string, first, last, max uint64) (response models.TransactionList, err error) { + err = client.get(&response, fmt.Sprintf("/account/%s/transactions", addr), transactionsByAddrParams{first, last, max}) return } diff --git a/daemon/algod/server_test.go b/daemon/algod/server_test.go index 99a48ae28e..4848e833a5 100644 --- a/daemon/algod/server_test.go +++ b/daemon/algod/server_test.go @@ -19,20 +19,35 @@ package algod // this should make dummy requests against the API and check the results for consistency import ( + "fmt" + "net" "testing" "github.com/stretchr/testify/require" ) +func isTCPPortAvailable(host string, port int) bool { + l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", host, port)) + if l != nil { + l.Close() + } + return err == nil +} func TestFirstListenerSetupGetsPort8080WhenPassedPortZero(t *testing.T) { // this test will fail if there is already a listener on the testing machine's port 8080 - // (ex if a dev has a node running on port 8080 and runs the test, it will fail) - defaultAddr := "127.0.0.1:0" - expectedAddr := "127.0.0.1:8080" + // (except if a dev has a node running on port 8080 and runs the test; in that case, we can't run this test.) + targetPort := 8080 + host := "127.0.0.1" + // check if port 8080 is busy : + if !isTCPPortAvailable(host, targetPort) { + t.Skipf("Cannot run this test since port 8080 is already in use.") + } + defaultAddr := host + ":0" + expectedAddr := fmt.Sprintf("%s:%d", host, targetPort) listener, err := makeListener(defaultAddr) require.NoError(t, err) actualAddr := listener.Addr().String() - require.Equal(t, expectedAddr, actualAddr, "if port 8080 is occupied when this test runs, it will fail") + require.Equalf(t, expectedAddr, actualAddr, "if port %d is occupied when this test runs, it will fail", targetPort) } func TestSecondListenerSetupGetsAnotherPortWhen8080IsBusy(t *testing.T) { diff --git a/docker/Dockerfile b/docker/Dockerfile index 2bf3908670..15d4a70057 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,15 +1,18 @@ FROM ubuntu:18.04 ENV GOLANG_VERSION 1.12 +ENV DEBIAN_FRONTEND noninteractive + +RUN apt update && apt-get install -y git libboost-all-dev wget sqlite3 autoconf sudo tzdata bsdmainutils -RUN apt update && apt install -y git libboost-all-dev wget sqlite3 autoconf WORKDIR /root RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local ENV GOROOT /usr/local/go -ENV GOPATH $HOME/go +ENV GOPATH /go +ENV GOBIN /go/bin ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH RUN mkdir -p $GOPATH/src/github.com/algorand WORKDIR $GOPATH/src/github.com/algorand RUN git clone https://github.com/algorand/go-algorand WORKDIR $GOPATH/src/github.com/algorand/go-algorand -RUN git checkout master && scripts/configure_dev.sh && make +RUN git checkout master && ./scripts/configure_dev.sh && make install ENTRYPOINT ["/bin/bash"] diff --git a/installer/rpm/yum-cron-algorand.conf b/installer/rpm/yum-cron-algorand.conf index d48b177347..b35acbeaf8 100644 --- a/installer/rpm/yum-cron-algorand.conf +++ b/installer/rpm/yum-cron-algorand.conf @@ -24,8 +24,7 @@ apply_updates = yes # minutes before running. This is useful for e.g. staggering the # times that multiple systems will access update servers. If # random_sleep is 0 or negative, the program will run immediately. -# 6*60 = 360 -random_sleep = 360 +random_sleep = 57 [emitters] diff --git a/libgoal/libgoal.go b/libgoal/libgoal.go index 97ca13e569..f4d8f6fa52 100644 --- a/libgoal/libgoal.go +++ b/libgoal/libgoal.go @@ -428,13 +428,17 @@ type MultisigInfo struct { } // SendPaymentFromWallet signs a transaction using the given wallet and returns the resulted transaction id -func (c *Client) SendPaymentFromWallet(walletHandle, pw []byte, from, to string, fee, amount uint64, note []byte, closeTo string) (transactions.Transaction, error) { +func (c *Client) SendPaymentFromWallet(walletHandle, pw []byte, from, to string, fee, amount uint64, note []byte, closeTo string, firstValid, lastValid basics.Round) (transactions.Transaction, error) { // Build the transaction - tx, err := c.ConstructPayment(from, to, fee, amount, note, closeTo) + tx, err := c.ConstructPayment(from, to, fee, amount, note, closeTo, firstValid, lastValid) if err != nil { return transactions.Transaction{}, err } + return c.signAndBroadcastTransactionWithWallet(walletHandle, pw, tx) +} + +func (c *Client) signAndBroadcastTransactionWithWallet(walletHandle, pw []byte, tx transactions.Transaction) (transactions.Transaction, error) { // Sign the transaction kmd, err := c.ensureKmdClient() if err != nil { @@ -467,7 +471,9 @@ func (c *Client) SendPaymentFromWallet(walletHandle, pw []byte, from, to string, // ConstructPayment builds a payment transaction to be signed // If the fee is 0, the function will use the suggested one form the network -func (c *Client) ConstructPayment(from, to string, fee, amount uint64, note []byte, closeTo string) (transactions.Transaction, error) { +// if the lastValid is 0, firstValid + maxTxnLifetime will be used +// if the firstValid is 0, lastRound + 1 will be used +func (c *Client) ConstructPayment(from, to string, fee, amount uint64, note []byte, closeTo string, firstValid, lastValid basics.Round) (transactions.Transaction, error) { fromAddr, err := basics.UnmarshalChecksumAddress(from) if err != nil { return transactions.Transaction{}, err @@ -484,15 +490,25 @@ func (c *Client) ConstructPayment(from, to string, fee, amount uint64, note []by return transactions.Transaction{}, err } - round := params.LastRound cp := config.Consensus[protocol.ConsensusVersion(params.ConsensusVersion)] + if firstValid == 0 && lastValid == 0 { + firstValid = basics.Round(params.LastRound + 1) + lastValid = firstValid + basics.Round(cp.MaxTxnLife) + } else if firstValid != 0 && lastValid == 0 { + lastValid = firstValid + basics.Round(cp.MaxTxnLife) + } else if firstValid > lastValid { + return transactions.Transaction{}, fmt.Errorf("cannot construct payment: txn would first be valid on round %d which is after last valid round %d", firstValid, lastValid) + } else if lastValid-firstValid > basics.Round(cp.MaxTxnLife) { + return transactions.Transaction{}, fmt.Errorf("cannot construct payment: txn validity period ( %d to %d ) is greater than protocol max txn lifetime %d ", firstValid, lastValid, cp.MaxTxnLife) + } + tx := transactions.Transaction{ Type: protocol.PaymentTx, Header: transactions.Header{ Sender: fromAddr, Fee: basics.MicroAlgos{Raw: fee}, - FirstValid: basics.Round(round), - LastValid: basics.Round(round) + basics.Round(cp.MaxTxnLife), + FirstValid: firstValid, + LastValid: lastValid, Note: note, }, PaymentTxnFields: transactions.PaymentTxnFields{ diff --git a/libgoal/unencryptedWallet.go b/libgoal/unencryptedWallet.go index 943e0c20cb..2fce2396ec 100644 --- a/libgoal/unencryptedWallet.go +++ b/libgoal/unencryptedWallet.go @@ -35,7 +35,7 @@ func (c *Client) SendPaymentFromUnencryptedWallet(from, to string, fee, amount u return transactions.Transaction{}, err } - return c.SendPaymentFromWallet(wh, nil, from, to, fee, amount, note, "") + return c.SendPaymentFromWallet(wh, nil, from, to, fee, amount, note, "", 0, 0) } // GetUnencryptedWalletHandle returns the unencrypted wallet handle. If there diff --git a/logging/telemetryspec/event.go b/logging/telemetryspec/event.go index d652f216ef..8395dd98fc 100644 --- a/logging/telemetryspec/event.go +++ b/logging/telemetryspec/event.go @@ -41,6 +41,11 @@ type StartupEventDetails struct { // HeartbeatEvent is sent periodically to indicate node is running const HeartbeatEvent Event = "Heartbeat" +// HeartbeatEventDetails contains details for the StartupEvent +type HeartbeatEventDetails struct { + Metrics map[string]string +} + // CatchupStartEvent event const CatchupStartEvent Event = "CatchupStart" diff --git a/network/wsNetwork.go b/network/wsNetwork.go index 849a628eb7..05c9ee159a 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -120,6 +120,10 @@ var meanPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_me var medianPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_median_ping_seconds", Description: "Network round trip time to median peer in seconds."}) var maxPing = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peer_max_ping_seconds", Description: "Network round trip time to slowest peer in seconds."}) +var peers = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_peers", Description: "Number of active peers."}) +var incomingPeers = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_incoming_peers", Description: "Number of active incoming peers."}) +var outgoingPeers = metrics.MakeGauge(metrics.MetricName{Name: "algod_network_outgoing_peers", Description: "Number of active outgoing peers."}) + // Peer opaque interface for referring to a neighbor in the network type Peer interface{} @@ -879,6 +883,9 @@ func (wn *WebsocketNetwork) ServeHTTP(response http.ResponseWriter, request *htt Incoming: true, InstanceName: otherInstanceName, }) + + peers.Set(float64(wn.NumPeers()), nil) + incomingPeers.Set(float64(wn.numIncomingPeers()), nil) } func (wn *WebsocketNetwork) messageHandlerThread() { @@ -1498,6 +1505,9 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { InstanceName: myInstanceName, }) + peers.Set(float64(wn.NumPeers()), nil) + outgoingPeers.Set(float64(wn.numOutgoingPeers()), nil) + if wn.prioScheme != nil { challenge := response.Header.Get(PriorityChallengeHeader) if challenge != "" { @@ -1573,6 +1583,10 @@ func (wn *WebsocketNetwork) removePeer(peer *wsPeer, reason disconnectReason) { Reason: string(reason), }) + peers.Set(float64(wn.NumPeers()), nil) + incomingPeers.Set(float64(wn.numIncomingPeers()), nil) + outgoingPeers.Set(float64(wn.numOutgoingPeers()), nil) + wn.peersLock.Lock() defer wn.peersLock.Unlock() if peer.peerIndex < len(wn.peers) && wn.peers[peer.peerIndex] == peer { diff --git a/test/e2e-go/cli/perf/payment_test.go b/test/e2e-go/cli/perf/payment_test.go index 0aeb23c96e..c16293430c 100644 --- a/test/e2e-go/cli/perf/payment_test.go +++ b/test/e2e-go/cli/perf/payment_test.go @@ -58,7 +58,7 @@ func BenchmarkSendPayment(b *testing.B) { for i := 0; i < b.N; i++ { var nonce [8]byte crypto.RandBytes(nonce[:]) - tx, err = c.ConstructPayment(addr, addr, 1, 1, nonce[:], "") + tx, err = c.ConstructPayment(addr, addr, 1, 1, nonce[:], "", 0, 0) require.NoError(b, err) } }) @@ -74,7 +74,7 @@ func BenchmarkSendPayment(b *testing.B) { for i := 0; i < b.N; i++ { var nonce [8]byte crypto.RandBytes(nonce[:]) - _, err := c.SendPaymentFromWallet(wallet, nil, addr, addr, 1, 1, nonce[:], "") + _, err := c.SendPaymentFromWallet(wallet, nil, addr, addr, 1, 1, nonce[:], "", 0, 0) require.NoError(b, err) } }) diff --git a/test/e2e-go/features/multisig/multisig_test.go b/test/e2e-go/features/multisig/multisig_test.go index b13ca6d791..c09f8564ee 100644 --- a/test/e2e-go/features/multisig/multisig_test.go +++ b/test/e2e-go/features/multisig/multisig_test.go @@ -71,7 +71,7 @@ func TestBasicMultisig(t *testing.T) { fixture.SendMoneyAndWait(curStatus.LastRound, amountToFund, minTxnFee, fundingAddr, multisigAddr) // try to transact with 1 of 3 amountToSend := minAcctBalance - unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, nil, "") + unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, nil, "", 0, 0) r.NoError(err, "Unexpected error when constructing payment transaction") emptyPartial := crypto.MultisigSig{} emptySignature := crypto.Signature{} @@ -90,7 +90,7 @@ func TestBasicMultisig(t *testing.T) { r.True(fixture.WaitForTxnConfirmation(curStatus.LastRound+uint64(5), multisigAddr, txid)) // Need a new txid to avoid dup detection - unsignedTransaction, err = client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, []byte("foobar"), "") + unsignedTransaction, err = client.ConstructPayment(multisigAddr, addrs[0], minTxnFee, amountToSend, []byte("foobar"), "", 0, 0) r.NoError(err, "Unexpected error when constructing payment transaction") signatureWithOne, err = client.UnencryptedMultisigSignTransaction(unsignedTransaction, addrs[0], emptyPartial) r.NoError(err, "first signing returned error") @@ -196,7 +196,7 @@ func TestDuplicateKeys(t *testing.T) { fixture.SendMoneyAndWait(curStatus.LastRound, amountToFund, txnFee, fundingAddr, multisigAddr) // try to transact with "1" signature (though, this is a signature from "every" member of the multisig) amountToSend := minAcctBalance - unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], txnFee, amountToSend, nil, "") + unsignedTransaction, err := client.ConstructPayment(multisigAddr, addrs[0], txnFee, amountToSend, nil, "", 0, 0) r.NoError(err, "Unexpected error when constructing payment transaction") emptyPartial := crypto.MultisigSig{} emptySignature := crypto.Signature{} diff --git a/test/e2e-go/features/participation/onlineOfflineParticipation_test.go b/test/e2e-go/features/participation/onlineOfflineParticipation_test.go index 35a9144c6b..cf7956deef 100644 --- a/test/e2e-go/features/participation/onlineOfflineParticipation_test.go +++ b/test/e2e-go/features/participation/onlineOfflineParticipation_test.go @@ -59,7 +59,7 @@ func TestParticipationKeyOnlyAccountParticipatesCorrectly(t *testing.T) { amountToSend := uint64(10000) // arbitrary wh, err := client.GetUnencryptedWalletHandle() a.NoError(err, "should get unencrypted wallet handle") - _, err = client.SendPaymentFromWallet(wh, nil, partkeyOnlyAccount, richAccount, amountToSend, transactionFee, nil, "") + _, err = client.SendPaymentFromWallet(wh, nil, partkeyOnlyAccount, richAccount, amountToSend, transactionFee, nil, "", basics.Round(0), basics.Round(0)) a.Error(err, "attempt to send money from partkey-only account should be treated as though wallet is not controlled") // partkeyonly_account attempts to go offline, should fail (no rootkey to sign txn with) goOfflineUTx, err := client.MakeUnsignedGoOfflineTx(partkeyOnlyAccount, 0, 0, transactionFee) diff --git a/test/e2e-go/features/transactions/close_account_test.go b/test/e2e-go/features/transactions/close_account_test.go index 0b24f26fca..74656076c2 100644 --- a/test/e2e-go/features/transactions/close_account_test.go +++ b/test/e2e-go/features/transactions/close_account_test.go @@ -62,7 +62,7 @@ func TestAccountsCanClose(t *testing.T) { a.NoError(err) fixture.WaitForConfirmedTxn(status.LastRound+10, baseAcct, tx.ID().String()) - tx, err = client.SendPaymentFromWallet(walletHandle, nil, acct0, acct1, 1, 100000, nil, acct2) + tx, err = client.SendPaymentFromWallet(walletHandle, nil, acct0, acct1, 1, 100000, nil, acct2, 0, 0) a.NoError(err) fixture.WaitForConfirmedTxn(status.LastRound+10, acct0, tx.ID().String()) diff --git a/test/e2e-go/restAPI/restClient_test.go b/test/e2e-go/restAPI/restClient_test.go index 4ee5a479aa..e4972710f3 100644 --- a/test/e2e-go/restAPI/restClient_test.go +++ b/test/e2e-go/restAPI/restClient_test.go @@ -193,7 +193,7 @@ func TestTransactionsByAddr(t *testing.T) { t.Error("no addr with funds") } toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) - tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "") + tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "", 0, 0) require.NoError(t, err) txID := tx.ID() rnd, err := testClient.Status() @@ -211,7 +211,7 @@ func TestTransactionsByAddr(t *testing.T) { restClient, err := localFixture.NC.AlgodClient() require.NoError(t, err) - res, err := restClient.TransactionsByAddr(toAddress, 0, rnd.LastRound) + res, err := restClient.TransactionsByAddr(toAddress, 0, rnd.LastRound, 100) require.NoError(t, err) require.Equal(t, 1, len(res.Transactions)) @@ -256,7 +256,7 @@ func TestClientRejectsBadFromAddressWhenSending(t *testing.T) { require.NoError(t, err) badAccountAddress := "This is absolutely not a valid account address." goodAccountAddress := addresses[0] - _, err = testClient.SendPaymentFromWallet(wh, nil, badAccountAddress, goodAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, badAccountAddress, goodAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -269,7 +269,7 @@ func TestClientRejectsBadToAddressWhenSending(t *testing.T) { require.NoError(t, err) badAccountAddress := "This is absolutely not a valid account address." goodAccountAddress := addresses[0] - _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, badAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, badAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -289,7 +289,7 @@ func TestClientRejectsMutatedFromAddressWhenSending(t *testing.T) { require.NoError(t, err) } mutatedAccountAddress := mutateStringAtIndex(unmutatedAccountAddress, 0) - _, err = testClient.SendPaymentFromWallet(wh, nil, mutatedAccountAddress, goodAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, mutatedAccountAddress, goodAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -309,7 +309,7 @@ func TestClientRejectsMutatedToAddressWhenSending(t *testing.T) { require.NoError(t, err) } mutatedAccountAddress := mutateStringAtIndex(unmutatedAccountAddress, 0) - _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, mutatedAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, goodAccountAddress, mutatedAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -322,7 +322,7 @@ func TestClientRejectsSendingMoneyFromAccountForWhichItHasNoKey(t *testing.T) { require.NoError(t, err) goodAccountAddress := addresses[0] nodeDoesNotHaveKeyForThisAddress := "NJY27OQ2ZXK6OWBN44LE4K43TA2AV3DPILPYTHAJAMKIVZDWTEJKZJKO4A" - _, err = testClient.SendPaymentFromWallet(wh, nil, nodeDoesNotHaveKeyForThisAddress, goodAccountAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, nodeDoesNotHaveKeyForThisAddress, goodAccountAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -344,7 +344,7 @@ func TestClientOversizedNote(t *testing.T) { } maxTxnNoteBytes := config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnNoteBytes note := make([]byte, maxTxnNoteBytes+1) - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, note, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, note, "", 0, 0) require.Error(t, err) } @@ -363,7 +363,7 @@ func TestClientCanSendAndGetNote(t *testing.T) { toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) maxTxnNoteBytes := config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnNoteBytes note := make([]byte, maxTxnNoteBytes) - tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, note, "") + tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, note, "", 0, 0) require.NoError(t, err) txStatus, err := waitForTransaction(t, testClient, someAddress, tx.ID().String(), 15*time.Second) require.NoError(t, err) @@ -383,7 +383,7 @@ func TestClientCanGetTransactionStatus(t *testing.T) { t.Error("no addr with funds") } toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) - tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "") + tx, err := testClient.SendPaymentFromWallet(wh, nil, someAddress, toAddress, 10000, 100000, nil, "", 0, 0) t.Log(string(protocol.EncodeJSON(tx))) require.NoError(t, err) t.Log(tx.ID().String()) @@ -430,22 +430,22 @@ func TestSendingTooMuchFails(t *testing.T) { fromBalance, err := testClient.GetBalance(fromAddress) require.NoError(t, err) // too much amount - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, fromBalance+100, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, fromBalance+100, nil, "", 0, 0) t.Log(err) require.Error(t, err) // waaaay too much amount - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, math.MaxUint64, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, math.MaxUint64, nil, "", 0, 0) t.Log(err) require.Error(t, err) // too much fee - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, fromBalance+100, 10000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, fromBalance+100, 10000, nil, "", 0, 0) t.Log(err) require.Error(t, err) // waaaay too much fee - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, math.MaxUint64, 10000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, math.MaxUint64, 10000, nil, "", 0, 0) t.Log(err) require.Error(t, err) } @@ -482,7 +482,7 @@ func TestSendingFromEmptyAccountFails(t *testing.T) { toAddress, err = testClient.GenerateAddress(wh) require.NoError(t, err) } - _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, fromAddress, toAddress, 10000, 100000, nil, "", 0, 0) require.Error(t, err) } @@ -511,7 +511,7 @@ func TestSendingTooLittleToEmptyAccountFails(t *testing.T) { if someAddress == "" { t.Error("no addr with funds") } - _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, 1, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, 1, nil, "", 0, 0) require.Error(t, err) } @@ -531,7 +531,7 @@ func TestSendingLowFeeFails(t *testing.T) { t.Errorf("balance too low %d < %d", someBal, sendAmount) } toAddress := getDestAddr(t, testClient, addresses, someAddress, wh) - utx, err := testClient.ConstructPayment(someAddress, toAddress, 1, sendAmount, nil, "") + utx, err := testClient.ConstructPayment(someAddress, toAddress, 1, sendAmount, nil, "", 0, 0) require.NoError(t, err) utx.Fee.Raw = 1 stx, err := testClient.SignTransactionWithWallet(wh, nil, utx) @@ -586,7 +586,7 @@ func TestSendingNotClosingAccountFails(t *testing.T) { t.Error("no addr with funds") } amt := someBal - 10000 - 1 - _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, amt, nil, "") + _, err = testClient.SendPaymentFromWallet(wh, nil, someAddress, emptyAddress, 10000, amt, nil, "", 0, 0) require.Error(t, err) } diff --git a/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go b/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go index 8ace23ae54..acd2666735 100644 --- a/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go +++ b/test/e2e-go/stress/transactions/createManyAndGoOnline_test.go @@ -36,7 +36,7 @@ func cascadeCreateAndFundAccounts(amountToSend, transactionFee uint64, fundingAc a.NoError(err, "should be able to get unencrypted wallet handle") newAddress, err := client.GenerateAddress(wh) a.NoError(err, "should be able to generate new address") - tx, err := client.SendPaymentFromWallet(wh, nil, fundingAccount, newAddress, transactionFee, amountToSend, nil, "") + tx, err := client.SendPaymentFromWallet(wh, nil, fundingAccount, newAddress, transactionFee, amountToSend, nil, "", 0, 0) a.NoError(err, "should be no errors when funding new accounts, send number %v", i) i++ outputTxidsToAccounts[tx.ID().String()] = newAddress diff --git a/test/framework/fixtures/restClientFixture.go b/test/framework/fixtures/restClientFixture.go index 8be8d6d1fa..de7ab81131 100644 --- a/test/framework/fixtures/restClientFixture.go +++ b/test/framework/fixtures/restClientFixture.go @@ -276,7 +276,7 @@ func (f *RestClientFixture) SendMoneyAndWait(curRound, amountToSend, transaction // SendMoneyAndWaitFromWallet is as above, but for a specific wallet func (f *RestClientFixture) SendMoneyAndWaitFromWallet(walletHandle, walletPassword []byte, curRound, amountToSend, transactionFee uint64, fromAccount, toAccount string) (fundingTxid string) { client := f.LibGoalClient - fundingTx, err := client.SendPaymentFromWallet(walletHandle, walletPassword, fromAccount, toAccount, transactionFee, amountToSend, nil, "") + fundingTx, err := client.SendPaymentFromWallet(walletHandle, walletPassword, fromAccount, toAccount, transactionFee, amountToSend, nil, "", 0, 0) require.NoError(f.t, err, "client should be able to send money from rich to poor account") require.NotEmpty(f.t, fundingTx.ID().String(), "transaction ID should not be empty") waitingDeadline := curRound + uint64(5) diff --git a/tools/network/cloudflare/cloudflare.go b/tools/network/cloudflare/cloudflare.go index f62193706c..a4be998911 100644 --- a/tools/network/cloudflare/cloudflare.go +++ b/tools/network/cloudflare/cloudflare.go @@ -19,6 +19,7 @@ package cloudflare import ( "context" "fmt" + "io/ioutil" "net/http" "strings" ) @@ -29,19 +30,34 @@ const ( AutomaticTTL = 1 ) -// DNS is the cloudflare package main access class. Initiate an instance of this class to access the clouldflare APIs. -type DNS struct { - zoneID string +// Cred contains the credentials used to authenticate with the cloudflare API. +type Cred struct { authEmail string authKey string } +// DNS is the cloudflare package main access class. Initiate an instance of this class to access the clouldflare APIs. +type DNS struct { + zoneID string + Cred +} + +// NewCred creates a new credential structure used to authenticate with the cloudflare service. +func NewCred(authEmail string, authKey string) *Cred { + return &Cred{ + authEmail: authEmail, + authKey: authKey, + } +} + // NewDNS create a new instance of clouldflare DNS services class func NewDNS(zoneID string, authEmail string, authKey string) *DNS { return &DNS{ - zoneID: zoneID, - authEmail: authEmail, - authKey: authKey, + zoneID: zoneID, + Cred: Cred{ + authEmail: authEmail, + authKey: authKey, + }, } } @@ -241,3 +257,59 @@ func (d *DNS) UpdateSRVRecord(ctx context.Context, recordID string, name string, } return nil } + +// Zone represent a single zone on the cloudflare API. +type Zone struct { + DomainName string + ZoneID string +} + +// GetZones returns a list of zones that are associated with cloudflare. +func (c *Cred) GetZones(ctx context.Context) (zones []Zone, err error) { + request, err := getZonesRequest(c.authEmail, c.authKey) + if err != nil { + return nil, err + } + client := &http.Client{} + response, err := client.Do(request.WithContext(ctx)) + if err != nil { + return nil, err + } + + parsedResponse, err := parseGetZonesResponse(response) + if err != nil { + return nil, err + } + if parsedResponse.Success == false { + return nil, fmt.Errorf("failed to retrieve zone records : %v", parsedResponse) + } + + for _, z := range parsedResponse.Result { + zones = append(zones, + Zone{ + DomainName: z.Name, + ZoneID: z.ID, + }, + ) + } + return zones, err +} + +// ExportZone exports the zone into a BIND config bytes array +func (d *DNS) ExportZone(ctx context.Context) (exportedZoneBytes []byte, err error) { + request, err := exportZoneRequest(d.zoneID, d.authEmail, d.authKey) + if err != nil { + return nil, err + } + client := &http.Client{} + response, err := client.Do(request.WithContext(ctx)) + if err != nil { + return nil, err + } + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + return body, nil +} diff --git a/tools/network/cloudflare/zones.go b/tools/network/cloudflare/zones.go new file mode 100644 index 0000000000..944e0ed546 --- /dev/null +++ b/tools/network/cloudflare/zones.go @@ -0,0 +1,97 @@ +// Copyright (C) 2019 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package cloudflare + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/url" +) + +func getZonesRequest(authEmail, authKey string) (*http.Request, error) { + // construct the query + requestURI, err := url.Parse(cloudFlareURI) + if err != nil { + return nil, err + } + requestURI.Path = requestURI.Path + "zones" + request, err := http.NewRequest("GET", requestURI.String(), nil) + if err != nil { + return nil, err + } + addHeaders(request, authEmail, authKey) + return request, nil +} + +// GetZonesResult is the JSON response for a DNS create request +type GetZonesResult struct { + Success bool `json:"success"` + Errors []interface{} `json:"errors"` + Messages []interface{} `json:"messages"` + Result []GetZonesResultItem `json:"result"` + ResultInfo GetZonesResultPage `json:"result_info"` +} + +// GetZonesResultPage is the result of the response for the DNS create request +type GetZonesResultPage struct { + Page int `json:"page"` + PerPage int `json:"per_page"` + TotalPages int `json:"total_pages"` + Count int `json:"count"` + TotalCount int `json:"total_count"` +} + +// GetZonesResultItem is the result of the response for the DNS create request +type GetZonesResultItem struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Paused bool `json:"paused"` + Type string `json:"type"` + DevelopmentMode int `json:"development_mode"` + NameServers []string `json:"name_servers"` + OriginalNameServers []string `json:"original_name_servers"` +} + +func parseGetZonesResponse(response *http.Response) (*GetZonesResult, error) { + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + var parsedReponse GetZonesResult + if err := json.Unmarshal(body, &parsedReponse); err != nil { + return nil, err + } + return &parsedReponse, nil +} + +func exportZoneRequest(zoneID, authEmail, authKey string) (*http.Request, error) { + // construct the query + requestURI, err := url.Parse(cloudFlareURI) + if err != nil { + return nil, err + } + requestURI.Path = requestURI.Path + "zones/" + zoneID + "/dns_records/export" + request, err := http.NewRequest("GET", requestURI.String(), nil) + if err != nil { + return nil, err + } + addHeaders(request, authEmail, authKey) + return request, nil +} diff --git a/util/db/dbutil_test.go b/util/db/dbutil_test.go index 60362bb071..50b96990b8 100644 --- a/util/db/dbutil_test.go +++ b/util/db/dbutil_test.go @@ -20,6 +20,9 @@ import ( "database/sql" "errors" "fmt" + "io/ioutil" + "path/filepath" + "runtime" "sync" "sync/atomic" "testing" @@ -206,7 +209,18 @@ func TestDBConcurrency(t *testing.T) { } func TestDBConcurrencyRW(t *testing.T) { - fn := fmt.Sprintf("/dev/shm/%s.%d.sqlite3", t.Name(), crypto.RandUint64()) + dbFolder := "/dev/shm" + os := runtime.GOOS + if os == "darwin" { + var err error + dbFolder, err = ioutil.TempDir("", "TestDBConcurrencyRW") + if err != nil { + panic(err) + } + } + + fn := fmt.Sprintf("/%s.%d.sqlite3", t.Name(), crypto.RandUint64()) + fn = filepath.Join(dbFolder, fn) acc, err := MakeAccessor(fn, false, false) require.NoError(t, err) diff --git a/util/metrics/counter.go b/util/metrics/counter.go index fde3d4fbd6..0b9d18c009 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -172,3 +172,22 @@ func (counter *Counter) WriteMetric(buf *strings.Builder, parentLabels string) { buf.WriteString("\n") } } + +// AddMetric adds the metric into the map +func (counter *Counter) AddMetric(values map[string]string) { + counter.Lock() + defer counter.Unlock() + + if len(counter.values) < 1 { + return + } + + for _, l := range counter.values { + sum := l.counter + if len(l.labels) == 0 { + sum += float64(atomic.LoadUint64(&counter.intValue)) + } + + values[counter.name] = strconv.FormatFloat(sum, 'f', -1, 32) + } +} diff --git a/util/metrics/gauge.go b/util/metrics/gauge.go index f9aec394d4..b0925baefe 100644 --- a/util/metrics/gauge.go +++ b/util/metrics/gauge.go @@ -184,3 +184,19 @@ func (gauge *Gauge) WriteMetric(buf *strings.Builder, parentLabels string) { buf.WriteString("\n") } } + +// AddMetric adds the metric into the map +func (gauge *Gauge) AddMetric(values map[string]string) { + gauge.Lock() + defer gauge.Unlock() + + gauge.filterExpiredMetrics() + + if len(gauge.valuesIndices) < 1 { + return + } + + for _, l := range gauge.valuesIndices { + values[gauge.name] = strconv.FormatFloat(l.gauge, 'f', -1, 32) + } +} diff --git a/util/metrics/registry.go b/util/metrics/registry.go index 66018266c4..24d57a7094 100644 --- a/util/metrics/registry.go +++ b/util/metrics/registry.go @@ -66,3 +66,12 @@ func (r *Registry) WriteMetrics(buf *strings.Builder, parentLabels string) { m.WriteMetric(buf, parentLabels) } } + +// AddMetrics will add all the metrics that were registered to this registry +func (r *Registry) AddMetrics(values map[string]string) { + r.metricsMu.Lock() + defer r.metricsMu.Unlock() + for _, m := range r.metrics { + m.AddMetric(values) + } +} diff --git a/util/metrics/registryCommon.go b/util/metrics/registryCommon.go index bcf3fe9897..49db57f131 100644 --- a/util/metrics/registryCommon.go +++ b/util/metrics/registryCommon.go @@ -25,6 +25,7 @@ import ( // Metric represent any collectable metric type Metric interface { WriteMetric(buf *strings.Builder, parentLabels string) + AddMetric(values map[string]string) } // Registry represents a single set of metrics registry diff --git a/util/metrics/registry_test.go b/util/metrics/registry_test.go new file mode 100644 index 0000000000..b9c6c2d7d9 --- /dev/null +++ b/util/metrics/registry_test.go @@ -0,0 +1,46 @@ +// +build telemetry + +package metrics + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWriteAdd(t *testing.T) { + // Test AddMetrics and WriteMetrics with a counter + counter := MakeCounter(MetricName{Name: "gauge-name", Description: "gauge description"}) + counter.Add(12.34, nil) + + results := make(map[string]string) + DefaultRegistry().AddMetrics(results) + + require.Equal(t, 1, len(results)) + require.True(t, hasKey(results, "gauge-name")) + require.Equal(t, "12.34", results["gauge-name"]) + + bufBefore := strings.Builder{} + DefaultRegistry().WriteMetrics(&bufBefore, "label") + require.True(t, bufBefore.Len() > 0) + + // Test that WriteMetrics does not change after adding a StringGauge + stringGauge := MakeStringGauge() + stringGauge.Set("string-key", "value") + + DefaultRegistry().AddMetrics(results) + + require.True(t, hasKey(results, "string-key")) + require.Equal(t, "value", results["string-key"]) + require.True(t, hasKey(results, "gauge-name")) + require.Equal(t, "12.34", results["gauge-name"]) + + // not included in string builder + bufAfter := strings.Builder{} + DefaultRegistry().WriteMetrics(&bufAfter, "label") + require.Equal(t, bufBefore.String(), bufAfter.String()) + + stringGauge.Deregister(nil) + counter.Deregister(nil) +} diff --git a/util/metrics/stringGauge.go b/util/metrics/stringGauge.go new file mode 100644 index 0000000000..accd2b1618 --- /dev/null +++ b/util/metrics/stringGauge.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "strings" +) + +// MakeStringGauge create a new StringGauge. +func MakeStringGauge() *StringGauge { + c := &StringGauge{ + values: make(map[string]string), + } + c.Register(nil) + return c +} + +// Register registers the StringGauge with the default/specific registry +func (stringGauge *StringGauge) Register(reg *Registry) { + if reg == nil { + DefaultRegistry().Register(stringGauge) + } else { + reg.Register(stringGauge) + } +} + +// Deregister deregisters the StringGauge with the default/specific registry +func (stringGauge *StringGauge) Deregister(reg *Registry) { + if reg == nil { + DefaultRegistry().Deregister(stringGauge) + } else { + reg.Deregister(stringGauge) + } +} + +// Set updates a key with a value. +func (stringGauge *StringGauge) Set(key string, value string) { + stringGauge.values[key] = value +} + +// WriteMetric omit string gauges from the metrics report, not sure how they act with prometheus +func (stringGauge *StringGauge) WriteMetric(buf *strings.Builder, parentLabels string) { +} + +// AddMetric sets all the key value pairs in the provided map. +func (stringGauge *StringGauge) AddMetric(values map[string]string) { + for k, v := range stringGauge.values { + values[k] = v + } +} diff --git a/util/metrics/stringGaugeCommon.go b/util/metrics/stringGaugeCommon.go new file mode 100644 index 0000000000..c2c1e57f76 --- /dev/null +++ b/util/metrics/stringGaugeCommon.go @@ -0,0 +1,11 @@ +package metrics + +import ( + "github.com/algorand/go-deadlock" +) + +// StringGauge represents a map of key value pairs available to be written with the AddMetric +type StringGauge struct { + deadlock.Mutex + values map[string]string +} diff --git a/util/metrics/stringGauge_test.go b/util/metrics/stringGauge_test.go new file mode 100644 index 0000000000..6d2ae89482 --- /dev/null +++ b/util/metrics/stringGauge_test.go @@ -0,0 +1,36 @@ +package metrics + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func hasKey(data map[string]string, key string) bool { + _, ok := data[key] + return ok +} + +func TestMetricStringGauge(t *testing.T) { + stringGauge := MakeStringGauge() + stringGauge.Set("number-key", "1") + stringGauge.Set("string-key", "value") + + results := make(map[string]string) + DefaultRegistry().AddMetrics(results) + + // values are populated + require.Equal(t, 2, len(results)) + require.True(t, hasKey(results, "number-key")) + require.Equal(t, "1", results["number-key"]) + require.True(t, hasKey(results, "string-key")) + require.Equal(t, "value", results["string-key"]) + + // not included in string builder + buf := strings.Builder{} + DefaultRegistry().WriteMetrics(&buf, "not used") + require.Equal(t, "", buf.String()) + + stringGauge.Deregister(nil) +} diff --git a/vendor/github.com/algorand/websocket/conn.go b/vendor/github.com/algorand/websocket/conn.go index 90125afb3e..244d2983d2 100644 --- a/vendor/github.com/algorand/websocket/conn.go +++ b/vendor/github.com/algorand/websocket/conn.go @@ -469,6 +469,19 @@ func (c *Conn) flushThread() { defer c.bwLock.Unlock() for true { + if c.bw == nil { + return + } + c.conn.SetWriteDeadline(c.writeDeadline) + err := c.bw.Flush() + if err != nil { + c.writeErrMu.Lock() + if c.writeErr != nil { + c.writeErr = err + } + c.writeErrMu.Unlock() + } + c.bwCond.Wait() if c.bw == nil { return @@ -482,18 +495,6 @@ func (c *Conn) flushThread() { } c.bwLock.Lock() - if c.bw == nil { - return - } - c.conn.SetWriteDeadline(c.writeDeadline) - err := c.bw.Flush() - if err != nil { - c.writeErrMu.Lock() - if c.writeErr != nil { - c.writeErr = err - } - c.writeErrMu.Unlock() - } } }