rel/beta 2.0.12 release

.dockerignore

@@ -0,0 +1,44 @@
+*.log
+*.log.archive
+*.out
+*.prof
+coverage.html
+
+# cadaver
+*.cdv
+*.cdv.archive
+
+# swagger
+swagger.json
+swagger.json.validated
+kmdSwaggerWrappers.go
+bundledSpecInject.go
+
+# Exclude GoLand files
+.idea/
+
+# Exclude VSCode files
+.vscode/
+
+# Exclude go binaries built in-place
+cmd/algod/algod
+cmd/goal/goal
+cmd/updater/updater
+
+# Exclude our local temp directory
+tmp/dev_pkg
+tmp/out
+tmp/node_pkgs
+
+# Ignore vim backup and swap files
+*~
+*.swp
+*.swo
+
+# Mac
+.DS_Store
+
+# doc intermediates
+data/transactions/logic/*.md
+
+*.pem

.gitignore

@@ -41,6 +41,10 @@ crypto/libsodium-fork/**/Makefile.in
 crypto/libsodium-fork/aclocal.m4
 crypto/libsodium-fork/build-aux/
 
+# Ignore libsodium files generated during ci process
+crypto/copies
+crypto/libs
+
 # Ignore vim backup and swap files
 *~
 *.swp
@@ -54,3 +58,6 @@ data/transactions/logic/*.md
 
 *.pem
 
+# Folder for collecting release assets
+assets
+

Makefile

@@ -6,6 +6,7 @@ export GO111MODULE
 UNAME		:= $(shell uname)
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
+OS_TYPE     := $(shell ./scripts/ostype.sh)
 
 # If build number already set, use it - to ensure same build number across multiple platforms being built
 BUILDNUMBER      ?= $(shell ./scripts/compute_build_number.sh)
@@ -27,6 +28,13 @@ endif
 GOTAGSLIST  += osusergo netgo static_build
 GOBUILDMODE := -buildmode pie
 endif
+ifeq ($(ARCH), arm)
+ifneq ("$(wildcard /etc/alpine-release)","")
+EXTLDFLAGS  += -static
+GOTAGSLIST  += osusergo netgo static_build
+GOBUILDMODE := -buildmode pie
+endif
+endif
 endif
 
 GOTAGS      := --tags "$(GOTAGSLIST)"
@@ -44,7 +52,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GO111MODULE=off go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GO111MODULE=off cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./crypto ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement
+MSGP_GENERATE	:= ./protocol ./crypto ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node
 
 default: build
 
@@ -85,14 +93,16 @@ generate: deps
 msgp: $(patsubst %,%/msgp_gen.go,$(MSGP_GENERATE))
 
 %/msgp_gen.go: deps ALWAYS
-	$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ 1>/dev/null
+	$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand
 ALWAYS:
 
 # build our fork of libsodium, placing artifacts into crypto/lib/ and crypto/include/
-crypto/lib/libsodium.a:
-	cd crypto/libsodium-fork && \
-		./autogen.sh && \
-		./configure --disable-shared --prefix="$(SRCPATH)/crypto/" && \
+crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a:
+	mkdir -p crypto/copies/$(OS_TYPE)/$(ARCH)
+	cp -R crypto/libsodium-fork crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork
+	cd crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork && \
+		./autogen.sh --prefix $(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH) && \
+		./configure --disable-shared --prefix="$(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH)" && \
 		$(MAKE) && \
 		$(MAKE) install
 
@@ -108,7 +118,7 @@ ALGOD_API_FILES := $(shell find daemon/algod/api/server/common daemon/algod/api/
 ALGOD_API_SWAGGER_INJECT := daemon/algod/api/server/lib/bundledSpecInject.go
 
 # Note that swagger.json requires the go-swagger dep.
-$(ALGOD_API_SWAGGER_SPEC): $(ALGOD_API_FILES) crypto/lib/libsodium.a
+$(ALGOD_API_SWAGGER_SPEC): $(ALGOD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a
 	cd daemon/algod/api && \
 		PATH=$(GOPATH1)/bin:$$PATH \
 		go generate ./...
@@ -122,7 +132,7 @@ KMD_API_FILES := $(shell find daemon/kmd/api/ -type f | grep -v $(KMD_API_SWAGGE
 KMD_API_SWAGGER_WRAPPER := kmdSwaggerWrappers.go
 KMD_API_SWAGGER_INJECT := daemon/kmd/lib/kmdapi/bundledSpecInject.go
 
-$(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/lib/libsodium.a
+$(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a
 	cd daemon/kmd/lib/kmdapi && \
 		python genSwaggerWrappers.py $(KMD_API_SWAGGER_WRAPPER)
 	cd daemon/kmd && \
@@ -147,7 +157,7 @@ $(KMD_API_SWAGGER_INJECT): $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).valid
 
 build: buildsrc gen
 
-buildsrc: crypto/lib/libsodium.a node_exporter NONGO_BIN deps $(ALGOD_API_SWAGGER_INJECT) $(KMD_API_SWAGGER_INJECT)
+buildsrc: crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN deps $(ALGOD_API_SWAGGER_INJECT) $(KMD_API_SWAGGER_INJECT)
 	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 SOURCES_RACE := github.com/algorand/go-algorand/cmd/kmd
@@ -209,6 +219,8 @@ clean:
 	cd crypto/libsodium-fork && \
 		test ! -e Makefile || make clean
 	rm -rf crypto/lib
+	rm -rf crypto/libs
+	rm -rf crypto/copies
 
 # clean without crypto
 cleango:
@@ -265,3 +277,13 @@ install: build
 	scripts/dev_install.sh -p $(GOPATH1)/bin
 
 .PHONY: default fmt vet lint check_license check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN
+
+### TARGETS FOR CICD PROCESS
+
+ci-deps:
+	scripts/configure_dev-deps.sh && \
+	scripts/check_deps.sh
+
+ci-build: buildsrc gen
+	mkdir -p $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH) && \
+	PKG_ROOT=$(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH) NO_BUILD=True VARIATIONS=$(OS_TYPE)/$(ARCH) scripts/build_packages.sh $(OS_TYPE)/$(ARCH)

agreement/abstractions.go

@@ -201,9 +201,9 @@ type LedgerWriter interface {
 	// as above.
 	EnsureValidatedBlock(ValidatedBlock, Certificate)
 
-	// EnsureDigest waits until some Block that corresponds to a given
-	// Certificate appears in the ledger.  EnsureDigest does not wait for
-	// the block to be written to disk; use Wait() if needed.
+	// EnsureDigest signals the Ledger to attempt to fetch a Block matching
+	// the given Certificate.  EnsureDigest does not wait for the block to
+	// be written to disk; use Wait() if needed.
 	//
 	// The Ledger must guarantee that after this method returns, any Seed,
 	// Record, or Circulation call reflects the contents of the Block
@@ -215,7 +215,7 @@ type LedgerWriter interface {
 	// this is the case, the behavior of Ledger is undefined.
 	// (Implementations are encouraged to panic or otherwise fail loudly in
 	// this case, because it means that a fork has occurred.)
-	EnsureDigest(Certificate, chan struct{}, *AsyncVoteVerifier)
+	EnsureDigest(Certificate, *AsyncVoteVerifier)
 }
 
 // A KeyManager stores and deletes participation keys.

agreement/actions.go

@@ -46,6 +46,7 @@ const (
 
 	// ledger
 	ensure
+	stageDigest
 
 	// time
 	rezero
@@ -123,7 +124,7 @@ func (a networkAction) do(ctx context.Context, s *Service) {
 	if a.T == broadcastVotes {
 		tag := protocol.AgreementVoteTag
 		for i, uv := range a.UnauthenticatedVotes {
-			data := protocol.Encode(uv)
+			data := protocol.Encode(&uv)
 			sendErr := s.Network.Broadcast(tag, data)
 			if sendErr != nil {
 				s.log.Warnf("Network was unable to queue votes for broadcast(%v). %d / %d votes for round %d period %d step %d were dropped.",
@@ -144,16 +145,16 @@ func (a networkAction) do(ctx context.Context, s *Service) {
 	var data []byte
 	switch a.Tag {
 	case protocol.AgreementVoteTag:
-		data = protocol.Encode(a.UnauthenticatedVote)
+		data = protocol.Encode(&a.UnauthenticatedVote)
 	case protocol.VoteBundleTag:
-		data = protocol.Encode(a.UnauthenticatedBundle)
+		data = protocol.Encode(&a.UnauthenticatedBundle)
 	case protocol.ProposalPayloadTag:
 		msg := a.CompoundMessage
 		payload := transmittedPayload{
 			unauthenticatedProposal: msg.Proposal,
 			PriorVote:               msg.Vote,
 		}
-		data = protocol.Encode(payload)
+		data = protocol.Encode(&payload)
 	}
 
 	switch a.T {
@@ -178,6 +179,7 @@ type cryptoAction struct {
 	Proposal  proposalValue // TODO deprecate
 	Round     round
 	Period    period
+	Step      step
 	Pinned    bool
 	TaskIndex int
 }
@@ -197,15 +199,16 @@ func (a cryptoAction) do(ctx context.Context, s *Service) {
 	case verifyPayload:
 		s.demux.verifyPayload(ctx, a.M, a.Round, a.Period, a.Pinned)
 	case verifyBundle:
-		s.demux.verifyBundle(ctx, a.M, a.Round, a.Period)
+		s.demux.verifyBundle(ctx, a.M, a.Round, a.Period, a.Step)
 	}
 }
 
 type ensureAction struct {
 	nonpersistent
 
-	Payload     proposal
-	PayloadOk   bool
+	// the payload that we will give to the ledger
+	Payload proposal
+	// the certificate proving commitment
 	Certificate Certificate
 }
 
@@ -214,7 +217,7 @@ func (a ensureAction) t() actionType {
 }
 
 func (a ensureAction) String() string {
-	return fmt.Sprintf("%v: %.5v", a.t().String(), a.Payload.Digest().String())
+	return fmt.Sprintf("%s: %.5s: %v, %v, %.5s", a.t().String(), a.Payload.Digest().String(), a.Certificate.Round, a.Certificate.Period, a.Certificate.Proposal.BlockDigest.String())
 }
 
 func (a ensureAction) do(ctx context.Context, s *Service) {
@@ -227,7 +230,7 @@ func (a ensureAction) do(ctx context.Context, s *Service) {
 
 	if a.Payload.ve != nil {
 		logEvent.Type = logspec.RoundConcluded
-		s.log.with(logEvent).Infof("committed round %v with pre-validated block %v", a.Certificate.Round, a.Certificate.Proposal)
+		s.log.with(logEvent).Infof("committed round %d with pre-validated block %v", a.Certificate.Round, a.Certificate.Proposal)
 		s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{
 			Address: a.Certificate.Proposal.OriginalProposer.String(),
 			Hash:    a.Certificate.Proposal.BlockDigest.String(),
@@ -236,28 +239,48 @@ func (a ensureAction) do(ctx context.Context, s *Service) {
 		s.Ledger.EnsureValidatedBlock(a.Payload.ve, a.Certificate)
 	} else {
 		block := a.Payload.Block
-		if !a.PayloadOk {
-			logEvent.Type = logspec.RoundWaiting
-			s.log.with(logEvent).Infof("round %v concluded without block for %v; waiting on ledger", a.Certificate.Round, a.Certificate.Proposal)
-			s.Ledger.EnsureDigest(a.Certificate, s.quit, s.voteVerifier)
-		} else {
-			logEvent.Type = logspec.RoundConcluded
-			s.log.with(logEvent).Infof("committed round %v with block %v", a.Certificate.Round, a.Certificate.Proposal)
-			s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{
-				Address: a.Certificate.Proposal.OriginalProposer.String(),
-				Hash:    a.Certificate.Proposal.BlockDigest.String(),
-				Round:   uint64(a.Certificate.Round),
-			})
-			s.Ledger.EnsureBlock(block, a.Certificate)
-		}
+		logEvent.Type = logspec.RoundConcluded
+		s.log.with(logEvent).Infof("committed round %d with block %v", a.Certificate.Round, a.Certificate.Proposal)
+		s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{
+			Address: a.Certificate.Proposal.OriginalProposer.String(),
+			Hash:    a.Certificate.Proposal.BlockDigest.String(),
+			Round:   uint64(a.Certificate.Round),
+		})
+		s.Ledger.EnsureBlock(block, a.Certificate)
 	}
 	logEventStart := logEvent
 	logEventStart.Type = logspec.RoundStart
-	s.log.with(logEventStart).Infof("finished round %v", a.Certificate.Round)
+	s.log.with(logEventStart).Infof("finished round %d", a.Certificate.Round)
 	s.tracer.timeR().StartRound(a.Certificate.Round + 1)
 	s.tracer.timeR().RecStep(0, propose, bottom)
 }
 
+type stageDigestAction struct {
+	nonpersistent
+	// Certificate identifies a block and is a proof commitment
+	Certificate Certificate // a block digest is probably sufficient; keep certificate for now to match ledger interface
+}
+
+func (a stageDigestAction) t() actionType {
+	return stageDigest
+}
+
+func (a stageDigestAction) String() string {
+	return fmt.Sprintf("%s: %.5s. %v. %v", a.t().String(), a.Certificate.Proposal.BlockDigest.String(), a.Certificate.Round, a.Certificate.Period)
+}
+
+func (a stageDigestAction) do(ctx context.Context, service *Service) {
+	logEvent := logspec.AgreementEvent{
+		Hash:   a.Certificate.Proposal.BlockDigest.String(),
+		Round:  uint64(a.Certificate.Round),
+		Period: uint64(a.Certificate.Period),
+		Sender: a.Certificate.Proposal.OriginalProposer.String(),
+		Type:   logspec.RoundWaiting,
+	}
+	service.log.with(logEvent).Infof("round %v concluded without block for %v; (async) waiting on ledger", a.Certificate.Round, a.Certificate.Proposal)
+	service.Ledger.EnsureDigest(a.Certificate, service.voteVerifier)
+}
+
 type rezeroAction struct {
 	nonpersistent
 
@@ -408,10 +431,6 @@ func relayAction(e messageEvent, tag protocol.Tag, o interface{}) action {
 	return a
 }
 
-func verifyBundleAction(e messageEvent, r round, p period) action {
-	return cryptoAction{T: verifyBundle, M: e.Input, Round: r, Period: p}
-}
-
 func verifyVoteAction(e messageEvent, r round, p period, taskIndex int) action {
 	return cryptoAction{T: verifyVote, M: e.Input, Round: r, Period: p, TaskIndex: taskIndex}
 }
@@ -420,6 +439,10 @@ func verifyPayloadAction(e messageEvent, r round, p period, pinned bool) action
 	return cryptoAction{T: verifyPayload, M: e.Input, Round: r, Period: p, Pinned: pinned}
 }
 
+func verifyBundleAction(e messageEvent, r round, p period, s step) action {
+	return cryptoAction{T: verifyBundle, M: e.Input, Round: r, Period: p, Step: s}
+}
+
 func zeroAction(t actionType) action {
 	switch t {
 	case noop: