feat: add more metrics

Makefile

@@ -121,6 +121,10 @@ build: $(BUILD_DEPS)
 	rm -f venus
 	$(GO) build -o ./venus $(GOFLAGS) .
 
+debug: $(BUILD_DEPS)
+	rm -f venus
+	$(GO) build -o ./venus -gcflags=all="-N -l" $(GOFLAGS) .
+
 
 .PHONY: docker
 TAG:=test

app/node/node.go

@@ -31,8 +31,9 @@ import (
 	_ "github.com/filecoin-project/venus/pkg/crypto/bls"       // enable bls signatures
 	_ "github.com/filecoin-project/venus/pkg/crypto/delegated" // enable delegated signatures
 	_ "github.com/filecoin-project/venus/pkg/crypto/secp"      // enable secp signatures
-	"github.com/filecoin-project/venus/pkg/metrics"
+	metricsPKG "github.com/filecoin-project/venus/pkg/metrics"
 	"github.com/filecoin-project/venus/pkg/repo"
+	"github.com/ipfs-force-community/metrics"
 	"github.com/ipfs-force-community/sophon-auth/jwtclient"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
@@ -46,6 +47,10 @@ import (
 
 var log = logging.Logger("node") // nolint: deadcode
 
+var (
+	apiStatusGauge = metrics.NewInt64("api/status", "Status of the API server. 1 is up, 0 is down.", "")
+)
+
 // ConfigOpt mutates a node config post initialization
 type ConfigOpt func(*config.Config)
 
@@ -157,11 +162,11 @@ func (node *Node) OfflineMode() bool {
 // Start boots up the node.
 func (node *Node) Start(ctx context.Context) error {
 	var err error
-	if err = metrics.RegisterPrometheusEndpoint(node.repo.Config().Observability.Metrics); err != nil {
+	if err = metricsPKG.RegisterPrometheusEndpoint(node.repo.Config().Observability.Metrics); err != nil {
 		return errors.Wrap(err, "failed to setup metrics")
 	}
 
-	if node.jaeger, err = metrics.SetupJaegerTracing(node.network.Host.ID().Pretty(),
+	if node.jaeger, err = metricsPKG.SetupJaegerTracing(node.network.Host.ID().Pretty(),
 		node.repo.Config().Observability.Tracing); err != nil {
 		return errors.Wrap(err, "failed to setup tracing")
 	}
@@ -239,7 +244,7 @@ func (node *Node) Stop(ctx context.Context) {
 	}
 
 	if node.jaeger != nil {
-		if err := metrics.ShutdownJaeger(ctx, node.jaeger); err != nil {
+		if err := metricsPKG.ShutdownJaeger(ctx, node.jaeger); err != nil {
 			log.Warnf("error shutdown jaeger-tracing: %w", err)
 		}
 	}
@@ -297,8 +302,10 @@ func (node *Node) RunRPCAndWait(ctx context.Context, rootCmdDaemon *cmds.Command
 	}
 
 	go func() {
+		apiStatusGauge.Set(ctx, 1)
 		err := apiServ.Serve(netListener) // nolint
 		if err != nil && err != http.ErrServerClosed {
+			apiStatusGauge.Set(ctx, 0)
 			return
 		}
 	}()
@@ -319,6 +326,7 @@ func (node *Node) RunRPCAndWait(ctx context.Context, rootCmdDaemon *cmds.Command
 		if err := apiServ.Shutdown(ctx); err != nil {
 			log.Warnf("failed to shutdown server: %v", err)
 		}
+		apiStatusGauge.Set(ctx, 0)
 		node.Stop(ctx)
 		memguard.Purge()
 		log.Infof("venus shutdown gracefully ...")

app/submodule/chain/chaininfo_api.go

@@ -18,10 +18,12 @@ import (
 	acrypto "github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/network"
 	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/ipfs-force-community/metrics"
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	logging "github.com/ipfs/go-log/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.opencensus.io/tag"
 
 	"github.com/filecoin-project/venus/pkg/chain"
 	"github.com/filecoin-project/venus/pkg/constants"
@@ -41,6 +43,11 @@ type chainInfoAPI struct { //nolint
 
 var log = logging.Logger("chain")
 
+var (
+	tagKeyRandomnessRequestType = tag.MustNewKey("randomness_request_type")
+	randomnessRequestStatus     = metrics.NewInt64WithCounter("api/randomness_status", "Status of randomness request. 0 = fail, 1 = success", "", tagKeyRandomnessRequestType)
+)
+
 // NewChainInfoAPI new chain info api
 func NewChainInfoAPI(chain *ChainSubmodule) v1api.IChainInfo {
 	return &chainInfoAPI{chain: chain}
@@ -379,7 +386,7 @@ func (cia *chainInfoAPI) getNetworkName(ctx context.Context) (string, error) {
 }
 
 // StateGetRandomnessFromTickets is used to sample the chain for randomness.
-func (cia *chainInfoAPI) StateGetRandomnessFromTickets(ctx context.Context, personalization acrypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) {
+func (cia *chainInfoAPI) StateGetRandomnessFromTickets(ctx context.Context, personalization acrypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (rand abi.Randomness, err error) {
 	ts, err := cia.ChainGetTipSet(ctx, tsk)
 	if err != nil {
 		return nil, fmt.Errorf("loading tipset %s: %w", tsk, err)
@@ -391,11 +398,18 @@ func (cia *chainInfoAPI) StateGetRandomnessFromTickets(ctx context.Context, pers
 		return nil, fmt.Errorf("getting chain randomness: %w", err)
 	}
 
-	return chain.DrawRandomnessFromDigest(digest, personalization, randEpoch, entropy)
+	ctx, _ = tag.New(ctx, tag.Upsert(tagKeyRandomnessRequestType, utils.Name(personalization)))
+	ret, err := chain.DrawRandomnessFromDigest(digest, personalization, randEpoch, entropy)
+	if err != nil {
+		randomnessRequestStatus.Set(ctx, 0)
+		return nil, err
+	}
+	randomnessRequestStatus.Set(ctx, 1)
+	return ret, nil
 }
 
 // StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
-func (cia *chainInfoAPI) StateGetRandomnessFromBeacon(ctx context.Context, personalization acrypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) {
+func (cia *chainInfoAPI) StateGetRandomnessFromBeacon(ctx context.Context, personalization acrypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (rand abi.Randomness, err error) {
 	ts, err := cia.ChainGetTipSet(ctx, tsk)
 	if err != nil {
 		return nil, fmt.Errorf("loading tipset %s: %w", tsk, err)
@@ -406,7 +420,14 @@ func (cia *chainInfoAPI) StateGetRandomnessFromBeacon(ctx context.Context, perso
 		return nil, fmt.Errorf("getting beacon randomness: %w", err)
 	}
 
-	return chain.DrawRandomnessFromDigest(digest, personalization, randEpoch, entropy)
+	ctx, _ = tag.New(ctx, tag.Upsert(tagKeyRandomnessRequestType, utils.Name(personalization)))
+	ret, err := chain.DrawRandomnessFromDigest(digest, personalization, randEpoch, entropy)
+	if err != nil {
+		randomnessRequestStatus.Set(ctx, 0)
+		return nil, err
+	}
+	randomnessRequestStatus.Set(ctx, 1)
+	return ret, nil
 }
 
 func (cia *chainInfoAPI) StateGetRandomnessDigestFromTickets(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (abi.Randomness, error) {

go.mod

@@ -56,7 +56,7 @@ require (
 	github.com/hashicorp/golang-lru/arc/v2 v2.0.7
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c
-	github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601
+	github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d
 	github.com/ipfs-force-community/sophon-auth v1.14.0
 	github.com/ipfs/boxo v0.10.1
 	github.com/ipfs/go-cid v0.4.1
@@ -95,7 +95,6 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230923211252-36a87e1ba72f
-	github.com/whyrusleeping/go-logging v0.0.1
 	github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1
 	github.com/zyedidia/generic v1.2.1
 	go.opencensus.io v0.24.0
@@ -124,6 +123,7 @@ require (
 	github.com/ipfs/go-merkledag v0.11.0 // indirect
 	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
+	github.com/whyrusleeping/go-logging v0.0.1 // indirect
 )
 
 require (

go.sum

@@ -592,8 +592,8 @@ github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
 github.com/ipfs-force-community/go-jsonrpc v0.1.9 h1:5QavBltfvV6fz/+EbYsCkVxJ1MSJncZm6YuPs1SLdZU=
 github.com/ipfs-force-community/go-jsonrpc v0.1.9/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM=
-github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601 h1:zxKQ30KAD6KfvSFAx9tuqQXLDsEHyF+eVaUBXXYC2bU=
-github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA=
+github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d h1:EJQONl9uXat91SpVWE575+Xqyx14eRfTNjCHfdSYwKQ=
+github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA=
 github.com/ipfs-force-community/sophon-auth v1.14.0 h1:ctBJ6UHkcytEzfVPgiiHo0cW4FGQrE7r1H3Um0FcHbo=
 github.com/ipfs-force-community/sophon-auth v1.14.0/go.mod h1:d6J6u3zyIwcEajRho5BhVBcoIChEf0K76wP4yJEfEhc=
 github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=

pkg/chainsync/syncer/syncer.go

@@ -9,11 +9,14 @@ import (
 
 	"github.com/filecoin-project/venus/pkg/consensus"
 	"github.com/filecoin-project/venus/pkg/crypto"
+	"github.com/filecoin-project/venus/pkg/repo"
 	"github.com/filecoin-project/venus/pkg/statemanger"
 	"github.com/hashicorp/go-multierror"
+	"github.com/ipfs-force-community/metrics"
 
 	"golang.org/x/sync/errgroup"
 
+	actorsTypes "github.com/filecoin-project/go-state-types/actors"
 	syncTypes "github.com/filecoin-project/venus/pkg/chainsync/types"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
@@ -22,7 +25,6 @@ import (
 	"github.com/filecoin-project/venus/pkg/clock"
 	"github.com/filecoin-project/venus/pkg/constants"
 	"github.com/filecoin-project/venus/pkg/fork"
-	"github.com/filecoin-project/venus/pkg/metrics"
 	"github.com/filecoin-project/venus/pkg/metrics/tracing"
 	"github.com/filecoin-project/venus/pkg/net/exchange"
 	"github.com/filecoin-project/venus/venus-shared/actors/policy"
@@ -59,15 +61,19 @@ var (
 	// ErrUnexpectedStoreState indicates that the syncer's chain bsstore is violating expected invariants.
 	ErrUnexpectedStoreState = errors.New("the chain bsstore is in an unexpected state")
 
-	logSyncer    = logging.Logger("chainsync.syncer")
-	syncOneTimer *metrics.Float64Timer
-	reorgCnt     *metrics.Int64Counter // nolint
+	logSyncer = logging.Logger("chainsync.syncer")
 )
 
-func init() {
-	syncOneTimer = metrics.NewTimerMs("syncer/sync_one", "Duration of single tipset validation in milliseconds")
-	reorgCnt = metrics.NewInt64Counter("chain/reorg_count", "The number of reorgs that have occurred.")
-}
+// metrics handlers
+var (
+	// epoch should not use as a label, because it has a lot of values
+	syncOneTimer      = metrics.NewTimerMs("sync/sync_one", "Duration of single tipset validation in milliseconds")
+	syncStatus        = metrics.NewInt64("sync/status", "The current status of the syncer. 0 = sync delay, 1 = sync done", "")
+	reorgCnt          = metrics.NewCounter("chain/reorg_count", "The number of reorgs that have occurred.") // nolint
+	epochGauge        = metrics.NewInt64("chain/epoch", "The current epoch.", "")
+	netVersionGuage   = metrics.NewInt64("chain/net_version", "The current network version.", "")
+	actorVersionGuage = metrics.NewInt64("chain/actor_version", "The current actor version.", "")
+)
 
 // StateProcessor does semantic validation on fullblocks.
 type StateProcessor interface {
@@ -195,8 +201,8 @@ func (syncer *Syncer) syncOne(ctx context.Context, parent, next *types.TipSet) e
 		return nil
 	}
 
-	stopwatch := syncOneTimer.Start(ctx)
-	defer stopwatch.Stop(ctx)
+	stopwatch := syncOneTimer.Start()
+	defer stopwatch(ctx)
 
 	var err error
 
@@ -235,6 +241,22 @@ func (syncer *Syncer) syncOne(ctx context.Context, parent, next *types.TipSet) e
 
 	syncer.chainStore.PersistTipSetKey(ctx, next.Key())
 
+	// chain related metrics
+	height := next.Height()
+	epochGauge.Set(ctx, int64(height))
+
+	timeStamp := next.MinTimestamp()
+	if timeStamp+repo.Config.NetworkParams.BlockDelay*uint64(time.Second) >= uint64(syncer.clock.Now().Unix()) {
+		// update to latest
+		syncStatus.Set(ctx, 1)
+	} else {
+		syncStatus.Set(ctx, 0)
+	}
+	netVersion := syncer.fork.GetNetworkVersion(ctx, height)
+	netVersionGuage.Set(ctx, int64(netVersion))
+	actorVersion, _ := actorsTypes.VersionForNetwork(netVersion)
+	actorVersionGuage.Set(ctx, int64(actorVersion))
+
 	return nil
 }