Merge branch 'main' into improve-refcloser-cache

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -72,6 +72,14 @@ body:
         be automatically formatted into code, so no need for backticks. Or paste
         gists, pastebins links here
       render: Shell
+  - type: textarea
+    id: stuck_node
+    attributes:
+      label: Is the node "stuck"? Has it stopped syncing?
+      description: Please share the state of the node e.g. what height is stuck at, what is the state of DAS?
+      placeholder: >
+        Please consult our RPC docs https://node-rpc-docs.celestia.org/ and share output from any or all of
+        Das.SamplingStats, Header.SyncState and Header.NetworkHead
   - type: textarea
     id: misc
     attributes:

.github/workflows/go-ci.yml

@@ -41,7 +41,7 @@ jobs:
           go-version: ${{ inputs.go-version }}
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6.0.1
+        uses: golangci/golangci-lint-action@v6.1.0
         with:
           args: --timeout 10m
           version: v1.59

.github/workflows/labels.yml

@@ -17,3 +17,33 @@ jobs:
           mode: minimum
           count: 1
           labels: "kind:fix, kind:misc, kind:break!, kind:refactor, kind:feat, kind:deps, kind:docs, kind:ci, kind:chore, kind:testing" # yamllint disable-line rule:line-length
+
+  # will attempt to apply a breaking label
+  # on opening the PR but not enforce it on repeated changes
+  # so we don't get trapped by false positives (for now)
+  # we can expand to all cases after
+  apply-breaking:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'opened' && github.actor != 'dependabot[bot]' }}
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run check for breaking
+        id: breaking_change
+        run: |
+          git fetch origin main
+          make detect-breaking
+
+      - name: Add label if breaking changes detected
+        if: failure()
+        run: gh issue edit "$NUMBER" --add-label "$LABELS"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          NUMBER: ${{ github.event.pull_request.number }}
+          LABELS: kind:break!

Dockerfile

@@ -25,7 +25,7 @@ RUN uname -a &&\
     CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
     make build && make cel-key
 
-FROM docker.io/alpine:3.20.1
+FROM docker.io/alpine:3.20.2
 
 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000
 ARG UID=10001

Makefile

@@ -39,7 +39,7 @@ build:
 	@go build -o build/ ${LDFLAGS} ./cmd/celestia
 .PHONY: build
 
-## build-jemalloc: Build celestia-node binary with jemalloc allocator for BadgerDB instead of Go's native one
+## build-jemalloc: Build celestia-node binary with jemalloc allocator for BadgerDB.
 build-jemalloc: jemalloc
 	@echo "--> Building Celestia with jemalloc"
 	@go build -o build/ ${LDFLAGS} -tags jemalloc ./cmd/celestia
@@ -51,14 +51,14 @@ clean:
 	@rm -rf build/*
 .PHONY: clean
 
-## cover: generate to code coverage report.
+## cover: Generate code coverage report.
 cover:
 	@echo "--> Generating Code Coverage"
 	@go install github.com/ory/go-acc@latest
 	@go-acc -o coverage.txt `go list ./... | grep -v nodebuilder/tests` -- -v
 .PHONY: cover
 
-## deps: install dependencies.
+## deps: Install dependencies.
 deps:
 	@echo "--> Installing Dependencies"
 	@go mod download
@@ -73,7 +73,7 @@ else
 endif
 .PHONY: install
 
-## install-global: Install the celestia-node binary (only for systems that support GNU coreutils, i.e. Linux).
+## install-global: Install the celestia-node binary (only for systems that support GNU coreutils like Linux).
 install-global:
 	@echo "--> Installing Celestia"
 	@install -v ./build/* -t ${PREFIX}/bin
@@ -109,7 +109,8 @@ install-key:
 	@go install ./cmd/cel-key
 .PHONY: install-key
 
-## fmt: Formats only *.go (excluding *.pb.go *pb_test.go). Runs `gofmt & goimports` internally.
+## fmt: Formats only *.go (excluding *.pb.go *pb_test.go).
+# Runs `gofmt & goimports` internally.
 fmt: sort-imports
 	@find . -name '*.go' -type f -not -path "*.git*" -not -name '*.pb.go' -not -name '*pb_test.go' | xargs gofmt -w -s
 	@go mod tidy -compat=1.20
@@ -118,39 +119,40 @@ fmt: sort-imports
 	@markdownlint --fix --quiet --config .markdownlint.yaml .
 .PHONY: fmt
 
-## lint: Linting *.go files using golangci-lint. Look for .golangci.yml for the list of linters. Also lint *.md files using markdownlint.
+## lint: Lint *.go files with golangci-lint and *.md files with markdownlint.
+# Look at .golangci.yml for the list of linters.
 lint: lint-imports
 	@echo "--> Running linter"
 	@golangci-lint run
 	@markdownlint --config .markdownlint.yaml '**/*.md'
 	@cfmt --m=120 ./...
 .PHONY: lint
 
-## test-unit: Running unit tests
+## test-unit: Run unit tests.
 test-unit:
 	@echo "--> Running unit tests"
 	@go test $(VERBOSE) -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` $(LOG_AND_FILTER)
 .PHONY: test-unit
 
-## test-unit-race: Running unit tests with data race detector
+## test-unit-race: Run unit tests with data race detector.
 test-unit-race:
 	@echo "--> Running unit tests with data race detector"
 	@go test $(VERBOSE) -race -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` $(LOG_AND_FILTER)
 .PHONY: test-unit-race
 
-## test-integration: Running /integration tests located in nodebuilder/tests
+## test-integration: Run integration tests located in nodebuilder/tests.
 test-integration:
 	@echo "--> Running integrations tests $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH)"
 	@go test $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH) ./nodebuilder/tests
 .PHONY: test-integration
 
-## test-integration-race: Running integration tests with data race detector located in node/tests
+## test-integration-race: Run integration tests with data race detector located in nodebuilder/tests.
 test-integration-race:
 	@echo "--> Running integration tests with data race detector -tags=$(TAGS)"
 	@go test -race -tags=$(TAGS) ./nodebuilder/tests
 .PHONY: test-integration-race
 
-## benchmark: Running all benchmarks
+## benchmark: Run all benchmarks.
 benchmark:
 	@echo "--> Running benchmarks"
 	@go test -run="none" -bench=. -benchtime=100x -benchmem ./...
@@ -173,14 +175,14 @@ pb-gen:
 	done;
 .PHONY: pb-gen
 
-## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api
+## openrpc-gen: Generate OpenRPC spec for celestia-node's RPC API.
 openrpc-gen:
 	@go run ${LDFLAGS} ./cmd/celestia docgen
 .PHONY: openrpc-gen
 
 ## lint-imports: Lint only Go imports.
-## flag -set-exit-status doesn't exit with code 1 as it should, so we use find until it is fixed by goimports-reviser
 lint-imports:
+# flag -set-exit-status doesn't exit with code 1 as it should, so we use find until it is fixed by goimports-reviser
 	@echo "--> Running imports linter"
 	@for file in `find . -type f -name '*.go'`; \
 		do goimports-reviser -list-diff -set-exit-status -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/celestia-node" -output stdout $$file \
@@ -199,13 +201,13 @@ adr-gen:
 	@curl -sSL https://raw.githubusercontent.com/celestiaorg/.github/main/adr-template.md > docs/architecture/adr-$(NUM)-$(TITLE).md
 .PHONY: adr-gen
 
-## telemetry-infra-up: launches local telemetry infrastructure. This includes grafana, jaeger, loki, pyroscope, and an otel-collector.
-## you can access the grafana instance at localhost:3000 and login with admin:admin.
+## telemetry-infra-up: Launch local telemetry infra (grafana, jaeger, loki, pyroscope, and otel-collector).
+# You can access the grafana instance at localhost:3000 and login with admin:admin.
 telemetry-infra-up:
 	PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml up
 .PHONY: telemetry-infra-up
 
-## telemetry-infra-down: tears the telemetry infrastructure down. The stores for grafana, prometheus, and loki will persist.
+## telemetry-infra-down: Tears the telemetry infra down. Persists the stores for grafana, prometheus, and loki.
 telemetry-infra-down:
 	PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml down
 .PHONY: telemetry-infra-down
@@ -222,17 +224,44 @@ goreleaser-build:
 	goreleaser build --snapshot --clean --single-target
 .PHONY: goreleaser-build
 
-## goreleaser-release: Builds the release celestia binaries as defined in .goreleaser.yaml. This requires there be a git tag for the release in the local git history.
+## goreleaser-release: Builds the release celestia binaries as defined in .goreleaser.yaml.
+# This requires there be a git tag for the release in the local git history.
 goreleaser-release:
 	goreleaser release --clean --fail-fast --skip-publish
 .PHONY: goreleaser-release
 
+# detect changed files and parse output
+# to inspect changes to nodebuilder/**/config.go fields
+CHANGED_FILES      = $(shell git diff --name-only origin/main...HEAD)
+detect-breaking:
+	@BREAK=false
+	@for file in ${CHANGED_FILES}; do \
+		if echo $$file | grep -qE '\.proto$$'; then \
+			BREAK=true; \
+		fi; \
+		if echo $$file | grep -qE 'nodebuilder/.*/config\.go'; then \
+			DIFF_OUTPUT=$$(git diff origin/main...HEAD $$file); \
+			if echo "$$DIFF_OUTPUT" | grep -qE 'type Config struct|^\s+\w+\s+Config'; then \
+				BREAK=true; \
+			fi; \
+		fi; \
+	done; \
+	if [ "$$BREAK" = true ]; then \
+		echo "break detected"; \
+		exit 1; \
+	else \
+		echo "no break detected"; \
+		exit 0; \
+	fi
+.PHONY: detect-breaking
+
+
 # Copied from https://github.com/dgraph-io/badger/blob/main/Makefile
 USER_ID      = $(shell id -u)
 HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc")
 JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
 
-## jemalloc installs jemalloc allocator
+## jemalloc: Install jemalloc allocator.
 jemalloc:
 	@if [ -z "$(HAS_JEMALLOC)" ] ; then \
 		mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; \

blob/commitment_proof.go

@@ -4,9 +4,9 @@ import (
 	"bytes"
 	"fmt"
 
-	"github.com/celestiaorg/celestia-app/pkg/appconsts"
-	"github.com/celestiaorg/celestia-app/pkg/shares"
+	"github.com/celestiaorg/celestia-app/v2/pkg/appconsts"
 	"github.com/celestiaorg/celestia-app/v2/pkg/proof"
+	"github.com/celestiaorg/go-square/inclusion"
 	"github.com/celestiaorg/nmt"
 	"github.com/celestiaorg/nmt/namespace"
 
@@ -98,7 +98,7 @@ func (commitmentProof *CommitmentProof) Verify(root []byte, subtreeRootThreshold
 	// the subtree roots width is defined in ADR-013:
 	//
 	//https://github.com/celestiaorg/celestia-app/blob/main/docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md
-	subtreeRootsWidth := shares.SubTreeWidth(numberOfShares, subtreeRootThreshold)
+	subtreeRootsWidth := inclusion.SubTreeWidth(numberOfShares, subtreeRootThreshold)
 
 	// verify the proof of the subtree roots
 	subtreeRootsCursor := 0

blob/helper_test.go

@@ -4,11 +4,12 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/celestiaorg/celestia-app/v2/pkg/appconsts"
 	squareblob "github.com/celestiaorg/go-square/blob"
 	"github.com/celestiaorg/go-square/namespace"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestBlobsToShares(t *testing.T) {

blob/service.go

@@ -16,7 +16,7 @@ import (
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
 
-	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/v2/pkg/appconsts"
 	pkgproof "github.com/celestiaorg/celestia-app/v2/pkg/proof"
 	"github.com/celestiaorg/go-square/inclusion"
 	appns "github.com/celestiaorg/go-square/namespace"

core/header_test.go

@@ -2,12 +2,15 @@ package core
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/tendermint/tendermint/libs/rand"
 
+	"github.com/celestiaorg/celestia-app/v2/pkg/appconsts"
+
 	"github.com/celestiaorg/celestia-node/header"
 	"github.com/celestiaorg/celestia-node/header/headertest"
 	"github.com/celestiaorg/celestia-node/share"
@@ -48,3 +51,12 @@ func TestMismatchedDataHash_ComputedRoot(t *testing.T) {
 	assert.Contains(t, err.Error(), "mismatch between data hash commitment from"+
 		" core header and computed data root")
 }
+
+func TestBadAppVersion(t *testing.T) {
+	header := headertest.RandExtendedHeader(t)
+	header.RawHeader.Version.App = appconsts.LatestVersion + 1
+
+	err := header.Validate()
+	assert.Contains(t, err.Error(), fmt.Sprintf("has version %d, this node supports up to version %d. Please "+
+		"upgrade to support new version", header.RawHeader.Version.App, appconsts.LatestVersion))
+}

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/BurntSushi/toml v1.4.0
 	github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b
 	github.com/benbjohnson/clock v1.3.5
-	github.com/celestiaorg/celestia-app v1.13.0
 	github.com/celestiaorg/celestia-app/v2 v2.0.0
 	github.com/celestiaorg/go-fraud v0.2.1
 	github.com/celestiaorg/go-header v0.6.2

go.sum

@@ -354,8 +354,6 @@ github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOC
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 h1:h1Y4V3EMQ2mFmNtWt2sIhZIuyASInj1a9ExI8xOsTOw=
 github.com/celestiaorg/blobstream-contracts/v3 v3.1.0/go.mod h1:x4DKyfKOSv1ZJM9NwV+Pw01kH2CD7N5zTFclXIVJ6GQ=
-github.com/celestiaorg/celestia-app v1.13.0 h1:7MWEox6lim6WDyiP84Y2/ERfWUJxWPfZlKxzO6OFcig=
-github.com/celestiaorg/celestia-app v1.13.0/go.mod h1:CF9VZwWAlTU0Is/BOsmxqkbkYnnmrgl0YRlSBIzr0m0=
 github.com/celestiaorg/celestia-app/v2 v2.0.0 h1:cGrkLbqbaNqj+g+LhvmQ0iFKFyrD/GdU5z3WQI/bq8E=
 github.com/celestiaorg/celestia-app/v2 v2.0.0/go.mod h1:0wP0W+GLghvsODxLhAYToKsy0RFKeg1HdZftQG90W5A=
 github.com/celestiaorg/celestia-core v1.38.0-tm-v0.34.29 h1:HwbA4OegRvXX0aNchBA7Cmu+oIxnH7xRcOhISuDP0ak=

header/header.go

@@ -11,8 +11,7 @@ import (
 	"github.com/tendermint/tendermint/light"
 	core "github.com/tendermint/tendermint/types"
 
-	v1 "github.com/celestiaorg/celestia-app/v2/pkg/appconsts/v1"
-	v2 "github.com/celestiaorg/celestia-app/v2/pkg/appconsts/v2"
+	"github.com/celestiaorg/celestia-app/v2/pkg/appconsts"
 	"github.com/celestiaorg/celestia-app/v2/pkg/da"
 	libhead "github.com/celestiaorg/go-header"
 	"github.com/celestiaorg/rsmt2d"
@@ -115,13 +114,10 @@ func (eh *ExtendedHeader) Validate() error {
 		return fmt.Errorf("ValidateBasic error on RawHeader at height %d: %w", eh.Height(), err)
 	}
 
-	if eh.RawHeader.Version.App != v1.Version && eh.RawHeader.Version.App != v2.Version {
-		return fmt.Errorf(
-			"app version mismatch, expected: %d or %d, got %d",
-			v1.Version,
-			v2.Version,
-			eh.RawHeader.Version.App,
-		)
+	if eh.RawHeader.Version.App == 0 || eh.RawHeader.Version.App > appconsts.LatestVersion {
+		return fmt.Errorf("header received at height %d has version %d, this node supports up "+
+			"to version %d. Please upgrade to support new version. Note, 0 is not a valid version",
+			eh.RawHeader.Height, eh.RawHeader.Version.App, appconsts.LatestVersion)
 	}
 
 	err = eh.Commit.ValidateBasic()