diff --git a/.dockerignore b/.dockerignore index c88fb144fe506..f61d8aea3f3d4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -40,9 +40,7 @@ cpu.out *.log /gitea -/gitea-vet /debug -/integrations.test /bin /dist @@ -54,12 +52,6 @@ cpu.out /indexers /log /tests/integration/gitea-integration-* -/tests/integration/indexers-* -/tests/e2e/gitea-e2e-* -/tests/e2e/indexers-* -/tests/e2e/reports -/tests/e2e/test-artifacts -/tests/e2e/test-snapshots /tests/*.ini /node_modules /yarn.lock diff --git a/.github/workflows/cache-seeder.yml b/.github/workflows/cache-seeder.yml index d0801a10783f5..358179e385ea5 100644 --- a/.github/workflows/cache-seeder.yml +++ b/.github/workflows/cache-seeder.yml @@ -57,7 +57,7 @@ jobs: include: - { job: lint-backend, tags: "bindata sqlite sqlite_unlock_notify", target: "lint-backend" } - { job: lint-go-windows, tags: "bindata sqlite sqlite_unlock_notify", target: "lint-go-windows" } - - { job: lint-go-gogit, tags: "bindata sqlite sqlite_unlock_notify gogit", target: "lint-go" } + - { job: lint-go-gogit, tags: "bindata gogit sqlite sqlite_unlock_notify", target: "lint-go" } steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 diff --git a/.github/workflows/part-docker-dryrun.yml b/.github/workflows/part-docker-dryrun.yml new file mode 100644 index 0000000000000..e5eca3dfc1fd9 --- /dev/null +++ b/.github/workflows/part-docker-dryrun.yml @@ -0,0 +1,35 @@ +# Reusable workflow that performs the container build steps for a single platform. +# Used by `pull-docker-dryrun.yml` to run builds in parallel per-platform. +on: + workflow_call: + inputs: + platform: + description: 'The target platform(s) to build for (e.g. linux/amd64)' + required: true + type: string + +jobs: + build-dryrun: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + - name: Build rootful image + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + context: . + platforms: ${{ inputs.platform }} + push: false + file: Dockerfile + cache-from: type=registry,ref=ghcr.io/go-gitea/gitea:buildcache-rootful + - name: Build rootless image + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + context: . + platforms: ${{ inputs.platform }} + push: false + file: Dockerfile.rootless + cache-from: type=registry,ref=ghcr.io/go-gitea/gitea:buildcache-rootless diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index d49fc33daddf3..a9958a8a1ffd0 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -58,13 +58,12 @@ jobs: env: TAGS: bindata - name: run migration tests - run: make test-pgsql-migration + run: GITEA_TEST_DATABASE=pgsql make test-migration - name: run tests - run: make test-pgsql + run: GITEA_TEST_DATABASE=pgsql make test-integration timeout-minutes: 50 env: TAGS: bindata gogit - TEST_TAGS: gogit TEST_LDAP: 1 test-sqlite: @@ -84,18 +83,21 @@ jobs: with: cache-name: sqlite - run: make deps-backend - - run: GOEXPERIMENT='' make backend + - run: make backend env: TAGS: bindata gogit sqlite sqlite_unlock_notify + GOEXPERIMENT: - name: run migration tests - run: make test-sqlite-migration + run: GITEA_TEST_DATABASE=sqlite make test-migration + env: + TAGS: bindata gogit - name: run tests - run: GOEXPERIMENT='' make test-sqlite + run: GITEA_TEST_DATABASE=sqlite make test-integration timeout-minutes: 50 env: - TAGS: bindata gogit sqlite sqlite_unlock_notify - RACE_ENABLED: true - TEST_TAGS: gogit sqlite sqlite_unlock_notify + TAGS: bindata gogit + GOEXPERIMENT: + GOTEST_FLAGS: -race -timeout=40m test-unit: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' @@ -156,16 +158,15 @@ jobs: - name: unit-tests run: make test-backend test-check env: - TAGS: bindata - RACE_ENABLED: true - GOTESTFLAGS: -timeout=20m + TAGS: bindata sqlite sqlite_unlock_notify + GOTEST_FLAGS: -race -timeout=20m GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} - name: unit-tests-gogit - run: GOEXPERIMENT='' make test-backend test-check + run: make test-backend test-check env: - TAGS: bindata gogit - RACE_ENABLED: true - GOTESTFLAGS: -timeout=20m + TAGS: bindata gogit sqlite sqlite_unlock_notify + GOEXPERIMENT: + GOTEST_FLAGS: -race -timeout=20m GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} test-mysql: @@ -215,10 +216,9 @@ jobs: env: TAGS: bindata - name: run migration tests - run: make test-mysql-migration + run: GITEA_TEST_DATABASE=mysql make test-migration - name: run tests - # run: make integration-test-coverage (at the moment, no coverage is really handled) - run: make test-mysql + run: GITEA_TEST_DATABASE=mysql make test-integration env: TAGS: bindata TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" @@ -258,9 +258,9 @@ jobs: - run: make backend env: TAGS: bindata - - run: make test-mssql-migration + - run: GITEA_TEST_DATABASE=mssql make test-migration - name: run tests - run: make test-mssql + run: GITEA_TEST_DATABASE=mssql make test-integration timeout-minutes: 50 env: TAGS: bindata diff --git a/.github/workflows/pull-docker-dryrun.yml b/.github/workflows/pull-docker-dryrun.yml index e0c0fff815d88..702506f660b14 100644 --- a/.github/workflows/pull-docker-dryrun.yml +++ b/.github/workflows/pull-docker-dryrun.yml @@ -7,34 +7,33 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +permissions: + contents: read + jobs: files-changed: uses: ./.github/workflows/files-changed.yml permissions: contents: read - container: + # dryrun build is slow, so run them in parallel per-platform + container-amd64: if: needs.files-changed.outputs.docker == 'true' - needs: files-changed - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - name: Build regular container image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - context: . - platforms: linux/amd64,linux/arm64,linux/riscv64 - push: false - cache-from: type=registry,ref=ghcr.io/go-gitea/gitea:buildcache-rootful - - name: Build rootless container image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - context: . - push: false - platforms: linux/amd64,linux/arm64,linux/riscv64 - file: Dockerfile.rootless - cache-from: type=registry,ref=ghcr.io/go-gitea/gitea:buildcache-rootless + needs: [files-changed] + uses: ./.github/workflows/part-docker-dryrun.yml + with: + platform: linux/amd64 + + container-arm64: + if: needs.files-changed.outputs.docker == 'true' + needs: [files-changed] + uses: ./.github/workflows/part-docker-dryrun.yml + with: + platform: linux/arm64 + + container-riscv64: + if: needs.files-changed.outputs.docker == 'true' + needs: [files-changed] + uses: ./.github/workflows/part-docker-dryrun.yml + with: + platform: linux/riscv64 diff --git a/.github/workflows/pull-e2e-tests.yml b/.github/workflows/pull-e2e-tests.yml index afa9587022760..3b298d94e1d67 100644 --- a/.github/workflows/pull-e2e-tests.yml +++ b/.github/workflows/pull-e2e-tests.yml @@ -39,10 +39,13 @@ jobs: - run: make deps-frontend - run: make frontend - run: make deps-backend - - run: make gitea-e2e + - run: make backend + env: + TAGS: bindata sqlite sqlite_unlock_notify - run: make playwright - run: make test-e2e timeout-minutes: 10 env: + TAGS: bindata sqlite sqlite_unlock_notify FORCE_COLOR: 1 GITEA_TEST_E2E_DEBUG: 1 diff --git a/.gitignore b/.gitignore index 019ee94c7a3bd..d4a677b6fa2e0 100644 --- a/.gitignore +++ b/.gitignore @@ -55,10 +55,7 @@ cpu.out *.log.*.gz /gitea -/gitea-e2e -/gitea-vet /debug -/integrations.test /bin /dist @@ -68,7 +65,6 @@ cpu.out /indexers /log /public/assets/img/avatar -/tests/e2e-output /tests/integration/gitea-integration-* /tests/integration/indexers-* /tests/*.ini diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f0c548dcb4e3..f1871f147013b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,11 +139,11 @@ Here's how to run the test suite: - run tests (we suggest running them on Linux) -| Command | Action | | -| :------------------------------------------ | :------------------------------------------------------- | ------------------------------------------- | -|``make test[\#SpecificTestName]`` | run unit test(s) | | -|``make test-sqlite[\#SpecificTestName]`` | run [integration](tests/integration) test(s) for SQLite | [More details](tests/integration/README.md) | -|``make test-e2e`` | run [end-to-end](tests/e2e) test(s) using Playwright | | +| Command | Action | | +|:----------------------------------------------|:-----------------------------------------------------| ------------------------------------------- | +| ``make test-backend[\#SpecificTestName]`` | run unit test(s) | | +| ``make test-integration[\#SpecificTestName]`` | run [integration](tests/integration) test(s) | [More details](tests/integration/README.md) | +| ``make test-e2e`` | run [end-to-end](tests/e2e) test(s) using Playwright | | - E2E test environment variables diff --git a/Makefile b/Makefile index f85d042279352..7db7313915ead 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,9 @@ export GOEXPERIMENT ?= jsonv2 GO ?= go SHASUM ?= shasum -a 256 -HAS_GO := $(shell hash $(GO) > /dev/null 2>&1 && echo yes) COMMA := , -XGO_VERSION := go-1.25.x +XGO_VERSION := go-1.26.x AIR_PACKAGE ?= github.com/air-verse/air@v1 # renovate: datasource=go EDITORCONFIG_CHECKER_PACKAGE ?= github.com/editorconfig-checker/editorconfig-checker/v3/cmd/editorconfig-checker@v3 # renovate: datasource=go @@ -22,15 +21,28 @@ XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1 # renovate: datasource=go ACTIONLINT_PACKAGE ?= github.com/rhysd/actionlint/cmd/actionlint@v1.7.11 # renovate: datasource=go -DOCKER_IMAGE ?= gitea/gitea -DOCKER_TAG ?= latest -DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG) - +HAS_GO := $(shell hash $(GO) > /dev/null 2>&1 && echo yes) ifeq ($(HAS_GO), yes) CGO_EXTRA_CFLAGS := -DSQLITE_MAX_VARIABLE_NUMBER=32766 CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS) endif +MAKE_EVIDENCE_DIR := .make_evidence + +# Use sqlite as default database if running tests, only do so for local tests, not in CI. +# CI should explicitly set the database to avoid unexpected results. +ifneq ($(findstring test-,$(MAKECMDGOALS)),) + ifeq ($(CI),) + GITEA_TEST_DATABASE ?= sqlite + endif +endif + +TAGS ?= +ifeq ($(GITEA_TEST_DATABASE),sqlite) + TAGS += sqlite sqlite_unlock_notify +endif +TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags + CGO_ENABLED ?= 0 ifneq (,$(findstring sqlite,$(TAGS))$(findstring pam,$(TAGS))) CGO_ENABLED = 1 @@ -49,15 +61,16 @@ else ifeq ($(patsubst Windows%,Windows,$(OS)),Windows) IS_WINDOWS := yes endif endif + +# GOFLAGS and EXTRA_GOFLAGS are for the 'go build' command only ifeq ($(IS_WINDOWS),yes) GOFLAGS := -v -buildmode=exe EXECUTABLE ?= gitea.exe - EXECUTABLE_E2E ?= gitea-e2e.exe else GOFLAGS := -v EXECUTABLE ?= gitea - EXECUTABLE_E2E ?= gitea-e2e endif +EXTRA_GOFLAGS ?= ifeq ($(shell sed --version 2>/dev/null | grep -q GNU && echo gnu),gnu) SED_INPLACE := sed -i @@ -65,15 +78,8 @@ else SED_INPLACE := sed -i '' endif -EXTRA_GOFLAGS ?= - -MAKE_EVIDENCE_DIR := .make_evidence - -GOTESTFLAGS ?= -ifeq ($(RACE_ENABLED),true) - GOFLAGS += -race - GOTESTFLAGS += -race -endif +# GOTEST_FLAGS is for unit test and integration test +GOTEST_FLAGS ?= -timeout 40m STORED_VERSION_FILE := VERSION @@ -126,12 +132,6 @@ AIR_TMP_DIR := .air GO_LICENSE_FILE := assets/go-licenses.json -TAGS ?= -TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS)) -TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags - -TEST_TAGS ?= $(TAGS_SPLIT) sqlite sqlite_unlock_notify - TAR_EXCLUDES := .git data indexers queues log node_modules $(EXECUTABLE) $(DIST) $(MAKE_EVIDENCE_DIR) $(AIR_TMP_DIR) GO_DIRS := build cmd models modules routers services tests tools @@ -170,7 +170,13 @@ TEST_MSSQL_PASSWORD ?= MwantsaSecurePassword1 # Include local Makefile # Makefile.local is listed in .gitignore -sinclude Makefile.local +ifneq ("$(wildcard Makefile.local)","") + include Makefile.local +endif + +$(foreach v, $(filter TEST_%, $(.VARIABLES)), $(eval MAKEFILE_VARS+=$v=$($v))) +$(foreach v, $(filter GITEA_TEST_%, $(.VARIABLES)), $(eval MAKEFILE_VARS+=$v=$($v))) +export MAKEFILE_VARS .PHONY: all all: build @@ -179,15 +185,8 @@ all: build help: Makefile ## print Makefile help information. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m[TARGETS] default target: build\033[0m\n\n\033[35mTargets:\033[0m\n"} /^[0-9A-Za-z._-]+:.*?##/ { printf " \033[36m%-45s\033[0m %s\n", $$1, $$2 }' Makefile #$(MAKEFILE_LIST) @printf " \033[36m%-46s\033[0m %s\n" "test-e2e" "test end to end using playwright" - @printf " \033[36m%-46s\033[0m %s\n" "test[#TestSpecificName]" "run unit test" - @printf " \033[36m%-46s\033[0m %s\n" "test-sqlite[#TestSpecificName]" "run integration test for sqlite" - -.PHONY: git-check -git-check: - @if git lfs >/dev/null 2>&1 ; then : ; else \ - echo "Gitea requires git with lfs support to run tests." ; \ - exit 1; \ - fi + @printf " \033[36m%-46s\033[0m %s\n" "test-backend[#TestSpecificName]" "run unit test (sqlite only)" + @printf " \033[36m%-46s\033[0m %s\n" "test-integration[#TestSpecificName]" "run integration test for GITEA_TEST_DATABASE (sqlite, mysql, pgsql, mssql)" .PHONY: clean-all clean-all: clean ## delete backend, frontend and integration files @@ -195,14 +194,8 @@ clean-all: clean ## delete backend, frontend and integration files .PHONY: clean clean: ## delete backend and integration files - rm -rf $(EXECUTABLE) $(EXECUTABLE_E2E) $(DIST) $(BINDATA_DEST_WILDCARD) \ - integrations*.test \ - tests/integration/gitea-integration-* \ - tests/integration/indexers-* \ - tests/sqlite.ini tests/mysql.ini tests/pgsql.ini tests/mssql.ini man/ \ - tests/e2e/gitea-e2e-*/ \ - tests/e2e/indexers-*/ \ - tests/e2e/reports/ tests/e2e/test-artifacts/ tests/e2e/test-snapshots/ + rm -f $(EXECUTABLE) test-*.test tests/*.ini + rm -rf $(DIST) $(BINDATA_DEST_WILDCARD) man tests/integration/gitea-integration-* .PHONY: fmt fmt: ## format the Go and template code @@ -389,13 +382,10 @@ watch-frontend: node_modules ## start vite dev server for frontend watch-backend: ## watch backend files and continuously rebuild GITEA_RUN_MODE=dev $(GO) run $(AIR_PACKAGE) -c .air.toml -.PHONY: test -test: test-frontend test-backend ## test everything - .PHONY: test-backend test-backend: ## test backend files - @echo "Running go test with $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..." - @$(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' $(GO_TEST_PACKAGES) + @echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)'..." + @$(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $(GO_TEST_PACKAGES) .PHONY: test-frontend test-frontend: node_modules ## test frontend files @@ -413,10 +403,10 @@ test-check: exit 1; \ fi -.PHONY: test\#% -test\#%: - @echo "Running go test with -tags '$(TEST_TAGS)'..." - @$(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_TEST_PACKAGES) +.PHONY: test-backend\#% +test-backend\#%: + @echo "Running go test with -tags '$(TAGS)'..." + @$(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' -run $(subst .,/,$*) $(GO_TEST_PACKAGES) .PHONY: coverage coverage: @@ -426,8 +416,8 @@ coverage: .PHONY: unit-test-coverage unit-test-coverage: - @echo "Running unit-test-coverage $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..." - @$(GO) test $(GOTESTFLAGS) -timeout=20m -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_TEST_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 + @echo "Running unit-test-coverage $(GOTEST_FLAGS) -tags '$(TAGS)'..." + @$(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' -cover -coverprofile coverage.out $(GO_TEST_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 .PHONY: tidy tidy: ## run go mod tidy @@ -454,193 +444,42 @@ go-licenses: $(GO_LICENSE_FILE) ## regenerate go licenses $(GO_LICENSE_FILE): go.mod go.sum GO=$(GO) $(GO) run build/generate-go-licenses.go $(GO_LICENSE_FILE) -generate-ini-sqlite: - sed -e 's|{{WORK_PATH}}|$(CURDIR)/tests/$(or $(TEST_TYPE),integration)/gitea-$(or $(TEST_TYPE),integration)-sqlite|g' \ - -e 's|{{TEST_LOGGER}}|$(or $(TEST_LOGGER),test$(COMMA)file)|g' \ - tests/sqlite.ini.tmpl > tests/sqlite.ini - -.PHONY: test-sqlite -test-sqlite: integrations.sqlite.test generate-ini-sqlite - GITEA_TEST_CONF=tests/sqlite.ini ./integrations.sqlite.test - -.PHONY: test-sqlite\#% -test-sqlite\#%: integrations.sqlite.test generate-ini-sqlite - GITEA_TEST_CONF=tests/sqlite.ini ./integrations.sqlite.test -test.run $(subst .,/,$*) - -.PHONY: test-sqlite-migration -test-sqlite-migration: migrations.sqlite.test migrations.individual.sqlite.test - -generate-ini-mysql: - sed -e 's|{{TEST_MYSQL_HOST}}|${TEST_MYSQL_HOST}|g' \ - -e 's|{{TEST_MYSQL_DBNAME}}|${TEST_MYSQL_DBNAME}|g' \ - -e 's|{{TEST_MYSQL_USERNAME}}|${TEST_MYSQL_USERNAME}|g' \ - -e 's|{{TEST_MYSQL_PASSWORD}}|${TEST_MYSQL_PASSWORD}|g' \ - -e 's|{{WORK_PATH}}|$(CURDIR)/tests/$(or $(TEST_TYPE),integration)/gitea-$(or $(TEST_TYPE),integration)-mysql|g' \ - -e 's|{{TEST_LOGGER}}|$(or $(TEST_LOGGER),test$(COMMA)file)|g' \ - tests/mysql.ini.tmpl > tests/mysql.ini - -.PHONY: test-mysql -test-mysql: integrations.mysql.test generate-ini-mysql - GITEA_TEST_CONF=tests/mysql.ini ./integrations.mysql.test - -.PHONY: test-mysql\#% -test-mysql\#%: integrations.mysql.test generate-ini-mysql - GITEA_TEST_CONF=tests/mysql.ini ./integrations.mysql.test -test.run $(subst .,/,$*) - -.PHONY: test-mysql-migration -test-mysql-migration: migrations.mysql.test migrations.individual.mysql.test - -generate-ini-pgsql: - sed -e 's|{{TEST_PGSQL_HOST}}|${TEST_PGSQL_HOST}|g' \ - -e 's|{{TEST_PGSQL_DBNAME}}|${TEST_PGSQL_DBNAME}|g' \ - -e 's|{{TEST_PGSQL_USERNAME}}|${TEST_PGSQL_USERNAME}|g' \ - -e 's|{{TEST_PGSQL_PASSWORD}}|${TEST_PGSQL_PASSWORD}|g' \ - -e 's|{{TEST_PGSQL_SCHEMA}}|${TEST_PGSQL_SCHEMA}|g' \ - -e 's|{{TEST_MINIO_ENDPOINT}}|${TEST_MINIO_ENDPOINT}|g' \ - -e 's|{{WORK_PATH}}|$(CURDIR)/tests/$(or $(TEST_TYPE),integration)/gitea-$(or $(TEST_TYPE),integration)-pgsql|g' \ - -e 's|{{TEST_LOGGER}}|$(or $(TEST_LOGGER),test$(COMMA)file)|g' \ - tests/pgsql.ini.tmpl > tests/pgsql.ini - -.PHONY: test-pgsql -test-pgsql: integrations.pgsql.test generate-ini-pgsql - GITEA_TEST_CONF=tests/pgsql.ini ./integrations.pgsql.test - -.PHONY: test-pgsql\#% -test-pgsql\#%: integrations.pgsql.test generate-ini-pgsql - GITEA_TEST_CONF=tests/pgsql.ini ./integrations.pgsql.test -test.run $(subst .,/,$*) - -.PHONY: test-pgsql-migration -test-pgsql-migration: migrations.pgsql.test migrations.individual.pgsql.test - -generate-ini-mssql: - sed -e 's|{{TEST_MSSQL_HOST}}|${TEST_MSSQL_HOST}|g' \ - -e 's|{{TEST_MSSQL_DBNAME}}|${TEST_MSSQL_DBNAME}|g' \ - -e 's|{{TEST_MSSQL_USERNAME}}|${TEST_MSSQL_USERNAME}|g' \ - -e 's|{{TEST_MSSQL_PASSWORD}}|${TEST_MSSQL_PASSWORD}|g' \ - -e 's|{{WORK_PATH}}|$(CURDIR)/tests/$(or $(TEST_TYPE),integration)/gitea-$(or $(TEST_TYPE),integration)-mssql|g' \ - -e 's|{{TEST_LOGGER}}|$(or $(TEST_LOGGER),test$(COMMA)file)|g' \ - tests/mssql.ini.tmpl > tests/mssql.ini - -.PHONY: test-mssql -test-mssql: integrations.mssql.test generate-ini-mssql - GITEA_TEST_CONF=tests/mssql.ini ./integrations.mssql.test - -.PHONY: test-mssql\#% -test-mssql\#%: integrations.mssql.test generate-ini-mssql - GITEA_TEST_CONF=tests/mssql.ini ./integrations.mssql.test -test.run $(subst .,/,$*) - -.PHONY: test-mssql-migration -test-mssql-migration: migrations.mssql.test migrations.individual.mssql.test - -.PHONY: playwright -playwright: deps-frontend - @# on GitHub Actions VMs, playwright's system deps are pre-installed - @pnpm exec playwright install $(if $(GITHUB_ACTIONS),,--with-deps) chromium firefox $(PLAYWRIGHT_FLAGS) - -.PHONY: test-e2e -test-e2e: playwright $(EXECUTABLE_E2E) - @EXECUTABLE=$(EXECUTABLE_E2E) ./tools/test-e2e.sh $(GITEA_TEST_E2E_FLAGS) - -.PHONY: bench-sqlite -bench-sqlite: integrations.sqlite.test generate-ini-sqlite - GITEA_TEST_CONF=tests/sqlite.ini ./integrations.sqlite.test -test.cpuprofile=cpu.out -test.run DontRunTests -test.bench . - -.PHONY: bench-mysql -bench-mysql: integrations.mysql.test generate-ini-mysql - GITEA_TEST_CONF=tests/mysql.ini ./integrations.mysql.test -test.cpuprofile=cpu.out -test.run DontRunTests -test.bench . - -.PHONY: bench-mssql -bench-mssql: integrations.mssql.test generate-ini-mssql - GITEA_TEST_CONF=tests/mssql.ini ./integrations.mssql.test -test.cpuprofile=cpu.out -test.run DontRunTests -test.bench . - -.PHONY: bench-pgsql -bench-pgsql: integrations.pgsql.test generate-ini-pgsql - GITEA_TEST_CONF=tests/pgsql.ini ./integrations.pgsql.test -test.cpuprofile=cpu.out -test.run DontRunTests -test.bench . - -.PHONY: integration-test-coverage -integration-test-coverage: integrations.cover.test generate-ini-mysql - GITEA_TEST_CONF=tests/mysql.ini ./integrations.cover.test -test.coverprofile=integration.coverage.out - -.PHONY: integration-test-coverage-sqlite -integration-test-coverage-sqlite: integrations.cover.sqlite.test generate-ini-sqlite - GITEA_TEST_CONF=tests/sqlite.ini ./integrations.cover.sqlite.test -test.coverprofile=integration.coverage.out - -integrations.mysql.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration -o integrations.mysql.test - -integrations.pgsql.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration -o integrations.pgsql.test - -integrations.mssql.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration -o integrations.mssql.test - -integrations.sqlite.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration -o integrations.sqlite.test -tags '$(TEST_TAGS)' - -integrations.cover.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration -coverpkg $(shell echo $(GO_TEST_PACKAGES) | tr ' ' ',') -o integrations.cover.test +.PHONY: test-integration +test-integration: + @# Use a compiled binary: testlogger forwards gitea logs to t.Log, so `go test -v` + @# would flood output per passing test. testcache can't help these tests anyway — + @# they mutate the work directory, so cache inputs change between runs. + $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -c code.gitea.io/gitea/tests/integration -o ./test-integration-$(GITEA_TEST_DATABASE).test + ./test-integration-$(GITEA_TEST_DATABASE).test -integrations.cover.sqlite.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration -coverpkg $(shell echo $(GO_TEST_PACKAGES) | tr ' ' ',') -o integrations.cover.sqlite.test -tags '$(TEST_TAGS)' +.PHONY: test-integration\#% +test-integration\#%: + $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -run $(subst .,/,$*) code.gitea.io/gitea/tests/integration -.PHONY: migrations.mysql.test -migrations.mysql.test: $(GO_SOURCES) generate-ini-mysql - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration/migration-test -o migrations.mysql.test - GITEA_TEST_CONF=tests/mysql.ini ./migrations.mysql.test +.PHONY: test-migration +test-migration: migrations.integration.test migrations.individual.test -.PHONY: migrations.pgsql.test -migrations.pgsql.test: $(GO_SOURCES) generate-ini-pgsql - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration/migration-test -o migrations.pgsql.test - GITEA_TEST_CONF=tests/pgsql.ini ./migrations.pgsql.test +.PHONY: migrations.integration.test +migrations.integration.test: + $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' code.gitea.io/gitea/tests/integration/migration-test -.PHONY: migrations.mssql.test -migrations.mssql.test: $(GO_SOURCES) generate-ini-mssql - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration/migration-test -o migrations.mssql.test - GITEA_TEST_CONF=tests/mssql.ini ./migrations.mssql.test +.PHONY: migrations.individual.test +migrations.individual.test: + @# tests of multiple packages use the same database, don't run in parallel + $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES) -.PHONY: migrations.sqlite.test -migrations.sqlite.test: $(GO_SOURCES) generate-ini-sqlite - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/tests/integration/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)' - GITEA_TEST_CONF=tests/sqlite.ini ./migrations.sqlite.test +.PHONY: migrations.individual.test\#% +migrations.individual.test\#%: + $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' code.gitea.io/gitea/models/migrations/$* -.PHONY: migrations.individual.mysql.test -migrations.individual.mysql.test: $(GO_SOURCES) generate-ini-mysql - GITEA_TEST_CONF=tests/mysql.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES) - -.PHONY: migrations.individual.sqlite.test\#% -migrations.individual.sqlite.test\#%: $(GO_SOURCES) generate-ini-sqlite - GITEA_TEST_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' code.gitea.io/gitea/models/migrations/$* - -.PHONY: migrations.individual.pgsql.test -migrations.individual.pgsql.test: $(GO_SOURCES) generate-ini-pgsql - GITEA_TEST_CONF=tests/pgsql.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES) - -.PHONY: migrations.individual.pgsql.test\#% -migrations.individual.pgsql.test\#%: $(GO_SOURCES) generate-ini-pgsql - GITEA_TEST_CONF=tests/pgsql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' code.gitea.io/gitea/models/migrations/$* - -.PHONY: migrations.individual.mssql.test -migrations.individual.mssql.test: $(GO_SOURCES) generate-ini-mssql - GITEA_TEST_CONF=tests/mssql.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES) - -.PHONY: migrations.individual.mssql.test\#% -migrations.individual.mssql.test\#%: $(GO_SOURCES) generate-ini-mssql - GITEA_TEST_CONF=tests/mssql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' code.gitea.io/gitea/models/migrations/$* - -.PHONY: migrations.individual.sqlite.test -migrations.individual.sqlite.test: $(GO_SOURCES) generate-ini-sqlite - GITEA_TEST_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES) - -.PHONY: migrations.individual.sqlite.test\#% -migrations.individual.sqlite.test\#%: $(GO_SOURCES) generate-ini-sqlite - GITEA_TEST_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' code.gitea.io/gitea/models/migrations/$* - -.PHONY: check -check: test +.PHONY: playwright +playwright: deps-frontend + @# on GitHub Actions VMs, playwright's system deps are pre-installed + @pnpm exec playwright install $(if $(GITHUB_ACTIONS),,--with-deps) chromium firefox $(PLAYWRIGHT_FLAGS) -.PHONY: install $(TAGS_PREREQ) -install: $(wildcard *.go) - CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) install -v -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' +.PHONY: test-e2e +test-e2e: playwright backend + @EXECUTABLE=$(EXECUTABLE) ./tools/test-e2e.sh $(GITEA_TEST_E2E_FLAGS) .PHONY: build build: frontend backend ## build everything @@ -673,9 +512,6 @@ ifneq ($(and $(STATIC),$(findstring pam,$(TAGS))),) endif CGO_ENABLED="$(CGO_ENABLED)" CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(EXTLDFLAGS) $(LDFLAGS)' -o $@ -$(EXECUTABLE_E2E): $(GO_SOURCES) $(FRONTEND_DEST) - CGO_ENABLED=1 $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TEST_TAGS)' -ldflags '-s -w $(EXTLDFLAGS) $(LDFLAGS)' -o $@ - .PHONY: release release: frontend generate release-windows release-linux release-darwin release-freebsd release-copy release-compress vendor release-sources release-check @@ -833,11 +669,6 @@ generate-manpage: ## generate manpage @gzip -9 man/man1/gitea.1 && echo man/man1/gitea.1.gz created @#TODO A small script that formats config-cheat-sheet.en-us.md nicely for use as a config man page -.PHONY: docker -docker: - docker build --disable-content-trust=false -t $(DOCKER_REF) . -# support also build args docker build --build-arg GITEA_VERSION=v1.2.3 --build-arg TAGS="bindata sqlite sqlite_unlock_notify" . - # Disable parallel execution because it would break some targets that don't # specify exact dependencies like 'backend' which does currently not depend # on 'frontend' to enable Node.js-less builds from source tarballs. diff --git a/models/migrations/base/db_test.go b/models/migrations/base/db_test.go index 00635ca72e213..dc4e502e4218a 100644 --- a/models/migrations/base/db_test.go +++ b/models/migrations/base/db_test.go @@ -17,11 +17,11 @@ func TestMain(m *testing.M) { func Test_DropTableColumns(t *testing.T) { x, deferable := PrepareTestEnv(t, 0) + defer deferable() + // FIXME: this logic seems wrong. Need to add an assertion here in the future, but it seems causing failure. if x == nil || t.Failed() { - defer deferable() - return + t.Skip("PrepareTestEnv did not yield a usable engine") } - defer deferable() type DropTest struct { ID int64 `xorm:"pk autoincr"` diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 7482829f1f2fc..0ec979a513157 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -15,7 +15,6 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/tempdir" "code.gitea.io/gitea/modules/testlogger" "code.gitea.io/gitea/modules/util" @@ -120,6 +119,8 @@ func deleteDB() error { if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE [%s]", setting.Database.Name)); err != nil { return err } + default: + return fmt.Errorf("unsupported database type: %s", setting.Database.Type) } return nil @@ -203,14 +204,10 @@ func LoadTableSchemasMap(t *testing.T, x *xorm.Engine) map[string]*schemas.Table func mainTest(m *testing.M) int { testlogger.Init() - - tempWorkPath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("migration-test-data-") + err := setting.PrepareIntegrationTestConfig() if err != nil { - return testlogger.MainErrorf("Unable to create temporary dir for migration test: %v", err) + return testlogger.MainErrorf("Unable to prepare integration test config: %v", err) } - defer cleanup() - - setting.MockBuiltinPaths(tempWorkPath, "", "") setting.SetupGiteaTestEnv() if err = git.InitFull(); err != nil { diff --git a/modules/setting/testenv.go b/modules/setting/testenv.go index d8663d07e2452..24d489d77c2ef 100644 --- a/modules/setting/testenv.go +++ b/modules/setting/testenv.go @@ -4,6 +4,7 @@ package setting import ( + "errors" "fmt" "os" "path/filepath" @@ -13,6 +14,8 @@ import ( "code.gitea.io/gitea/modules/auth/password/hash" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" + + "github.com/kballard/go-shellquote" ) var giteaTestSourceRoot *string // intentionally use a pointer to make sure the uninitialized access panics @@ -21,6 +24,16 @@ func GetGiteaTestSourceRoot() string { return *giteaTestSourceRoot } +func detectGiteaTestRoot() string { + _, filename, _, _ := runtime.Caller(0) + giteaRoot := filepath.Dir(filepath.Dir(filepath.Dir(filename))) + fixturesDir := filepath.Join(giteaRoot, "models", "fixtures") + if _, err := os.Stat(fixturesDir); err != nil { + panic("in gitea source code directory, fixtures directory not found: " + fixturesDir) + } + return giteaRoot +} + func SetupGiteaTestEnv() { if giteaTestSourceRoot != nil { return // already initialized @@ -41,12 +54,7 @@ func SetupGiteaTestEnv() { initGiteaRoot := func() string { giteaRoot := os.Getenv("GITEA_TEST_ROOT") if giteaRoot == "" { - _, filename, _, _ := runtime.Caller(0) - giteaRoot = filepath.Dir(filepath.Dir(filepath.Dir(filename))) - fixturesDir := filepath.Join(giteaRoot, "models", "fixtures") - if _, err := os.Stat(fixturesDir); err != nil { - panic("in gitea source code directory, fixtures directory not found: " + fixturesDir) - } + giteaRoot = detectGiteaTestRoot() } giteaTestSourceRoot = &giteaRoot return giteaRoot @@ -117,3 +125,51 @@ func SetupGiteaTestEnv() { _ = os.Setenv("GITEA_ROOT", giteaRoot) _ = os.Setenv("GITEA_CONF", giteaConf) // test fixture git hooks use "$GITEA_ROOT/$GITEA_CONF" in their scripts } + +func PrepareIntegrationTestConfig() error { + giteaTestRoot := detectGiteaTestRoot() + isInCI := os.Getenv("CI") != "" + testDatabase := os.Getenv("GITEA_TEST_DATABASE") + if testDatabase == "" { + if isInCI { + return errors.New("GITEA_TEST_DATABASE environment variable not set") + } + // for local development, default to sqlite. CI needs to explicitly set a database to avoid unexpected results + testDatabase = "sqlite" + _, _ = fmt.Fprintf(os.Stderr, "Environment variable GITEA_TEST_DATABASE not set - defaulting to %s\n", testDatabase) + } + + _ = os.Setenv("GITEA_TEST_ROOT", giteaTestRoot) + _ = os.Setenv("GITEA_TEST_CONF", filepath.Join("tests", testDatabase+".ini")) + + workPath := filepath.Join(giteaTestRoot, "tests/integration/gitea-integration-"+testDatabase) + if err := os.MkdirAll(workPath, 0o755); err != nil { + return err + } + + confFile := filepath.Join(giteaTestRoot, "tests", testDatabase+".ini") + tmplBuf, err := os.ReadFile(confFile + ".tmpl") + if err != nil { + return err + } + tmpl := string(tmplBuf) + envVars, err := shellquote.Split(os.Getenv("MAKEFILE_VARS")) + if err != nil { + return err + } + envVarMap := map[string]string{ + "TEST_WORK_PATH": workPath, + "TEST_LOGGER": "test,file", + } + for _, env := range append(os.Environ(), envVars...) { + k, v, _ := strings.Cut(env, "=") + k = strings.TrimSpace(k) + v = strings.TrimSpace(v) + envVarMap[k] = v + } + for k, v := range envVarMap { + tmpl = strings.ReplaceAll(tmpl, fmt.Sprintf("{{%s}}", k), v) + } + err = os.WriteFile(confFile, []byte(tmpl), 0o644) + return err +} diff --git a/tests/integration/README.md b/tests/integration/README.md index 3129be5f5ffea..a5057e6ef7e3c 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,106 +1,94 @@ # Integration tests -Integration tests can be run with make commands for the -appropriate backends, namely: -```shell -make test-sqlite -make test-pgsql -make test-mysql -make test-mssql -``` +Integration tests can be run with command `make test-integration`. +Environment variable `GITEA_TEST_DATABASE` can be used to specify the database type for testing. -Make sure to perform a clean build before running tests: -``` -make clean build -``` +If you encounter some errors like mismatched database version, SSH push errors, etc., +you can try to perform a clean build by: `make clean build`. -## Run tests via local act_runner +## Run sqlite integration tests -### Run all jobs +Start tests directly (empty `GITEA_TEST_DATABASE` defaults to sqlite): ``` -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -``` - -Warning: This file defines many jobs, so it will be resource-intensive and therefor not recommended. - -### Run single job - -```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -j +make test-integration ``` -You can list all job names via: - -```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -l -``` +## Run MySQL integration tests -## Run sqlite integration tests -Start tests -``` -make test-sqlite -``` +Set up a MySQL database inside docker: -## Run MySQL integration tests -Setup a MySQL database inside docker ``` docker run -e "MYSQL_DATABASE=test" -e "MYSQL_ALLOW_EMPTY_PASSWORD=yes" -p 3306:3306 --rm --name mysql mysql:latest #(just ctrl-c to stop db and clean the container) docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" --rm --name elasticsearch elasticsearch:7.6.0 #(in a second terminal, just ctrl-c to stop db and clean the container) ``` -Start tests based on the database container + +Start tests based on the database container: + ``` -TEST_MYSQL_HOST=localhost:3306 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=root TEST_MYSQL_PASSWORD='' make test-mysql +GITEA_TEST_DATABASE=mysql TEST_MYSQL_HOST=localhost:3306 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=root TEST_MYSQL_PASSWORD='' make test-integration ``` ## Run pgsql integration tests -Setup a pgsql database inside docker + +Set up a pgsql database inside docker: + ``` docker run -e "POSTGRES_DB=test" -e "POSTGRES_USER=postgres" -e "POSTGRES_PASSWORD=postgres" -p 5432:5432 --rm --name pgsql postgres:latest #(just ctrl-c to stop db and clean the container) ``` -Setup minio inside docker + +Set up minio inside docker: + ``` docker run --rm -p 9000:9000 -e MINIO_ROOT_USER=123456 -e MINIO_ROOT_PASSWORD=12345678 --name minio bitnamilegacy/minio:2023.8.31 ``` -Start tests based on the database container + +Start tests based on the database container: + ``` -TEST_MINIO_ENDPOINT=localhost:9000 TEST_PGSQL_HOST=localhost:5432 TEST_PGSQL_DBNAME=postgres TEST_PGSQL_USERNAME=postgres TEST_PGSQL_PASSWORD=postgres make test-pgsql +GITEA_TEST_DATABASE=pgsql TEST_MINIO_ENDPOINT=localhost:9000 TEST_PGSQL_HOST=localhost:5432 TEST_PGSQL_DBNAME=postgres TEST_PGSQL_USERNAME=postgres TEST_PGSQL_PASSWORD=postgres make test-integration ``` ## Run mssql integration tests -Setup a mssql database inside docker + +Set up a mssql database inside docker: + ``` docker run -e "ACCEPT_EULA=Y" -e "MSSQL_PID=Standard" -e "SA_PASSWORD=MwantsaSecurePassword1" -p 1433:1433 --rm --name mssql microsoft/mssql-server-linux:latest #(just ctrl-c to stop db and clean the container) ``` -Start tests based on the database container + +Start tests based on the database container: + ``` -TEST_MSSQL_HOST=localhost:1433 TEST_MSSQL_DBNAME=gitea_test TEST_MSSQL_USERNAME=sa TEST_MSSQL_PASSWORD=MwantsaSecurePassword1 make test-mssql +GITEA_TEST_DATABASE=mssql TEST_MSSQL_HOST=localhost:1433 TEST_MSSQL_DBNAME=gitea_test TEST_MSSQL_USERNAME=sa TEST_MSSQL_PASSWORD=MwantsaSecurePassword1 make test-integration ``` ## Running individual tests Example command to run GPG test: -For SQLite: - ``` -make test-sqlite#GPG +GITEA_TEST_DATABASE=... make test-integration#GPG ``` -For other databases(replace `mssql` to `mysql`, or `pgsql`): +## Run Gitea Actions tests via local act_runner + +### Run all jobs ``` -TEST_MSSQL_HOST=localhost:1433 TEST_MSSQL_DBNAME=test TEST_MSSQL_USERNAME=sa TEST_MSSQL_PASSWORD=MwantsaSecurePassword1 make test-mssql#GPG +act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest ``` -## Setting timeouts for declaring long-tests and long-flushes +Warning: This file defines many jobs, so it will be resource-intensive and therefore not recommended. -We appreciate that some testing machines may not be very powerful and -the default timeouts for declaring a slow test or a slow clean-up flush -may not be appropriate. +### Run single job -You can set the following environment variables: +```SHELL +act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -j +``` + +You can list all job names via: -```bash -GITEA_TEST_SLOW_RUN="10s" GITEA_TEST_SLOW_FLUSH="1s" make test-sqlite +```SHELL +act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -l ``` diff --git a/tests/integration/README_ZH.md b/tests/integration/README_ZH.md deleted file mode 100644 index a458ac1e858c9..0000000000000 --- a/tests/integration/README_ZH.md +++ /dev/null @@ -1,91 +0,0 @@ -# 关于集成测试 - -使用如下 make 命令可以运行指定的集成测试: -```shell -make test-mysql -make test-pgsql -make test-sqlite -``` - -在执行集成测试命令前请确保清理了之前的构建环境,清理命令如下: -``` -make clean build -``` - -## 如何在本地 act_runner 上运行测试 - -### 运行所有任务 - -``` -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -``` - -警告:由于在此文件中定义了许多任务,因此此操作将花费太多的CPU和内存来运行。所以不建议这样做。 - -### 运行单个任务 - -```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -j -``` - -您可以通过以下方式列出所有任务名称: -```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -l -``` - -## 如何使用 sqlite 数据库进行集成测试 -使用该命令执行集成测试 -``` -make test-sqlite -``` - -## 如何使用 mysql 数据库进行集成测试 -首先在docker容器里部署一个 mysql 数据库 -``` -docker run -e "MYSQL_DATABASE=test" -e "MYSQL_ALLOW_EMPTY_PASSWORD=yes" -p 3306:3306 --rm --name mysql mysql:8 #(just ctrl-c to stop db and clean the container) -``` -之后便可以基于这个数据库进行集成测试 -``` -TEST_MYSQL_HOST=localhost:3306 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=root TEST_MYSQL_PASSWORD='' make test-mysql -``` - -## 如何使用 pgsql 数据库进行集成测试 -同上,首先在 docker 容器里部署一个 pgsql 数据库 -``` -docker run -e "POSTGRES_DB=test" -e "POSTGRES_USER=postgres" -e "POSTGRES_PASSWORD=postgres" -p 5432:5432 --rm --name pgsql postgres:latest #(just ctrl-c to stop db and clean the container) -``` -在docker内设置minio -``` -docker run --rm -p 9000:9000 -e MINIO_ROOT_USER=123456 -e MINIO_ROOT_PASSWORD=12345678 --name minio bitnamilegacy/minio:2023.8.31 -``` -之后便可以基于这个数据库进行集成测试 -``` -TEST_MINIO_ENDPOINT=localhost:9000 TEST_PGSQL_HOST=localhost:5432 TEST_PGSQL_DBNAME=postgres TEST_PGSQL_USERNAME=postgres TEST_PGSQL_PASSWORD=postgres make test-pgsql -``` - -## Run mssql integration tests -同上,首先在 docker 容器里部署一个 mssql 数据库 -``` -docker run -e "ACCEPT_EULA=Y" -e "MSSQL_PID=Standard" -e "SA_PASSWORD=MwantsaSecurePassword1" -p 1433:1433 --rm --name mssql microsoft/mssql-server-linux:latest #(just ctrl-c to stop db and clean the container) -``` -之后便可以基于这个数据库进行集成测试 -``` -TEST_MSSQL_HOST=localhost:1433 TEST_MSSQL_DBNAME=gitea_test TEST_MSSQL_USERNAME=sa TEST_MSSQL_PASSWORD=MwantsaSecurePassword1 make test-mssql -``` - -## 如何进行自定义的集成测试 - -下面的示例展示了怎样在集成测试中只进行 GPG 测试: - -sqlite 数据库: - -``` -make test-sqlite#GPG -``` - -其它数据库(把 MSSQL 替换为 MYSQL, PGSQL): - -``` -TEST_MSSQL_HOST=localhost:1433 TEST_MSSQL_DBNAME=test TEST_MSSQL_USERNAME=sa TEST_MSSQL_PASSWORD=MwantsaSecurePassword1 make test-mssql#GPG -``` - diff --git a/tests/integration/api_org_avatar_test.go b/tests/integration/api_org_avatar_test.go index cc1452c1535da..62c54e670617a 100644 --- a/tests/integration/api_org_avatar_test.go +++ b/tests/integration/api_org_avatar_test.go @@ -7,9 +7,11 @@ import ( "encoding/base64" "net/http" "os" + "path/filepath" "testing" auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" @@ -24,7 +26,7 @@ func TestAPIUpdateOrgAvatar(t *testing.T) { token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteOrganization) // Test what happens if you use a valid image - avatar, err := os.ReadFile("tests/integration/avatar.png") + avatar, err := os.ReadFile(filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/avatar.png")) assert.NoError(t, err) if err != nil { assert.FailNow(t, "Unable to open avatar.png") @@ -48,7 +50,7 @@ func TestAPIUpdateOrgAvatar(t *testing.T) { MakeRequest(t, req, http.StatusBadRequest) // Test what happens if you use a file that is not an image - text, err := os.ReadFile("tests/integration/README.md") + text, err := os.ReadFile(filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/README.md")) assert.NoError(t, err) if err != nil { assert.FailNow(t, "Unable to open README.md") diff --git a/tests/integration/api_repo_avatar_test.go b/tests/integration/api_repo_avatar_test.go index 6677885f7eb20..e4d0e06d003c7 100644 --- a/tests/integration/api_repo_avatar_test.go +++ b/tests/integration/api_repo_avatar_test.go @@ -8,12 +8,14 @@ import ( "fmt" "net/http" "os" + "path/filepath" "testing" auth_model "code.gitea.io/gitea/models/auth" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" @@ -28,7 +30,7 @@ func TestAPIUpdateRepoAvatar(t *testing.T) { token := getUserToken(t, user2.LowerName, auth_model.AccessTokenScopeWriteRepository) // Test what happens if you use a valid image - avatar, err := os.ReadFile("tests/integration/avatar.png") + avatar, err := os.ReadFile(filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/avatar.png")) assert.NoError(t, err) if err != nil { assert.FailNow(t, "Unable to open avatar.png") @@ -52,7 +54,7 @@ func TestAPIUpdateRepoAvatar(t *testing.T) { MakeRequest(t, req, http.StatusBadRequest) // Test what happens if you use a file that is not an image - text, err := os.ReadFile("tests/integration/README.md") + text, err := os.ReadFile(filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/README.md")) assert.NoError(t, err) if err != nil { assert.FailNow(t, "Unable to open README.md") diff --git a/tests/integration/api_repo_file_create_test.go b/tests/integration/api_repo_file_create_test.go index cbd1e4963e8f7..26cc0aad0e688 100644 --- a/tests/integration/api_repo_file_create_test.go +++ b/tests/integration/api_repo_file_create_test.go @@ -124,35 +124,6 @@ func getExpectedFileResponseForCreate(info apiFileResponseInfo) *api.FileRespons return ret } -func BenchmarkAPICreateFileSmall(b *testing.B) { - onGiteaRun(b, func(b *testing.B, u *url.URL) { - user2 := unittest.AssertExistsAndLoadBean(b, &user_model.User{ID: 2}) // owner of the repo1 & repo16 - repo1 := unittest.AssertExistsAndLoadBean(b, &repo_model.Repository{ID: 1}) // public repo - - b.ResetTimer() - for n := 0; b.Loop(); n++ { - treePath := fmt.Sprintf("update/file%d.txt", n) - _, _ = createFile(user2, repo1, treePath) - } - }) -} - -func BenchmarkAPICreateFileMedium(b *testing.B) { - data := make([]byte, 10*1024*1024) - - onGiteaRun(b, func(b *testing.B, u *url.URL) { - user2 := unittest.AssertExistsAndLoadBean(b, &user_model.User{ID: 2}) // owner of the repo1 & repo16 - repo1 := unittest.AssertExistsAndLoadBean(b, &repo_model.Repository{ID: 1}) // public repo - - b.ResetTimer() - for n := 0; b.Loop(); n++ { - treePath := fmt.Sprintf("update/file%d.txt", n) - copy(data, treePath) - _, _ = createFile(user2, repo1, treePath) - } - }) -} - func TestAPICreateFile(t *testing.T) { onGiteaRun(t, func(t *testing.T, u *url.URL) { user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) // owner of the repo1 & repo16 diff --git a/tests/integration/api_user_avatar_test.go b/tests/integration/api_user_avatar_test.go index 5b4546f1504a2..8c002d7a33861 100644 --- a/tests/integration/api_user_avatar_test.go +++ b/tests/integration/api_user_avatar_test.go @@ -7,9 +7,11 @@ import ( "encoding/base64" "net/http" "os" + "path/filepath" "testing" auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" @@ -24,7 +26,7 @@ func TestAPIUpdateUserAvatar(t *testing.T) { token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteUser) // Test what happens if you use a valid image - avatar, err := os.ReadFile("tests/integration/avatar.png") + avatar, err := os.ReadFile(filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/avatar.png")) assert.NoError(t, err) if err != nil { assert.FailNow(t, "Unable to open avatar.png") @@ -48,7 +50,7 @@ func TestAPIUpdateUserAvatar(t *testing.T) { MakeRequest(t, req, http.StatusBadRequest) // Test what happens if you use a file that is not an image - text, err := os.ReadFile("tests/integration/README.md") + text, err := os.ReadFile(filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/README.md")) assert.NoError(t, err) if err != nil { assert.FailNow(t, "Unable to open README.md") diff --git a/tests/integration/gpg_ssh_git_test.go b/tests/integration/gpg_ssh_git_test.go index a95ff77dab4f6..a8ec79cd9d4e4 100644 --- a/tests/integration/gpg_ssh_git_test.go +++ b/tests/integration/gpg_ssh_git_test.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "testing" auth_model "code.gitea.io/gitea/models/auth" @@ -325,10 +326,11 @@ func crudActionCreateFile(_ *testing.T, ctx APITestContext, user *user_model.Use } func importTestingKey() (*openpgp.Entity, error) { - if _, _, err := process.GetManager().Exec("gpg --import tests/integration/private-testing.key", "gpg", "--import", "tests/integration/private-testing.key"); err != nil { + keyPath := filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/private-testing.key") + if _, _, err := process.GetManager().Exec("gpg --import "+keyPath, "gpg", "--import", keyPath); err != nil { return nil, err } - keyringFile, err := os.Open("tests/integration/private-testing.key") + keyringFile, err := os.Open(keyPath) if err != nil { return nil, err } diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 60dc7e77a4811..62aea184f3642 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -87,7 +87,7 @@ func testMain(m *testing.M) int { graceful.InitManager(managerCtx) defer cancel() - err := tests.InitTest() + err := tests.InitIntegrationTest() if err != nil { return testlogger.MainErrorf("InitTest error: %v", err) } diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index c1acdb999b1ca..2d6fc947a75ed 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -37,6 +37,7 @@ var currentEngine *xorm.Engine func initMigrationTest(t *testing.T) func() { testlogger.Init() + require.NoError(t, setting.PrepareIntegrationTestConfig()) setting.SetupGiteaTestEnv() assert.NotEmpty(t, setting.RepoRootPath) @@ -49,12 +50,12 @@ func initMigrationTest(t *testing.T) func() { } func availableVersions() ([]string, error) { - migrationsDir, err := os.Open("tests/integration/migration-test") + migrationsDir, err := os.Open(filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/migration-test")) if err != nil { return nil, err } defer migrationsDir.Close() - versionRE, err := regexp.Compile("gitea-v(?P.+)\\." + regexp.QuoteMeta(setting.Database.Type.String()) + "\\.sql.gz") + versionRE, err := regexp.Compile("gitea-v(?P.+)" + regexp.QuoteMeta("."+setting.Database.Type.String()+".sql.gz")) if err != nil { return nil, err } @@ -75,7 +76,7 @@ func availableVersions() ([]string, error) { } func readSQLFromFile(version string) (string, error) { - filename := fmt.Sprintf("tests/integration/migration-test/gitea-v%s.%s.sql.gz", version, setting.Database.Type) + filename := filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/migration-test", fmt.Sprintf("gitea-v%s.%s.sql.gz", version, setting.Database.Type)) if _, err := os.Stat(filename); os.IsNotExist(err) { return "", nil @@ -97,7 +98,7 @@ func readSQLFromFile(version string) (string, error) { if err != nil { return "", err } - return string(bytes.TrimPrefix(buf, []byte{'\xef', '\xbb', '\xbf'})), nil + return string(bytes.TrimPrefix(buf, []byte("\xef\xbb\xbf"))), nil } func restoreOldDB(t *testing.T, version string) { @@ -230,6 +231,8 @@ func restoreOldDB(t *testing.T, version string) { assert.NoError(t, err, "Failure whilst running: %s\nError: %v", statement, err) } db.Close() + default: + assert.Failf(t, "unsupported database type", "setting.Database.Type=%v", setting.Database.Type) } } diff --git a/tests/mssql.ini.tmpl b/tests/mssql.ini.tmpl index 6f031691f01ee..aba67157df2e6 100644 --- a/tests/mssql.ini.tmpl +++ b/tests/mssql.ini.tmpl @@ -1,4 +1,4 @@ -WORK_PATH = {{WORK_PATH}} +WORK_PATH = {{TEST_WORK_PATH}} APP_NAME = Gitea: Git with a cup of tea RUN_MODE = prod diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index 02069b403e6b9..4572cfd724d1c 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -1,4 +1,4 @@ -WORK_PATH = {{WORK_PATH}} +WORK_PATH = {{TEST_WORK_PATH}} APP_NAME = Gitea: Git with a cup of tea RUN_MODE = prod diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index 50a649fc7915b..5e15ca7ba6ba6 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -1,4 +1,4 @@ -WORK_PATH = {{WORK_PATH}} +WORK_PATH = {{TEST_WORK_PATH}} APP_NAME = Gitea: Git with a cup of tea RUN_MODE = prod diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index b43b658bb089e..03393c620b91a 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -1,4 +1,4 @@ -WORK_PATH = {{WORK_PATH}} +WORK_PATH = {{TEST_WORK_PATH}} APP_NAME = Gitea: Git with a cup of tea RUN_MODE = prod @@ -88,7 +88,7 @@ ENABLED = true [markup.html] ENABLED = true FILE_EXTENSIONS = .html -RENDER_COMMAND = go run tools/test-echo.go +RENDER_COMMAND = go run {{GITEA_TEST_ROOT}}/tools/test-echo.go ;RENDER_COMMAND = cat ;IS_INPUT_FILE = true RENDER_CONTENT_MODE = sanitized @@ -96,7 +96,7 @@ RENDER_CONTENT_MODE = sanitized [markup.no-sanitizer] ENABLED = true FILE_EXTENSIONS = .no-sanitizer -RENDER_COMMAND = go run tools/test-echo.go +RENDER_COMMAND = go run {{GITEA_TEST_ROOT}}/tools/test-echo.go ; This test case is reused, at first it is used to test "no-sanitizer" (sandbox doesn't take effect here) ; Then it will be updated and used to test "iframe + sandbox-disabled" RENDER_CONTENT_MODE = no-sanitizer diff --git a/tests/test_utils.go b/tests/test_utils.go index d5a8008cefb9b..f16754b056221 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -6,7 +6,6 @@ package tests import ( "database/sql" "fmt" - "os" "path/filepath" "strings" "testing" @@ -25,17 +24,14 @@ import ( "github.com/stretchr/testify/assert" ) -func InitTest() error { +func InitIntegrationTest() error { testlogger.Init() - if os.Getenv("GITEA_TEST_CONF") == "" { - // By default, use sqlite.ini for testing, then IDE like GoLand can start the test process with debugger. - // It's easier for developers to debug bugs step by step with a debugger. - // Notice: when doing "ssh push", Gitea executes sub processes, debugger won't work for the sub processes. - giteaConf := "tests/sqlite.ini" - _ = os.Setenv("GITEA_TEST_CONF", giteaConf) - _, _ = fmt.Fprintf(os.Stderr, "Environment variable GITEA_TEST_CONF not set - defaulting to %s\n", giteaConf) + err := setting.PrepareIntegrationTestConfig() + if err != nil { + return err } + setting.SetupGiteaTestEnv() setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" @@ -132,6 +128,9 @@ func InitTest() error { return err } } + case setting.Database.Type.IsSQLite3(): + default: + return fmt.Errorf("unsupported database type: %s", setting.Database.Type) } routers.InitWebInstalled(graceful.GetManager().HammerContext())