Add support for sha256 repositories (#23894)

Currently only SHA1 repositories are supported by Gitea. This adds
support for alternate SHA256 with the additional aim of easier support
for additional hash types in the future.

Fixes: #13794
Limited by: go-git/go-git#899
Depend on: #28138

<img width="776" alt="图片" src="https://github.com/go-gitea/gitea/assets/81045/5448c9a7-608e-4341-a149-5dd0069c9447">

---------

Co-authored-by: Lunny Xiao <[email protected]>
Co-authored-by: 6543 <[email protected]>

models/git/commit_status.go

@@ -37,7 +37,7 @@ type CommitStatus struct {
 	SHA         string                 `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"`
 	TargetURL   string                 `xorm:"TEXT"`
 	Description string                 `xorm:"TEXT"`
-	ContextHash string                 `xorm:"char(40) index"`
+	ContextHash string                 `xorm:"VARCHAR(64) index"`
 	Context     string                 `xorm:"TEXT"`
 	Creator     *user_model.User       `xorm:"-"`
 	CreatorID   int64

models/issues/comment.go

@@ -270,7 +270,7 @@ type Comment struct {
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
 
 	// Reference issue in commit message
-	CommitSHA string `xorm:"VARCHAR(40)"`
+	CommitSHA string `xorm:"VARCHAR(64)"`
 
 	Attachments []*repo_model.Attachment `xorm:"-"`
 	Reactions   ReactionList             `xorm:"-"`

models/issues/pull.go

@@ -171,11 +171,11 @@ type PullRequest struct {
 	HeadBranch          string
 	HeadCommitID        string `xorm:"-"`
 	BaseBranch          string
-	MergeBase           string `xorm:"VARCHAR(40)"`
+	MergeBase           string `xorm:"VARCHAR(64)"`
 	AllowMaintainerEdit bool   `xorm:"NOT NULL DEFAULT false"`
 
 	HasMerged      bool               `xorm:"INDEX"`
-	MergedCommitID string             `xorm:"VARCHAR(40)"`
+	MergedCommitID string             `xorm:"VARCHAR(64)"`
 	MergerID       int64              `xorm:"INDEX"`
 	Merger         *user_model.User   `xorm:"-"`
 	MergedUnix     timeutil.TimeStamp `xorm:"updated INDEX"`

models/issues/review.go

@@ -116,7 +116,7 @@ type Review struct {
 	Content          string `xorm:"TEXT"`
 	// Official is a review made by an assigned approver (counts towards approval)
 	Official  bool   `xorm:"NOT NULL DEFAULT false"`
-	CommitID  string `xorm:"VARCHAR(40)"`
+	CommitID  string `xorm:"VARCHAR(64)"`
 	Stale     bool   `xorm:"NOT NULL DEFAULT false"`
 	Dismissed bool   `xorm:"NOT NULL DEFAULT false"`
 

models/migrations/fixtures/Test_RepositoryFormat/repository.yml

@@ -0,0 +1,11 @@
+# type Repository struct {
+#   ID        int64  `xorm:"pk autoincr"`
+# }
+-
+  id: 1
+-
+  id: 2
+-
+  id: 3
+-
+  id: 10

models/migrations/migrations.go

@@ -556,6 +556,8 @@ var migrations = []Migration{
 	NewMigration("Add ignore stale approval column on branch table", v1_22.AddIgnoreStaleApprovalsColumnToProtectedBranchTable),
 	// v285 -> v286
 	NewMigration("Add PreviousDuration to ActionRun", v1_22.AddPreviousDurationToActionRun),
+	// v286 -> v287
+	NewMigration("Add support for SHA256 git repositories", v1_22.AdjustDBForSha256),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_22/v286.go

@@ -0,0 +1,104 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+package v1_22 //nolint
+
+import (
+	"errors"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func expandHashReferencesToSha256(x *xorm.Engine) error {
+	alteredTables := [][2]string{
+		{"commit_status", "context_hash"},
+		{"comment", "commit_sha"},
+		{"pull_request", "merge_base"},
+		{"pull_request", "merged_commit_id"},
+		{"review", "commit_id"},
+		{"review_state", "commit_sha"},
+		{"repo_archiver", "commit_id"},
+		{"release", "sha1"},
+		{"repo_indexer_status", "commit_sha"},
+	}
+
+	db := x.NewSession()
+	defer db.Close()
+
+	if err := db.Begin(); err != nil {
+		return err
+	}
+
+	if !setting.Database.Type.IsSQLite3() {
+		if setting.Database.Type.IsMSSQL() {
+			// drop indexes that need to be re-created afterwards
+			droppedIndexes := []string{
+				"DROP INDEX commit_status.IDX_commit_status_context_hash",
+				"DROP INDEX review_state.UQE_review_state_pull_commit_user",
+				"DROP INDEX repo_archiver.UQE_repo_archiver_s",
+			}
+			for _, s := range droppedIndexes {
+				_, err := db.Exec(s)
+				if err != nil {
+					return errors.New(s + " " + err.Error())
+				}
+			}
+		}
+
+		for _, alts := range alteredTables {
+			var err error
+			if setting.Database.Type.IsMySQL() {
+				_, err = db.Exec(fmt.Sprintf("ALTER TABLE `%s` MODIFY COLUMN `%s` VARCHAR(64)", alts[0], alts[1]))
+			} else if setting.Database.Type.IsMSSQL() {
+				_, err = db.Exec(fmt.Sprintf("ALTER TABLE `%s` ALTER COLUMN `%s` VARCHAR(64)", alts[0], alts[1]))
+			} else {
+				_, err = db.Exec(fmt.Sprintf("ALTER TABLE `%s` ALTER COLUMN `%s` TYPE VARCHAR(64)", alts[0], alts[1]))
+			}
+			if err != nil {
+				return fmt.Errorf("alter column '%s' of table '%s' failed: %w", alts[1], alts[0], err)
+			}
+		}
+
+		if setting.Database.Type.IsMSSQL() {
+			recreateIndexes := []string{
+				"CREATE INDEX IDX_commit_status_context_hash ON commit_status(context_hash)",
+				"CREATE UNIQUE INDEX UQE_review_state_pull_commit_user ON review_state(user_id, pull_id, commit_sha)",
+				"CREATE UNIQUE INDEX UQE_repo_archiver_s ON repo_archiver(repo_id, type, commit_id)",
+			}
+			for _, s := range recreateIndexes {
+				_, err := db.Exec(s)
+				if err != nil {
+					return errors.New(s + " " + err.Error())
+				}
+			}
+		}
+	}
+	log.Debug("Updated database tables to hold SHA256 git hash references")
+
+	return db.Commit()
+}
+
+func addObjectFormatNameToRepository(x *xorm.Engine) error {
+	type Repository struct {
+		ObjectFormatName string `xorm:"VARCHAR(6) NOT NULL DEFAULT 'sha1'"`
+	}
+
+	if err := x.Sync(new(Repository)); err != nil {
+		return err
+	}
+
+	// Here to catch weird edge-cases where column constraints above are
+	// not applied by the DB backend
+	_, err := x.Exec("UPDATE repository set object_format_name = 'sha1' WHERE object_format_name = '' or object_format_name IS NULL")
+	return err
+}
+
+func AdjustDBForSha256(x *xorm.Engine) error {
+	if err := expandHashReferencesToSha256(x); err != nil {
+		return err
+	}
+	return addObjectFormatNameToRepository(x)
+}

models/migrations/v1_22/v286_test.go

@@ -0,0 +1,62 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_22 //nolint
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+
+	"github.com/stretchr/testify/assert"
+	"xorm.io/xorm"
+)
+
+func PrepareOldRepository(t *testing.T) (*xorm.Engine, func()) {
+	type Repository struct { // old struct
+		ID int64 `xorm:"pk autoincr"`
+	}
+
+	// Prepare and load the testing database
+	return base.PrepareTestEnv(t, 0, new(Repository))
+}
+
+func Test_RepositoryFormat(t *testing.T) {
+	x, deferable := PrepareOldRepository(t)
+	defer deferable()
+
+	type Repository struct {
+		ID               int64  `xorm:"pk autoincr"`
+		ObjectFormatName string `xorg:"not null default('sha1')"`
+	}
+
+	repo := new(Repository)
+
+	// check we have some records to migrate
+	count, err := x.Count(new(Repository))
+	assert.NoError(t, err)
+	assert.EqualValues(t, 4, count)
+
+	assert.NoError(t, AdjustDBForSha256(x))
+
+	repo.ID = 20
+	repo.ObjectFormatName = "sha256"
+	_, err = x.Insert(repo)
+	assert.NoError(t, err)
+
+	count, err = x.Count(new(Repository))
+	assert.NoError(t, err)
+	assert.EqualValues(t, 5, count)
+
+	repo = new(Repository)
+	ok, err := x.ID(2).Get(repo)
+	assert.NoError(t, err)
+	assert.EqualValues(t, true, ok)
+	assert.EqualValues(t, "sha1", repo.ObjectFormatName)
+
+	repo = new(Repository)
+	ok, err = x.ID(20).Get(repo)
+	assert.NoError(t, err)
+	assert.EqualValues(t, true, ok)
+	assert.EqualValues(t, "sha256", repo.ObjectFormatName)
+}

models/pull/review_state.go

@@ -39,7 +39,7 @@ type ReviewState struct {
 	ID           int64                  `xorm:"pk autoincr"`
 	UserID       int64                  `xorm:"NOT NULL UNIQUE(pull_commit_user)"`
 	PullID       int64                  `xorm:"NOT NULL INDEX UNIQUE(pull_commit_user) DEFAULT 0"` // Which PR was the review on?
-	CommitSHA    string                 `xorm:"NOT NULL VARCHAR(40) UNIQUE(pull_commit_user)"`     // Which commit was the head commit for the review?
+	CommitSHA    string                 `xorm:"NOT NULL VARCHAR(64) UNIQUE(pull_commit_user)"`     // Which commit was the head commit for the review?
 	UpdatedFiles map[string]ViewedState `xorm:"NOT NULL LONGTEXT JSON"`                            // Stores for each of the changed files of a PR whether they have been viewed, changed since last viewed, or not viewed
 	UpdatedUnix  timeutil.TimeStamp     `xorm:"updated"`                                           // Is an accurate indicator of the order of commits as we do not expect it to be possible to make reviews on previous commits
 }

models/repo/archiver.go

@@ -33,7 +33,7 @@ type RepoArchiver struct { //revive:disable-line:exported
 	RepoID      int64           `xorm:"index unique(s)"`
 	Type        git.ArchiveType `xorm:"unique(s)"`
 	Status      ArchiverStatus
-	CommitID    string             `xorm:"VARCHAR(40) unique(s)"`
+	CommitID    string             `xorm:"VARCHAR(64) unique(s)"`
 	CreatedUnix timeutil.TimeStamp `xorm:"INDEX NOT NULL created"`
 }
 

models/repo/release.go

@@ -75,7 +75,7 @@ type Release struct {
 	Target           string
 	TargetBehind     string `xorm:"-"` // to handle non-existing or empty target
 	Title            string
-	Sha1             string `xorm:"VARCHAR(40)"`
+	Sha1             string `xorm:"VARCHAR(64)"`
 	NumCommits       int64
 	NumCommitsBehind int64              `xorm:"-"`
 	Note             string             `xorm:"TEXT"`

models/repo/repo.go

@@ -180,7 +180,7 @@ type Repository struct {
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
 	CloseIssuesViaCommitInAnyBranch bool               `xorm:"NOT NULL DEFAULT false"`
 	Topics                          []string           `xorm:"TEXT JSON"`
-	ObjectFormatName                string             `xorm:"-"`
+	ObjectFormatName                string             `xorm:"VARCHAR(6) NOT NULL DEFAULT 'sha1'"`
 
 	TrustModel TrustModelType
 
@@ -276,10 +276,6 @@ func (repo *Repository) AfterLoad() {
 	repo.NumOpenMilestones = repo.NumMilestones - repo.NumClosedMilestones
 	repo.NumOpenProjects = repo.NumProjects - repo.NumClosedProjects
 	repo.NumOpenActionRuns = repo.NumActionRuns - repo.NumClosedActionRuns
-
-	// this is a temporary behaviour to support old repos, next step is to store the object format in the database
-	// and read from database so this line could be removed. To not depend on git module, we use a constant variable here
-	repo.ObjectFormatName = "sha1"
 }
 
 // LoadAttributes loads attributes of the repository.

models/repo/repo_indexer.go

@@ -27,7 +27,7 @@ const (
 type RepoIndexerStatus struct { //revive:disable-line:exported
 	ID          int64           `xorm:"pk autoincr"`
 	RepoID      int64           `xorm:"INDEX(s)"`
-	CommitSha   string          `xorm:"VARCHAR(40)"`
+	CommitSha   string          `xorm:"VARCHAR(64)"`
 	IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
 }