Add package registry cleanup rules

docs/content/doc/packages/storage.en-us.md

@@ -0,0 +1,84 @@
+---
+date: "2022-11-01T00:00:00+00:00"
+title: "Storage"
+slug: "packages/storage"
+draft: false
+toc: false
+menu:
+  sidebar:
+    parent: "packages"
+    name: "storage"
+    weight: 5
+    identifier: "storage"
+---
+
+# Storage
+
+This document describes the storage of the package registry and how it can be managed.
+
+**Table of Contents**
+
+{{< toc >}}
+
+## Deduplication
+
+The package registry has a build-in deduplication of uploaded blobs.
+If two identical files are uploaded only one blob is saved on the filesystem.
+This ensures no space is wasted for duplicated files.
+
+If two packages are uploaded with identical files, both packages will display the same size but on the filesystem they require only half of the size.
+Whenever a package gets deleted only the references to the underlaying blobs are removed.
+The blobs get not removed at this moment, so they still require space on the filesystem.
+When a new package gets uploaded the existing blobs may get referenced again.
+
+These unreferenced blobs get deleted by a [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}).
+The config setting `OLDER_THAN` configures how long unreferenced blobs are kept before they get deleted.
+
+## Cleanup Rules
+
+Package registries can become large over time without cleanup.
+It's recommended to delete unnecessary packages and set up cleanup rules to automatically manage the package registry usage.
+Every package owner (user or organization) manages the cleanup rules which are applied to their packages.
+
+|Setting|Description|
+|-|-|
+|Enabled|Turn the cleanup rule on or off.|
+|Type|Every rule manages a specific package type.|
+|Apply pattern to full package name|If enabled, the patterns below are applied to the full package name (`package/version`). Otherwise only the version (`version`) is used.|
+|Keep the most recent|How many versions to *always* keep for each package.|
+|Keep versions matching|The regex pattern that determines which versions to keep. An empty pattern keeps no version while `.+` keeps all versions. The container registry will always keep the `latest` version even if not configured.|
+|Remove versions older than|Remove only versions older than the selected days.|
+|Remove versions matching|The regex pattern that determines which versions to remove. An empty pattern or `.+` leads to the removal of every package if no other setting tells otherwise.|
+
+Every cleanup rule can show a preview of the affected packages.
+This can be used to check if the cleanup rules is proper configured.
+
+### Regex examples
+
+Regex patterns are automatically surrounded with `\A` and `\z` anchors.
+Do not include any `\A`, `\z`, `^` or `$` token in the regex patterns as they are not necessary.
+The patterns are case-insensitive which matches the behaviour of the package registry in Gitea.
+
+|Pattern|Description|
+|-|-|
+|`.*`|Match every possible version.|
+|`v.+`|Match versions that start with `v`.|
+|`release`|Match only the version `release`.|
+|`release.*`|Match versions that are either named or start with `release`.|
+|`.+-temp-.+`|Match versions that contain `-temp-`.|
+|`v.+\|release`|Match versions that either start with `v` or are named `release`.|
+|`package/v.+\|other/release`|Match versions of the package `package` that start with `v` or the version `release` of the package `other`. This needs the setting *Apply pattern to full package name* enabled.|
+
+### How the cleanup rules work
+
+The cleanup rules are part of the [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}) and run periodicly.
+
+The cleanup rule:
+
+1. Collects all packages of the package type for the owners registry.
+1. For every package it collects all versions.
+1. Excludes from the list the # versions based on the *Keep the most recent* value.
+1. Excludes from the list any versions matching the *Keep versions matching* value.
+1. Excludes from the list the versions more recent than the *Remove versions older than* value.
+1. Excludes from the list any versions not matching the *Remove versions matching* value.
+1. Deletes the remaining versions.

models/migrations/migrations.go

@@ -435,6 +435,8 @@ var migrations = []Migration{
 	NewMigration("Add index for hook_task", v1_19.AddIndexForHookTask),
 	// v232 -> v233
 	NewMigration("Alter package_version.metadata_json to LONGTEXT", v1_19.AlterPackageVersionMetadataToLongText),
+	// v233 -> v234
+	NewMigration("Add package cleanup rule table", v1_19.CreatePackageCleanupRuleTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_19/v233.go

@@ -0,0 +1,29 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_19 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func CreatePackageCleanupRuleTable(x *xorm.Engine) error {
+	type PackageCleanupRule struct {
+		ID            int64              `xorm:"pk autoincr"`
+		Enabled       bool               `xorm:"INDEX NOT NULL DEFAULT false"`
+		OwnerID       int64              `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"`
+		Type          string             `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		KeepCount     int                `xorm:"NOT NULL DEFAULT 0"`
+		KeepPattern   string             `xorm:"NOT NULL DEFAULT ''"`
+		RemoveDays    int                `xorm:"NOT NULL DEFAULT 0"`
+		RemovePattern string             `xorm:"NOT NULL DEFAULT ''"`
+		MatchFullName bool               `xorm:"NOT NULL DEFAULT false"`
+		CreatedUnix   timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"`
+		UpdatedUnix   timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"`
+	}
+
+	return x.Sync2(new(PackageCleanupRule))
+}

models/packages/package.go

@@ -45,6 +45,21 @@ const (
 	TypeVagrant   Type = "vagrant"
 )
 
+var TypeList = []Type{
+	TypeComposer,
+	TypeConan,
+	TypeContainer,
+	TypeGeneric,
+	TypeHelm,
+	TypeMaven,
+	TypeNpm,
+	TypeNuGet,
+	TypePub,
+	TypePyPI,
+	TypeRubyGems,
+	TypeVagrant,
+}
+
 // Name gets the name of the package type
 func (pt Type) Name() string {
 	switch pt {

models/packages/package_cleanup_rule.go

@@ -0,0 +1,111 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/builder"
+)
+
+var ErrPackageCleanupRuleNotExist = errors.New("Package blob does not exist")
+
+func init() {
+	db.RegisterModel(new(PackageCleanupRule))
+}
+
+// PackageCleanupRule represents a rule which describes when to clean up package versions
+type PackageCleanupRule struct {
+	ID                   int64              `xorm:"pk autoincr"`
+	Enabled              bool               `xorm:"INDEX NOT NULL DEFAULT false"`
+	OwnerID              int64              `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"`
+	Type                 Type               `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	KeepCount            int                `xorm:"NOT NULL DEFAULT 0"`
+	KeepPattern          string             `xorm:"NOT NULL DEFAULT ''"`
+	KeepPatternMatcher   *regexp.Regexp     `xorm:"-"`
+	RemoveDays           int                `xorm:"NOT NULL DEFAULT 0"`
+	RemovePattern        string             `xorm:"NOT NULL DEFAULT ''"`
+	RemovePatternMatcher *regexp.Regexp     `xorm:"-"`
+	MatchFullName        bool               `xorm:"NOT NULL DEFAULT false"`
+	CreatedUnix          timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"`
+	UpdatedUnix          timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"`
+}
+
+func (pcr *PackageCleanupRule) CompiledPattern() error {
+	if pcr.KeepPatternMatcher != nil || pcr.RemovePatternMatcher != nil {
+		return nil
+	}
+
+	if pcr.KeepPattern != "" {
+		var err error
+		pcr.KeepPatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.KeepPattern))
+		if err != nil {
+			return err
+		}
+	}
+
+	if pcr.RemovePattern != "" {
+		var err error
+		pcr.RemovePatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.RemovePattern))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func InsertCleanupRule(ctx context.Context, pcr *PackageCleanupRule) (*PackageCleanupRule, error) {
+	_, err := db.GetEngine(ctx).Insert(pcr)
+	return pcr, err
+}
+
+func GetCleanupRuleByID(ctx context.Context, id int64) (*PackageCleanupRule, error) {
+	pcr := &PackageCleanupRule{}
+
+	has, err := db.GetEngine(ctx).ID(id).Get(pcr)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, ErrPackageCleanupRuleNotExist
+	}
+	return pcr, nil
+}
+
+func UpdateCleanupRule(ctx context.Context, pcr *PackageCleanupRule) error {
+	_, err := db.GetEngine(ctx).ID(pcr.ID).AllCols().Update(pcr)
+	return err
+}
+
+func GetCleanupRulesByOwner(ctx context.Context, ownerID int64) ([]*PackageCleanupRule, error) {
+	pcrs := make([]*PackageCleanupRule, 0, 10)
+	return pcrs, db.GetEngine(ctx).Where("owner_id = ?", ownerID).Find(&pcrs)
+}
+
+func DeleteCleanupRuleByID(ctx context.Context, ruleID int64) error {
+	_, err := db.GetEngine(ctx).ID(ruleID).Delete(&PackageCleanupRule{})
+	return err
+}
+
+func HasOwnerCleanupRuleForPackageType(ctx context.Context, ownerID int64, packageType Type) (bool, error) {
+	return db.GetEngine(ctx).
+		Where("owner_id = ? AND type = ?", ownerID, packageType).
+		Exist(&PackageCleanupRule{})
+}
+
+func IterateEnabledCleanupRules(ctx context.Context, callback func(context.Context, *PackageCleanupRule) error) error {
+	return db.Iterate(
+		ctx,
+		builder.Eq{"enabled": true},
+		callback,
+	)
+}

models/packages/package_version.go

@@ -319,3 +319,12 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
 	count, err := sess.FindAndCount(&pvs)
 	return pvs, count, err
 }
+
+// ExistVersion checks if a version matching the search options exist
+func ExistVersion(ctx context.Context, opts *PackageSearchOptions) (bool, error) {
+	return db.GetEngine(ctx).
+		Where(opts.toConds()).
+		Table("package_version").
+		Join("INNER", "package", "package.id = package_version.package_id").
+		Exist(new(PackageVersion))
+}

options/locale/locale_en-US.ini

@@ -86,6 +86,9 @@ remove = Remove
 remove_all = Remove All
 edit = Edit
 
+enabled = Enabled
+disabled = Disabled
+
 copy = Copy
 copy_url = Copy URL
 copy_branch = Copy branch name
@@ -3179,3 +3182,23 @@ settings.delete.description = Deleting a package is permanent and cannot be undo
 settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure?
 settings.delete.success = The package has been deleted.
 settings.delete.error = Failed to delete the package.
+owner.settings.cleanuprules.title = Manage Cleanup Rules
+owner.settings.cleanuprules.add = Add Cleanup Rule
+owner.settings.cleanuprules.edit = Edit Cleanup Rule
+owner.settings.cleanuprules.none = No cleanup rules available. Read the docs to learn more.
+owner.settings.cleanuprules.preview = Cleanup Rule Preview
+owner.settings.cleanuprules.preview.overview = %d packages are scheduled to be removed.
+owner.settings.cleanuprules.preview.none = Cleanup rule does not match any packages.
+owner.settings.cleanuprules.enabled = Enabled
+owner.settings.cleanuprules.pattern_full_match = Apply pattern to full package name
+owner.settings.cleanuprules.keep.title = Versions that match these rules are kept, even if they match a removal rule below.
+owner.settings.cleanuprules.keep.count = Keep the most recent
+owner.settings.cleanuprules.keep.count.1 = 1 version per package
+owner.settings.cleanuprules.keep.count.n = %d versions per package
+owner.settings.cleanuprules.keep.pattern = Keep versions matching
+owner.settings.cleanuprules.keep.pattern.container = The <code>latest</code> version is always kept for Container packages.
+owner.settings.cleanuprules.remove.title = Versions that match these rules are removed, unless a rule above says to keep them.
+owner.settings.cleanuprules.remove.days = Remove versions older than
+owner.settings.cleanuprules.remove.pattern = Remove versions matching
+owner.settings.cleanuprules.success.update = Cleanup rule has been updated.
+owner.settings.cleanuprules.success.delete = Cleanup rule has been deleted.

routers/web/org/setting_packages.go

@@ -0,0 +1,87 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package org
+
+import (
+	"fmt"
+	"net/http"
+
+	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/setting"
+	shared "code.gitea.io/gitea/routers/web/shared/packages"
+)
+
+const (
+	tplSettingsPackages            base.TplName = "org/settings/packages"
+	tplSettingsPackagesRuleEdit    base.TplName = "org/settings/packages_cleanup_rules_edit"
+	tplSettingsPackagesRulePreview base.TplName = "org/settings/packages_cleanup_rules_preview"
+)
+
+func Packages(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.SetPackagesContext(ctx, ctx.ContextUser)
+
+	ctx.HTML(http.StatusOK, tplSettingsPackages)
+}
+
+func PackagesRuleAdd(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.SetRuleAddContext(ctx)
+
+	ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
+}
+
+func PackagesRuleEdit(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.SetRuleEditContext(ctx, ctx.ContextUser)
+
+	ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
+}
+
+func PackagesRuleAddPost(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.PerformRuleAddPost(
+		ctx,
+		ctx.ContextUser,
+		fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name),
+		tplSettingsPackagesRuleEdit,
+	)
+}
+
+func PackagesRuleEditPost(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.PerformRuleEditPost(
+		ctx,
+		ctx.ContextUser,
+		fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name),
+		tplSettingsPackagesRuleEdit,
+	)
+}
+
+func PackagesRulePreview(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.SetRulePreviewContext(ctx, ctx.ContextUser)
+
+	ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
+}