diff --git a/cmd/yorkie/server.go b/cmd/yorkie/server.go index cca3b91d5..2bca53098 100644 --- a/cmd/yorkie/server.go +++ b/cmd/yorkie/server.go @@ -38,9 +38,11 @@ var ( flagConfPath string flagLogLevel string - adminTokenDuration time.Duration - housekeepingInterval time.Duration - clientDeactivateThreshold string + adminTokenDuration time.Duration + housekeepingDeactivateCandidatesInterval time.Duration + housekeepingDeleteDocumentsInterval time.Duration + documentHardDeletionGracefulPeriod time.Duration + clientDeactivateThreshold string mongoConnectionURI string mongoConnectionTimeout time.Duration @@ -69,7 +71,9 @@ func newServerCmd() *cobra.Command { conf.Backend.AuthWebhookCacheUnauthTTL = authWebhookCacheUnauthTTL.String() conf.Backend.ProjectInfoCacheTTL = projectInfoCacheTTL.String() - conf.Housekeeping.Interval = housekeepingInterval.String() + conf.Housekeeping.DeactivateCandidatesInterval = housekeepingDeactivateCandidatesInterval.String() + conf.Housekeeping.DeleteDocumentsInterval = housekeepingDeleteDocumentsInterval.String() + conf.Housekeeping.DocumentHardDeletionGracefulPeriod = documentHardDeletionGracefulPeriod if mongoConnectionURI != "" { conf.Mongo = &mongo.Config{ @@ -193,18 +197,36 @@ func init() { false, "Enable runtime profiling data via HTTP server.", ) + cmd.Flags().StringVar( + &conf.Housekeeping.DeactivateCandidatesInterval, + "housekeeping-interval-Deactivate-Candidates", + server.DefaultHousekeepingDeactivateCandidatesInterval.String(), + "housekeeping Interval deactivate candidates between housekeeping runs", + ) + cmd.Flags().StringVar( + &conf.Housekeeping.DeleteDocumentsInterval, + "housekeeping-interval-Delete-Documents", + server.DefaultHousekeepingDeleteDocumentsInterval.String(), + "housekeeping Interval delete documents between housekeeping runs", + ) cmd.Flags().DurationVar( - &housekeepingInterval, - "housekeeping-interval", - server.DefaultHousekeepingInterval, - "housekeeping interval between housekeeping runs", + &conf.Housekeeping.DocumentHardDeletionGracefulPeriod, + "housekeeping-DocumentHardDeletion-delete-graceful-period", + server.DefaultHousekeepingDocumentHardDeletionGracefulPeriod, + "Document deletion over time after a single housekeeping run", ) cmd.Flags().IntVar( - &conf.Housekeeping.CandidatesLimitPerProject, + &conf.Housekeeping.ClientDeactivationCandidateLimitPerProject, "housekeeping-candidates-limit-per-project", - server.DefaultHousekeepingCandidatesLimitPerProject, + server.DefaultHousekeepingClientDeactivationCandidateLimitPerProject, "candidates limit per project for a single housekeeping run", ) + cmd.Flags().IntVar( + &conf.Housekeeping.DocumentHardDeletionCandidateLimitPerProject, + "housekeeping-DocumentHardDeletion-limit-per-project", + server.DefaultHousekeepingDocumentHardDeletionCandidateLimitPerProject, + "Document Deletion limit per project for a single housekeeping run", + ) cmd.Flags().IntVar( &conf.Housekeeping.ProjectFetchSize, "housekeeping-project-fetch-size", diff --git a/server/backend/database/database.go b/server/backend/database/database.go index 4c45b1378..aaf6291d0 100644 --- a/server/backend/database/database.go +++ b/server/backend/database/database.go @@ -20,6 +20,7 @@ package database import ( "context" "errors" + gotime "time" "github.com/yorkie-team/yorkie/api/types" "github.com/yorkie-team/yorkie/pkg/document" @@ -163,6 +164,20 @@ type Database interface { candidatesLimit int, ) ([]*ClientInfo, error) + // FindDocumentHardDeletionCandidatesPerProject finds the documents that need to be deleted housekeeping per project. + FindDocumentHardDeletionCandidatesPerProject( + ctx context.Context, + project *ProjectInfo, + candidatesLimit int, + documentHardDeletionGracefulPeriod gotime.Duration, + ) ([]*DocInfo, error) + + // DeleteDocuments deletes document of the given key + DeleteDocuments( + ctx context.Context, + candidates []*DocInfo, + ) (int64, error) + // FindDocInfoByKey finds the document of the given key. FindDocInfoByKey( ctx context.Context, diff --git a/server/backend/database/memory/database.go b/server/backend/database/memory/database.go index 13a04789c..53d5a047c 100644 --- a/server/backend/database/memory/database.go +++ b/server/backend/database/memory/database.go @@ -673,6 +673,44 @@ func (d *DB) UpdateClientInfoAfterPushPull( return nil } +// FindDocumentHardDeletionCandidatesPerProject finds the documents that need housekeeping per project. +func (d *DB) FindDocumentHardDeletionCandidatesPerProject( + _ context.Context, + project *database.ProjectInfo, + candidatesLimit int, + documentHardDeletionGracefulPeriod gotime.Duration, +) ([]*database.DocInfo, error) { + txn := d.db.Txn(false) + defer txn.Abort() + + offset := gotime.Now().Add(-documentHardDeletionGracefulPeriod) + + var documents []*database.DocInfo + iterator, err := txn.ReverseLowerBound( + tblDocuments, + "project_id_removed_at", + project.ID.String(), + offset, + ) + + if err != nil { + return nil, fmt.Errorf("fetch hard deletion candidates: %w", err) + } + + for raw := iterator.Next(); raw != nil; raw = iterator.Next() { + document := raw.(*database.DocInfo) + if candidatesLimit <= len(documents) && candidatesLimit != 0 { + break + } + + if !document.RemovedAt.After(offset) { + documents = append(documents, document) + } + } + + return documents, nil +} + // FindDeactivateCandidatesPerProject finds the clients that need housekeeping per project. func (d *DB) FindDeactivateCandidatesPerProject( _ context.Context, @@ -717,6 +755,30 @@ func (d *DB) FindDeactivateCandidatesPerProject( return infos, nil } +// DeleteDocuments Deletes the documents completely. +func (d *DB) DeleteDocuments( + _ context.Context, + candidates []*database.DocInfo, +) (int64, error) { + if len(candidates) <= 0 { + return 0, nil + } + + txn := d.db.Txn(true) + defer txn.Abort() + + var deletedCount int64 + for _, candidate := range candidates { + if err := txn.Delete(tblDocuments, candidate); err != nil { + return 0, fmt.Errorf("fetch hard deletion candidates: %w", err) + } + deletedCount++ + } + txn.Commit() + + return deletedCount, nil +} + // FindDocInfoByKeyAndOwner finds the document of the given key. If the // createDocIfNotExist condition is true, create the document if it does not // exist. diff --git a/server/backend/database/memory/database_test.go b/server/backend/database/memory/database_test.go index d308940a7..8b03e50dc 100644 --- a/server/backend/database/memory/database_test.go +++ b/server/backend/database/memory/database_test.go @@ -111,4 +111,8 @@ func TestDB(t *testing.T) { t.Run("IsDocumentAttached test", func(t *testing.T) { testcases.RunIsDocumentAttachedTest(t, db, projectID) }) + + t.Run("DocumentHardDeletion test", func(t *testing.T) { + testcases.RunDocumentHardDeletionTest(t, db) + }) } diff --git a/server/backend/database/memory/indexes.go b/server/backend/database/memory/indexes.go index c30c352b4..d494a9a3f 100644 --- a/server/backend/database/memory/indexes.go +++ b/server/backend/database/memory/indexes.go @@ -136,6 +136,15 @@ var schema = &memdb.DBSchema{ }, }, }, + "project_id_removed_at": { + Name: "project_id_removed_at", + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &memdb.StringFieldIndex{Field: "ProjectID"}, + &memdb.TimeFieldIndex{Field: "RemovedAt"}, + }, + }, + }, "project_id_key_removed_at": { Name: "project_id_key_removed_at", Indexer: &memdb.CompoundIndex{ diff --git a/server/backend/database/mongo/client.go b/server/backend/database/mongo/client.go index 16ec85b63..cb6a7a818 100644 --- a/server/backend/database/mongo/client.go +++ b/server/backend/database/mongo/client.go @@ -227,6 +227,41 @@ func (c *Client) CreateProjectInfo( return info, nil } +// DeleteDocuments Deletes the documents completely. +func (c *Client) DeleteDocuments( + ctx context.Context, + candidates []*database.DocInfo, +) (int64, error) { + if len(candidates) <= 0 { + return 0, nil + } + + for _, docInfo := range candidates { + if docInfo.ID == "" { + return 0, fmt.Errorf("invalid document ID") + } + } + + var idList []types.ID + for _, docInfo := range candidates { + idList = append( + idList, + docInfo.ID, + ) + } + + deletedResult, err := c.collection(ColDocuments).DeleteMany( + ctx, + bson.M{"_id": bson.M{"$in": idList}}, + ) + + if err != nil { + return deletedResult.DeletedCount, fmt.Errorf("failed to delete documents: %w", err) + } + + return deletedResult.DeletedCount, nil +} + // FindNextNCyclingProjectInfos finds the next N cycling projects from the given projectID. func (c *Client) FindNextNCyclingProjectInfos( ctx context.Context, @@ -672,6 +707,34 @@ func (c *Client) UpdateClientInfoAfterPushPull( return nil } +// FindDocumentHardDeletionCandidatesPerProject finds the documents that need housekeeping per project. +func (c *Client) FindDocumentHardDeletionCandidatesPerProject( + ctx context.Context, + project *database.ProjectInfo, + candidatesLimit int, + documentHardDeletionGracefulPeriod gotime.Duration, +) ([]*database.DocInfo, error) { + + currentTime := gotime.Now() + hardDeletionGracefulPeriod := currentTime.Add(-documentHardDeletionGracefulPeriod) + + var DocInfos []*database.DocInfo + cursor, err := c.collection(ColDocuments).Find(ctx, bson.M{ + "project_id": project.ID, + "removed_at": bson.M{"$lte": hardDeletionGracefulPeriod}, + }, options.Find().SetLimit(int64(candidatesLimit))) + + if err != nil { + return nil, err + } + + if err := cursor.All(ctx, &DocInfos); err != nil { + return nil, fmt.Errorf("fetch hard deletion candidates: %w", err) + } + + return DocInfos, nil +} + // FindDeactivateCandidatesPerProject finds the clients that need housekeeping per project. func (c *Client) FindDeactivateCandidatesPerProject( ctx context.Context, diff --git a/server/backend/database/mongo/client_test.go b/server/backend/database/mongo/client_test.go index 6e0803b5d..6ed452801 100644 --- a/server/backend/database/mongo/client_test.go +++ b/server/backend/database/mongo/client_test.go @@ -128,4 +128,8 @@ func TestClient(t *testing.T) { t.Run("IsDocumentAttached test", func(t *testing.T) { testcases.RunIsDocumentAttachedTest(t, cli, dummyProjectID) }) + + t.Run("DocumentHardDeletion test", func(t *testing.T) { + testcases.RunDocumentHardDeletionTest(t, cli) + }) } diff --git a/server/backend/database/testcases/testcases.go b/server/backend/database/testcases/testcases.go index 2108f8c83..a5aad76eb 100644 --- a/server/backend/database/testcases/testcases.go +++ b/server/backend/database/testcases/testcases.go @@ -1592,3 +1592,59 @@ func AssertKeys(t *testing.T, expectedKeys []key.Key, infos []*database.DocInfo) } assert.EqualValues(t, expectedKeys, keys) } + +// RunDocumentHardDeletionTest runs the DocumentHardDeletion tests for the given db +func RunDocumentHardDeletionTest(t *testing.T, db database.Database) { + t.Run("housekeeping DocumentHardDeletion test", func(t *testing.T) { + ctx := context.Background() + docKey := helper.TestDocKey(t) + + // 00. Create a project + projectInfo, err := db.CreateProjectInfo(ctx, t.Name(), dummyOwnerID, clientDeactivateThreshold) + assert.NoError(t, err) + + // 01. Create a client and a document then attach the document to the client. + clientInfo, err := db.ActivateClient(ctx, projectInfo.ID, t.Name()) + assert.NoError(t, err) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) + assert.NoError(t, err) + docRefKey := docInfo.RefKey() + assert.NoError(t, clientInfo.AttachDocument(docInfo.ID, false)) + assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) + + doc := document.New(key.Key(t.Name())) + pack := doc.CreateChangePack() + + // 02. Set removed_at in docInfo and store changes + assert.NoError(t, clientInfo.RemoveDocument(docInfo.ID)) + err = db.CreateChangeInfos(ctx, projectInfo.ID, docInfo, 0, pack.Changes, true) + assert.NoError(t, err) + + // 03. Set the grace period to 0 seconds. + var candidates []*database.DocInfo + GracePeriod := "-1s" + documentHardDeletionGracefulPeriod, err := gotime.ParseDuration(GracePeriod) + assert.NoError(t, err) + + // 04. Find documents whose deleted_at time is less than or equal to current time minus GracePeriod. + fetchSize := 100 + candidates, err = db.FindDocumentHardDeletionCandidatesPerProject( + ctx, + projectInfo, + fetchSize, + documentHardDeletionGracefulPeriod, + ) + assert.NoError(t, err) + + // 05. Deletes document of the given key + // Compare the number of candidates for deletion with the number of deleted documents. + deletedDocumentsCount, err := db.DeleteDocuments(ctx, candidates) + assert.NoError(t, err) + assert.Equal(t, int(deletedDocumentsCount), len(candidates)) + + _, err = db.FindDocInfoByRefKey(ctx, docRefKey) + assert.ErrorIs(t, err, database.ErrDocumentNotFound) + + }) + +} diff --git a/server/backend/housekeeping/config.go b/server/backend/housekeeping/config.go index ff2902e9f..aab2e8301 100644 --- a/server/backend/housekeeping/config.go +++ b/server/backend/housekeeping/config.go @@ -25,11 +25,20 @@ import ( // Config is the configuration for the housekeeping service. type Config struct { - // Interval is the time between housekeeping runs. - Interval string `yaml:"Interval"` + // DeactivateCandidatesInterval is the time between housekeeping runs for deactivate candidates. + DeactivateCandidatesInterval string `yaml:"DeactivateCandidatesInterval"` - // CandidatesLimitPerProject is the maximum number of candidates to be returned per project. - CandidatesLimitPerProject int `yaml:"CandidatesLimitPerProject"` + // DeleteDocumentsInterval is the time between housekeeping runs for document deletion. + DeleteDocumentsInterval string `yaml:"DeleteDocumentsInterval"` + + // DocumentHardDeletionGracefulPeriod finds documents whose removed_at time is older than that time. + DocumentHardDeletionGracefulPeriod time.Duration `yaml:"HousekeepingDocumentHardDeletionGracefulPeriod"` + + // ClientDeactivationCandidateLimitPerProject is the maximum number of candidates to be returned per project. + ClientDeactivationCandidateLimitPerProject int `yaml:"ClientDeactivationCandidateLimitPerProject"` + + // DocumentHardDeletionCandidateLimitPerProject is the maximum number of candidates to be returned per project. + DocumentHardDeletionCandidateLimitPerProject int `yaml:"DocumentHardDeletionCandidateLimitPerProject"` // ProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. ProjectFetchSize int `yaml:"HousekeepingProjectFetchSize"` @@ -37,18 +46,40 @@ type Config struct { // Validate validates the configuration. func (c *Config) Validate() error { - if _, err := time.ParseDuration(c.Interval); err != nil { + if _, err := time.ParseDuration(c.DeactivateCandidatesInterval); err != nil { return fmt.Errorf( - `invalid argument %s for "--housekeeping-interval" flag: %w`, - c.Interval, + `invalid argument %s for "--housekeeping-interval-deactivate-candidates" flag: %w`, + c.DeactivateCandidatesInterval, err, ) } - if c.CandidatesLimitPerProject <= 0 { + if _, err := time.ParseDuration(c.DeleteDocumentsInterval); err != nil { return fmt.Errorf( - `invalid argument %d for "--housekeeping-candidates-limit-per-project" flag`, - c.ProjectFetchSize, + `invalid argument %s for "--housekeeping-interval-delete-documents" flag: %w`, + c.DeleteDocumentsInterval, + err, + ) + } + + if c.DocumentHardDeletionGracefulPeriod <= 0 { + return fmt.Errorf( + `invalid argument %v for "--housekeeping-project-delete-graceful-period"`, + c.DocumentHardDeletionGracefulPeriod, + ) + } + + if c.ClientDeactivationCandidateLimitPerProject <= 0 { + return fmt.Errorf( + `invalid argument %d for "--housekeeping-client-deactivateion-candidate-limit-per-project"`, + c.ClientDeactivationCandidateLimitPerProject, + ) + } + + if c.DocumentHardDeletionCandidateLimitPerProject <= 0 { + return fmt.Errorf( + `invalid argument %d for "--housekeeping-document-hard-deletion-limit-per-project"`, + c.DocumentHardDeletionCandidateLimitPerProject, ) } @@ -63,11 +94,13 @@ func (c *Config) Validate() error { } // ParseInterval parses the interval. -func (c *Config) ParseInterval() (time.Duration, error) { - interval, err := time.ParseDuration(c.Interval) +func (c *Config) ParseInterval( + interval string, +) (time.Duration, error) { + parseInterval, err := time.ParseDuration(interval) if err != nil { - return 0, fmt.Errorf("parse interval %s: %w", c.Interval, err) + return 0, fmt.Errorf("parse interval %s: %w", interval, err) } - return interval, nil + return parseInterval, nil } diff --git a/server/backend/housekeeping/config_test.go b/server/backend/housekeeping/config_test.go index 876904135..058ca83a0 100644 --- a/server/backend/housekeeping/config_test.go +++ b/server/backend/housekeeping/config_test.go @@ -27,22 +27,38 @@ import ( func TestConfig(t *testing.T) { t.Run("validate test", func(t *testing.T) { validConf := housekeeping.Config{ - Interval: "1m", - CandidatesLimitPerProject: 100, - ProjectFetchSize: 100, + DeactivateCandidatesInterval: "1m", + DeleteDocumentsInterval: "1m", + DocumentHardDeletionGracefulPeriod: 60, + ClientDeactivationCandidateLimitPerProject: 100, + DocumentHardDeletionCandidateLimitPerProject: 100, + ProjectFetchSize: 100, } assert.NoError(t, validConf.Validate()) conf1 := validConf - conf1.Interval = "hour" + conf1.DeactivateCandidatesInterval = "hour" assert.Error(t, conf1.Validate()) conf2 := validConf - conf2.CandidatesLimitPerProject = 0 + conf2.DeleteDocumentsInterval = "minute" assert.Error(t, conf2.Validate()) conf3 := validConf - conf3.ProjectFetchSize = -1 + conf3.DocumentHardDeletionGracefulPeriod = 0 assert.Error(t, conf3.Validate()) + + conf4 := validConf + conf4.ClientDeactivationCandidateLimitPerProject = 0 + assert.Error(t, conf4.Validate()) + + conf5 := validConf + conf5.DocumentHardDeletionCandidateLimitPerProject = 0 + assert.Error(t, conf5.Validate()) + + conf6 := validConf + conf6.ProjectFetchSize = 0 + assert.Error(t, conf6.Validate()) + }) } diff --git a/server/backend/housekeeping/housekeeping.go b/server/backend/housekeeping/housekeeping.go index 2f54281d3..1892c8a4a 100644 --- a/server/backend/housekeeping/housekeeping.go +++ b/server/backend/housekeeping/housekeeping.go @@ -29,8 +29,7 @@ import ( // Housekeeping is the housekeeping service. It periodically runs housekeeping // tasks. type Housekeeping struct { - Config *Config - + Config *Config scheduler gocron.Scheduler } @@ -82,6 +81,5 @@ func (h *Housekeeping) Stop() error { if err := h.scheduler.Shutdown(); err != nil { return fmt.Errorf("scheduler shutdown: %w", err) } - return nil } diff --git a/server/clients/housekeeping.go b/server/clients/housekeeping.go index 76ab28bd8..930a55ee3 100644 --- a/server/clients/housekeeping.go +++ b/server/clients/housekeeping.go @@ -26,8 +26,10 @@ import ( "github.com/yorkie-team/yorkie/server/logging" ) +// Identification key for distributed work const ( - deactivateCandidatesKey = "housekeeping/deactivateCandidates" + DocumentHardDeletionLockKey = "housekeeping/documentHardDeletionLock" + DeactivateCandidatesKey = "housekeeping/deactivateCandidates" ) // DeactivateInactives deactivates clients that have not been active for a @@ -35,13 +37,13 @@ const ( func DeactivateInactives( ctx context.Context, be *backend.Backend, - candidatesLimitPerProject int, + clientDeactivationCandidateLimitPerProject int, projectFetchSize int, housekeepingLastProjectID types.ID, ) (types.ID, error) { start := time.Now() - locker, err := be.Coordinator.NewLocker(ctx, deactivateCandidatesKey) + locker, err := be.Coordinator.NewLocker(ctx, DeactivateCandidatesKey) if err != nil { return database.DefaultProjectID, err } @@ -59,7 +61,7 @@ func DeactivateInactives( lastProjectID, candidates, err := FindDeactivateCandidates( ctx, be, - candidatesLimitPerProject, + clientDeactivationCandidateLimitPerProject, projectFetchSize, housekeepingLastProjectID, ) @@ -92,7 +94,7 @@ func DeactivateInactives( func FindDeactivateCandidates( ctx context.Context, be *backend.Backend, - candidatesLimitPerProject int, + clientDeactivationCandidateLimitPerProject int, projectFetchSize int, lastProjectID types.ID, ) (types.ID, []*database.ClientInfo, error) { @@ -103,7 +105,103 @@ func FindDeactivateCandidates( var candidates []*database.ClientInfo for _, project := range projects { - infos, err := be.DB.FindDeactivateCandidatesPerProject(ctx, project, candidatesLimitPerProject) + infos, err := be.DB.FindDeactivateCandidatesPerProject(ctx, project, clientDeactivationCandidateLimitPerProject) + if err != nil { + return database.DefaultProjectID, nil, err + } + + candidates = append(candidates, infos...) + } + + var topProjectID types.ID + if len(projects) < projectFetchSize { + topProjectID = database.DefaultProjectID + } else { + topProjectID = projects[len(projects)-1].ID + } + + return topProjectID, candidates, nil +} + +// DeleteDocuments deletes a document +func DeleteDocuments( + ctx context.Context, + be *backend.Backend, + documentHardDeletionCandidateLimitPerProject int, + documentHardDeletionGracefulPeriod time.Duration, + projectFetchSize int, + housekeepingLastProjectID types.ID, +) (types.ID, error) { + + start := time.Now() + locker, err := be.Coordinator.NewLocker(ctx, DocumentHardDeletionLockKey) + if err != nil { + return database.DefaultProjectID, err + } + + if err := locker.Lock(ctx); err != nil { + return database.DefaultProjectID, err + } + + defer func() { + if err := locker.Unlock(ctx); err != nil { + logging.From(ctx).Error(err) + } + }() + + lastProjectID, candidates, err := FindDocumentHardDeletionCandidates( + ctx, + be, + documentHardDeletionCandidateLimitPerProject, + projectFetchSize, + documentHardDeletionGracefulPeriod, + housekeepingLastProjectID, + ) + + if err != nil { + return database.DefaultProjectID, err + } + + deletedDocumentsCount, err := be.DB.DeleteDocuments(ctx, candidates) + + if err != nil { + return database.DefaultProjectID, err + } + + if len(candidates) > 0 { + logging.From(ctx).Infof( + "HSKP: candidates %d, hard deleted %d, %s", + len(candidates), + deletedDocumentsCount, + time.Since(start), + ) + } + + return lastProjectID, nil +} + +// FindDocumentHardDeletionCandidates finds the clients that need housekeeping. +func FindDocumentHardDeletionCandidates( + ctx context.Context, + be *backend.Backend, + documentHardDeletionCandidateLimitPerProject int, + projectFetchSize int, + deletedAfterTime time.Duration, + lastProjectID types.ID, +) (types.ID, []*database.DocInfo, error) { + projects, err := be.DB.FindNextNCyclingProjectInfos(ctx, projectFetchSize, lastProjectID) + if err != nil { + return database.DefaultProjectID, nil, err + } + + var candidates []*database.DocInfo + for _, project := range projects { + infos, err := be.DB.FindDocumentHardDeletionCandidatesPerProject( + ctx, + project, + documentHardDeletionCandidateLimitPerProject, + deletedAfterTime, + ) if err != nil { return database.DefaultProjectID, nil, err } diff --git a/server/config.go b/server/config.go index eea4f3a25..a2fa494ae 100644 --- a/server/config.go +++ b/server/config.go @@ -40,9 +40,12 @@ const ( DefaultProfilingPort = 8081 - DefaultHousekeepingInterval = 30 * time.Second - DefaultHousekeepingCandidatesLimitPerProject = 500 - DefaultHousekeepingProjectFetchSize = 100 + DefaultHousekeepingDeactivateCandidatesInterval = 30 * time.Second + DefaultHousekeepingDeleteDocumentsInterval = 30 * time.Second + DefaultHousekeepingDocumentHardDeletionGracefulPeriod = 14 * 24 * time.Hour + DefaultHousekeepingClientDeactivationCandidateLimitPerProject = 500 + DefaultHousekeepingDocumentHardDeletionCandidateLimitPerProject = 500 + DefaultHousekeepingProjectFetchSize = 100 DefaultMongoConnectionURI = "mongodb://localhost:27017" DefaultMongoConnectionTimeout = 5 * time.Second @@ -229,9 +232,12 @@ func newConfig(port int, profilingPort int) *Config { Port: profilingPort, }, Housekeeping: &housekeeping.Config{ - Interval: DefaultHousekeepingInterval.String(), - CandidatesLimitPerProject: DefaultHousekeepingCandidatesLimitPerProject, - ProjectFetchSize: DefaultHousekeepingProjectFetchSize, + DeactivateCandidatesInterval: DefaultHousekeepingDeactivateCandidatesInterval.String(), + DeleteDocumentsInterval: DefaultHousekeepingDeleteDocumentsInterval.String(), + DocumentHardDeletionGracefulPeriod: DefaultHousekeepingDocumentHardDeletionGracefulPeriod, + ClientDeactivationCandidateLimitPerProject: DefaultHousekeepingClientDeactivationCandidateLimitPerProject, + DocumentHardDeletionCandidateLimitPerProject: DefaultHousekeepingDocumentHardDeletionCandidateLimitPerProject, + ProjectFetchSize: DefaultHousekeepingProjectFetchSize, }, Backend: &backend.Config{ ClientDeactivateThreshold: DefaultClientDeactivateThreshold, diff --git a/server/config.sample.yml b/server/config.sample.yml index d5c72edbb..9290c886e 100644 --- a/server/config.sample.yml +++ b/server/config.sample.yml @@ -30,11 +30,20 @@ Profiling: # Housekeeping is the configuration for the housekeeping. Housekeeping: - # Interval is the time between housekeeping runs (default: 1m). - Interval: 1m + # DeactivateCandidatesInterval is the time between housekeeping runs (default: 1m). + DeactivateCandidatesInterval: 1m - # CandidatesLimitPerProject is the maximum number of candidates to be returned per project (default: 100). - CandidatesLimitPerProject: 100 + # DeleteDocumentsInterval is the time between housekeeping runs (default: 1m). + DeleteDocumentsInterval: 1m + + # HousekeepingDocumentHardDeletionGracefulPeriod finds documents whose removed_at time is older than that time. (default: 336h). + HousekeepingDocumentHardDeletionGracefulPeriod: 336h + + # ClientDeactivationCandidateLimitPerProject is the maximum number of candidates to be returned per project (default: 100). + ClientDeactivationCandidateLimitPerProject: 100 + + # DocumentHardDeletionCandidateLimitPerProject is the maximum number of candidates to be returned per project (default: 500). + DocumentHardDeletionCandidateLimitPerProject: 500 # ProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. (default: 100). ProjectFetchSize: 100 diff --git a/server/rpc/server_test.go b/server/rpc/server_test.go index 159c686ed..ac86e9022 100644 --- a/server/rpc/server_test.go +++ b/server/rpc/server_test.go @@ -83,9 +83,12 @@ func TestMain(m *testing.M) { ConnectionTimeout: helper.MongoConnectionTimeout, PingTimeout: helper.MongoPingTimeout, }, &housekeeping.Config{ - Interval: helper.HousekeepingInterval.String(), - CandidatesLimitPerProject: helper.HousekeepingCandidatesLimitPerProject, - ProjectFetchSize: helper.HousekeepingProjectFetchSize, + DeactivateCandidatesInterval: helper.HousekeepingDeactivateCandidatesInterval.String(), + DeleteDocumentsInterval: helper.HousekeepingDeleteDocumentsInterval.String(), + DocumentHardDeletionGracefulPeriod: helper.HousekeepingDocumentHardDeletionGracefulPeriod, + ClientDeactivationCandidateLimitPerProject: helper.HousekeepingClientDeactivationCandidateLimitPerProject, + DocumentHardDeletionCandidateLimitPerProject: helper.HousekeepingDocumentHardDeletionCandidateLimitPerProject, + ProjectFetchSize: helper.HousekeepingProjectFetchSize, }, met) if err != nil { log.Fatal(err) diff --git a/server/server.go b/server/server.go index 6a777f5d5..1be5039d9 100644 --- a/server/server.go +++ b/server/server.go @@ -94,7 +94,11 @@ func (r *Yorkie) Start() error { r.lock.Lock() defer r.lock.Unlock() - if err := r.RegisterHousekeepingTasks(r.backend); err != nil { + if err := r.RegisterDeactivateInactivesTasks(r.backend); err != nil { + return err + } + + if err := r.RegisterDeleteDocumentsTasks(r.backend); err != nil { return err } @@ -157,27 +161,54 @@ func (r *Yorkie) DeactivateClient(ctx context.Context, c1 *client.Client) error return err } -// RegisterHousekeepingTasks registers housekeeping tasks. -func (r *Yorkie) RegisterHousekeepingTasks(be *backend.Backend) error { - interval, err := be.Housekeeping.Config.ParseInterval() +// RegisterDeactivateInactivesTasks registers deactivate inactives housekeeping tasks. +func (r *Yorkie) RegisterDeactivateInactivesTasks(be *backend.Backend) error { + deactivateCandidatesInterval, err := + be.Housekeeping.Config.ParseInterval(be.Housekeeping.Config.DeactivateCandidatesInterval) + if err != nil { + return err + } + + housekeepingLastDeactivateInactivesProjectID := database.DefaultProjectID + return be.Housekeeping.RegisterTask(deactivateCandidatesInterval, func(ctx context.Context) error { + lastProjectDeactivateInactivesID, err := clients.DeactivateInactives( + ctx, + be, + be.Housekeeping.Config.ClientDeactivationCandidateLimitPerProject, + be.Housekeeping.Config.ProjectFetchSize, + housekeepingLastDeactivateInactivesProjectID, + ) + if err != nil { + return err + } + housekeepingLastDeactivateInactivesProjectID = lastProjectDeactivateInactivesID + return nil + }) +} + +// RegisterDeleteDocumentsTasks registers document hard delete housekeeping tasks. +func (r *Yorkie) RegisterDeleteDocumentsTasks(be *backend.Backend) error { + deleteDocumentsInterval, err := + be.Housekeeping.Config.ParseInterval(be.Housekeeping.Config.DeleteDocumentsInterval) if err != nil { return err } - housekeepingLastProjectID := database.DefaultProjectID - return be.Housekeeping.RegisterTask(interval, func(ctx context.Context) error { - lastProjectID, err := clients.DeactivateInactives( + housekeepingLastDeleteDocumentProjectID := database.DefaultProjectID + return be.Housekeeping.RegisterTask(deleteDocumentsInterval, func(ctx context.Context) error { + lastProjectDeleteDocumentID, err := clients.DeleteDocuments( ctx, be, - be.Housekeeping.Config.CandidatesLimitPerProject, + be.Housekeeping.Config.DocumentHardDeletionCandidateLimitPerProject, + be.Housekeeping.Config.DocumentHardDeletionGracefulPeriod, be.Housekeeping.Config.ProjectFetchSize, - housekeepingLastProjectID, + housekeepingLastDeleteDocumentProjectID, ) if err != nil { return err } - housekeepingLastProjectID = lastProjectID + housekeepingLastDeleteDocumentProjectID = lastProjectDeleteDocumentID return nil }) } diff --git a/test/complex/main_test.go b/test/complex/main_test.go index 9ce84b62d..b0f3c8fed 100644 --- a/test/complex/main_test.go +++ b/test/complex/main_test.go @@ -88,9 +88,12 @@ func TestMain(m *testing.M) { ConnectionTimeout: helper.MongoConnectionTimeout, PingTimeout: helper.MongoPingTimeout, }, &housekeeping.Config{ - Interval: helper.HousekeepingInterval.String(), - CandidatesLimitPerProject: helper.HousekeepingCandidatesLimitPerProject, - ProjectFetchSize: helper.HousekeepingProjectFetchSize, + DeactivateCandidatesInterval: helper.HousekeepingDeactivateCandidatesInterval.String(), + DeleteDocumentsInterval: helper.HousekeepingDeleteDocumentsInterval.String(), + DocumentHardDeletionGracefulPeriod: helper.HousekeepingDocumentHardDeletionGracefulPeriod, + ClientDeactivationCandidateLimitPerProject: helper.HousekeepingClientDeactivationCandidateLimitPerProject, + DocumentHardDeletionCandidateLimitPerProject: helper.HousekeepingDocumentHardDeletionCandidateLimitPerProject, + ProjectFetchSize: helper.HousekeepingProjectFetchSize, }, met) if err != nil { log.Fatal(err) diff --git a/test/helper/helper.go b/test/helper/helper.go index 346074c52..717462621 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -62,13 +62,16 @@ var ( ProfilingPort = 11102 - AdminUser = server.DefaultAdminUser - AdminPassword = server.DefaultAdminPassword - AdminPasswordForSignUp = AdminPassword + "123!" - UseDefaultProject = true - HousekeepingInterval = 10 * gotime.Second - HousekeepingCandidatesLimitPerProject = 10 - HousekeepingProjectFetchSize = 10 + AdminUser = server.DefaultAdminUser + AdminPassword = server.DefaultAdminPassword + AdminPasswordForSignUp = AdminPassword + "123!" + UseDefaultProject = true + HousekeepingDeactivateCandidatesInterval = 10 * gotime.Second + HousekeepingDeleteDocumentsInterval = 10 * gotime.Second + HousekeepingDocumentHardDeletionGracefulPeriod = 10 * gotime.Second + HousekeepingClientDeactivationCandidateLimitPerProject = 10 + HousekeepingDocumentHardDeletionCandidateLimitPerProject = 10 + HousekeepingProjectFetchSize = 10 AdminTokenDuration = "10s" ClientDeactivateThreshold = "10s" @@ -236,9 +239,12 @@ func TestConfig() *server.Config { Port: ProfilingPort + portOffset, }, Housekeeping: &housekeeping.Config{ - Interval: HousekeepingInterval.String(), - CandidatesLimitPerProject: HousekeepingCandidatesLimitPerProject, - ProjectFetchSize: HousekeepingProjectFetchSize, + DeactivateCandidatesInterval: HousekeepingDeactivateCandidatesInterval.String(), + DeleteDocumentsInterval: HousekeepingDeleteDocumentsInterval.String(), + DocumentHardDeletionGracefulPeriod: HousekeepingDocumentHardDeletionGracefulPeriod, + ClientDeactivationCandidateLimitPerProject: HousekeepingClientDeactivationCandidateLimitPerProject, + DocumentHardDeletionCandidateLimitPerProject: HousekeepingDocumentHardDeletionCandidateLimitPerProject, + ProjectFetchSize: HousekeepingProjectFetchSize, }, Backend: &backend.Config{ AdminUser: server.DefaultAdminUser,