implement project reset cleanup

backend/LexBoxApi/Controllers/TestingController.cs

@@ -3,6 +3,7 @@
 using LexBoxApi.Services;
 using LexCore.Auth;
 using LexCore.Exceptions;
+using LexCore.ServiceInterfaces;
 using LexData;
 using LexData.Entities;
 using Microsoft.AspNetCore.Authorization;
@@ -16,6 +17,7 @@ namespace LexBoxApi.Controllers;
 public class TestingController(
     LexAuthService lexAuthService,
     LexBoxDbContext lexBoxDbContext,
+    IHgService hgService,
     SeedingData seedingData)
     : ControllerBase
 {
@@ -84,4 +86,11 @@ public ActionResult ThrowsException()
     [AllowAnonymous]
     [ProducesResponseType(StatusCodes.Status500InternalServerError)]
     public ActionResult Test500NoError() => StatusCode(500);
+
+    [HttpGet("test-cleanup-reset-backups")]
+    [AdminRequired]
+    public async Task<string[]> TestCleanupResetBackups(bool dryRun = true)
+    {
+        return await hgService.CleanupResetBackups(dryRun);
+    }
 }

backend/LexBoxApi/Jobs/CleanupResetBackupJob.cs

@@ -12,8 +12,7 @@ protected override async Task ExecuteJob(IJobExecutionContext context)
     {
         logger.LogInformation("Starting cleanup reset backup job");
 
-        //todo implement job
-        await Task.Delay(TimeSpan.FromSeconds(1));
+        await hgService.CleanupResetBackups();
 
         logger.LogInformation("Finished cleanup reset backup job");
     }

backend/LexBoxApi/Services/HgService.cs

@@ -18,7 +18,7 @@
 
 namespace LexBoxApi.Services;
 
-public class HgService : IHgService, IHostedService
+public partial class HgService : IHgService, IHostedService
 {
     private const string DELETED_REPO_FOLDER = ProjectCode.DELETED_REPO_FOLDER;
     private const string TEMP_REPO_FOLDER = ProjectCode.TEMP_REPO_FOLDER;
@@ -99,13 +99,18 @@
     {
         var tmpRepo = new DirectoryInfo(GetTempRepoPath(code, "reset"));
         InitRepoAt(tmpRepo);
-        await SoftDeleteRepo(code, $"{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}__reset");
+        await SoftDeleteRepo(code, ResetSoftDeleteSuffix(DateTimeOffset.UtcNow));
         //we must init the repo as uploading a zip is optional
         tmpRepo.MoveTo(PrefixRepoFilePath(code));
         await InvalidateDirCache(code);
         await WaitForRepoEmptyState(code, RepoEmptyState.Empty);
     }
 
+    public static string ResetSoftDeleteSuffix(DateTimeOffset resetAt)
+    {
+        return $"{FileUtils.ToTimestamp(resetAt)}__reset";
+    }
+
     public async Task FinishReset(ProjectCode code, Stream zipFile)
     {
         var tempRepoPath = GetTempRepoPath(code, "upload");
@@ -144,6 +149,55 @@
         await WaitForRepoEmptyState(code, expectedState);
     }
 
+
+    public async Task<string[]> CleanupResetBackups(bool dryRun = false)
+    {
+        using var activity = LexBoxActivitySource.Get().StartActivity();
+        List<string> deletedRepos = [];
+        int deletedCount = 0;
+        int totalCount = 0;
+        foreach (var deletedRepo in Directory.EnumerateDirectories(Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER)))
+        {
+            totalCount++;
+            var deletedRepoName = Path.GetFileName(deletedRepo);
+            var resetDate = GetResetDate(deletedRepoName);
+            if (resetDate is null)
+            {
+                continue;
+            }
+            var resetAge = DateTimeOffset.UtcNow - resetDate.Value;
+            //enforce a minimum age threshold, just in case the threshold is set too low
+            var ageThreshold = TimeSpan.FromDays(Math.Max(_options.Value.ResetCleanupAgeDays, 5));
+            if (resetAge <= ageThreshold) continue;
+            try
+            {
+                if (!dryRun)
+                    await Task.Run(() => Directory.Delete(deletedRepo, true));
+                deletedRepos.Add(deletedRepoName);
+                deletedCount++;
+            }
+            catch (Exception e)
+            {
+                activity?.AddTag("app.hg.cleanupresetbackups.error", e.Message);
+            }
+        }
+        activity?.AddTag("app.hg.cleanupresetbackups", totalCount);
+        activity?.AddTag("app.hg.cleanupresetbackups.deleted", deletedCount);
+        activity?.AddTag("app.hg.cleanupresetbackups.dryrun", dryRun);
+        return deletedRepos.ToArray();
+    }
+
+    public static DateTimeOffset? GetResetDate(string repoName)
+    {
+        var match = ResetProjectsRegex().Match(repoName);
+        if (!match.Success) return null;
+        return FileUtils.ToDateTimeOffset(match.Groups[1].Value);
+    }
+
+    [GeneratedRegex(@"__(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})__reset$")]
+    public static partial Regex ResetProjectsRegex();
+
+
     /// <summary>
     /// deletes all files and folders in the repo folder except for .hg
     /// </summary>
@@ -227,7 +281,7 @@
 
     public async Task SoftDeleteRepo(ProjectCode code, string deletedRepoSuffix)
     {
-        var deletedRepoName = $"{code}__{deletedRepoSuffix}";
+        var deletedRepoName = DeletedRepoName(code, deletedRepoSuffix);
         await Task.Run(() =>
         {
             var deletedRepoPath = Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER);
@@ -240,6 +294,11 @@
         });
     }
 
+    public static string DeletedRepoName(ProjectCode code, string deletedRepoSuffix)
+    {
+        return $"{code}__{deletedRepoSuffix}";
+    }
+
     private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite |
                                              UnixFileMode.GroupExecute | UnixFileMode.SetGroup |
                                              UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
@@ -371,7 +430,7 @@
             {
                 var hash = await GetTipHash(code, timeoutSource.Token);
                 var isEmpty = hash == AllZeroHash;
                 done = expectedState switch
                 {
                     RepoEmptyState.Empty => isEmpty,
                     RepoEmptyState.NonEmpty => !isEmpty

backend/LexCore/Config/HgConfig.cs

@@ -17,4 +17,5 @@ public class HgConfig
     public required string HgResumableUrl { get; init; }
     public bool AutoUpdateLexEntryCountOnSendReceive { get; init; } = false;
     public bool RequireContainerVersionMatch { get; init; } = true;
+    public int ResetCleanupAgeDays { get; init; } = 31;
 }

backend/LexCore/ServiceInterfaces/IHgService.cs

@@ -24,4 +24,6 @@ public interface IHgService
     Task<HttpContent> InvalidateDirCache(ProjectCode code, CancellationToken token = default);
     bool HasAbandonedTransactions(ProjectCode projectCode);
     Task<string> HgCommandHealth();
+
+    Task<string[]> CleanupResetBackups(bool dryRun = false);
 }

backend/LexCore/Utils/FileUtils.cs

@@ -5,11 +5,22 @@ namespace LexCore.Utils;
 
 public static class FileUtils
 {
+    private static readonly string TimestampPattern = DateTimeFormatInfo.InvariantInfo.SortableDateTimePattern.Replace(':', '-');
     public static string ToTimestamp(DateTimeOffset dateTime)
     {
-        var timestamp = dateTime.ToString(DateTimeFormatInfo.InvariantInfo.SortableDateTimePattern);
+        var timestamp = dateTime.ToUniversalTime().ToString(TimestampPattern);
         // make it file-system friendly
-        return timestamp.Replace(':', '-');
+        return timestamp;
+    }
+
+    public static DateTimeOffset? ToDateTimeOffset(string timestamp)
+    {
+        if (DateTimeOffset.TryParseExact(timestamp, TimestampPattern, null, DateTimeStyles.AssumeUniversal, out var dateTime))
+        {
+            return dateTime;
+        }
+
+        return null;
     }
 
     public static void CopyFilesRecursively(DirectoryInfo source, DirectoryInfo target, UnixFileMode? permissions = null)

backend/Testing/Services/CleanupResetProjectsTests.cs

@@ -0,0 +1,53 @@
+using LexBoxApi.Services;
+using LexCore.Utils;
+using Shouldly;
+
+namespace Testing.Services;
+
+public class CleanupResetProjectsTests
+{
+    [Fact]
+    public void ResetRegexCanFindTimestampFromResetRepoName()
+    {
+        var date = DateTimeOffset.UtcNow;
+        var repoName = HgService.DeletedRepoName("test", HgService.ResetSoftDeleteSuffix(date));
+        var match = HgService.ResetProjectsRegex().Match(repoName);
+        match.Success.ShouldBeTrue();
+        match.Groups[1].Value.ShouldBe(FileUtils.ToTimestamp(date));
+    }
+
+    [Fact]
+    public void CanGetDateFromResetRepoName()
+    {
+        var expected = DateTimeOffset.Now;
+        var repoName = HgService.DeletedRepoName("test", HgService.ResetSoftDeleteSuffix(expected));
+        var actual = HgService.GetResetDate(repoName);
+        actual.ShouldNotBeNull();
+        TruncateToMinutes(actual.Value).ShouldBe(TruncateToMinutes(expected));
+    }
+
+    private DateTimeOffset TruncateToMinutes(DateTimeOffset date)
+    {
+        return new DateTimeOffset(date.Year, date.Month, date.Day, date.Hour, date.Minute, 0, date.Offset);
+    }
+
+    [Theory]
+    [InlineData("grobish-test-flex__2023-11-29T16-52-38__reset", "2023-11-29T16-52-38")]
+    public void ResetRegexCanFindTimestamp(string repoName, string timestamp)
+    {
+        var match = HgService.ResetProjectsRegex().Match(repoName);
+        match.Success.ShouldBeTrue();
+        match.Groups[1].Value.ShouldBe(timestamp);
+    }
+
+    [Theory]
+    [InlineData("grobish-test-flex__2023-11-29T16-52-38")]
+    //even if the code has a pattern that would match the reset with timestamp it mush be at the end
+    [InlineData("code-with-bad-name-2023-11-29T16-52-38__reset__2023-11-29T16-52-38")]
+    public void ResetRegexDoesNotMatchNonResets(string repoName)
+    {
+        var match = HgService.ResetProjectsRegex().Match(repoName);
+        match.Success.ShouldBeFalse();
+    }
+
+}