Fix for #468

sillsdev · Sep 9, 2024 · e6a2277 · e6a2277
1 parent c7f1214
commit e6a2277
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 10 deletions.
diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs
@@ -5,4 +5,5 @@ public class BuildJobOptions
     public const string Key = "BuildJob";
 
     public IList<ClearMLBuildQueue> ClearML { get; set; } = new List<ClearMLBuildQueue>();
+    public bool PreserveBuildFiles { get; set; } = false;
 }
diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs
@@ -7,10 +7,12 @@ public class PostprocessBuildJob(
     IDataAccessContext dataAccessContext,
     IBuildJobService buildJobService,
     ILogger<PostprocessBuildJob> logger,
-    ISharedFileService sharedFileService
+    ISharedFileService sharedFileService,
+    IOptionsMonitor<BuildJobOptions> options
 ) : HangfireBuildJob<(int, double)>(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger)
 {
     protected ISharedFileService SharedFileService { get; } = sharedFileService;
+    private readonly BuildJobOptions _options = options.CurrentValue;
 
     protected override async Task DoWorkAsync(
         string engineId,
@@ -75,6 +77,9 @@ JobCompletionStatus completionStatus
         if (completionStatus is JobCompletionStatus.Restarting)
             return;
 
+        if (_options.PreserveBuildFiles)
+            return;
+
         try
         {
             if (completionStatus is not JobCompletionStatus.Faulted)

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs
@@ -11,7 +11,8 @@ public class SmtTransferPostprocessBuildJob(
     IRepository<TrainSegmentPair> trainSegmentPairs,
     ISmtModelFactory smtModelFactory,
     ITruecaserFactory truecaserFactory,
-    IOptionsMonitor<SmtTransferEngineOptions> options
+    IOptionsMonitor<BuildJobOptions> buildOptions,
+    IOptionsMonitor<SmtTransferEngineOptions> engineOptions
 )
     : PostprocessBuildJob(
         platformService,
@@ -20,13 +21,14 @@ IOptionsMonitor<SmtTransferEngineOptions> options
         dataAccessContext,
         buildJobService,
         logger,
-        sharedFileService
+        sharedFileService,
+        buildOptions
     )
 {
     private readonly ISmtModelFactory _smtModelFactory = smtModelFactory;
     private readonly ITruecaserFactory _truecaserFactory = truecaserFactory;
     private readonly IRepository<TrainSegmentPair> _trainSegmentPairs = trainSegmentPairs;
-    private readonly IOptionsMonitor<SmtTransferEngineOptions> _options = options;
+    private readonly IOptionsMonitor<SmtTransferEngineOptions> _engineOptions = engineOptions;
 
     protected override async Task<int> SaveModelAsync(string engineId, string buildId)
     {
@@ -38,7 +40,7 @@ protected override async Task<int> SaveModelAsync(string engineId, string buildI
         )
         {
             await _smtModelFactory.UpdateEngineFromAsync(
-                Path.Combine(_options.CurrentValue.EnginesDir, engineId),
+                Path.Combine(_engineOptions.CurrentValue.EnginesDir, engineId),
                 engineStream,
                 CancellationToken.None
             );
@@ -54,7 +56,7 @@ private async Task<int> TrainOnNewSegmentPairsAsync(string engineId)
         if (segmentPairs.Count == 0)
             return segmentPairs.Count;
 
-        string engineDir = Path.Combine(_options.CurrentValue.EnginesDir, engineId);
+        string engineDir = Path.Combine(_engineOptions.CurrentValue.EnginesDir, engineId);
         var tokenizer = new LatinWordTokenizer();
         var detokenizer = new LatinWordDetokenizer();
         ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineDir);

diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs
@@ -307,14 +307,17 @@ public override object ActivateJob(Type jobType)
                 }
                 if (jobType == typeof(PostprocessBuildJob))
                 {
+                    var options = Substitute.For<IOptionsMonitor<BuildJobOptions>>();
+                    options.CurrentValue.Returns(new BuildJobOptions());
                     return new PostprocessBuildJob(
                         _env.PlatformService,
                         _env.Engines,
                         _env._lockFactory,
                         new MemoryDataAccessContext(),
                         _env.BuildJobService,
                         Substitute.For<ILogger<PostprocessBuildJob>>(),
-                        _env.SharedFileService
+                        _env.SharedFileService,
+                        options
                     );
                 }
                 return base.ActivateJob(jobType);

diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs
@@ -702,8 +702,10 @@ public override object ActivateJob(Type jobType)
                 }
                 if (jobType == typeof(SmtTransferPostprocessBuildJob))
                 {
-                    var options = Substitute.For<IOptionsMonitor<SmtTransferEngineOptions>>();
-                    options.CurrentValue.Returns(new SmtTransferEngineOptions());
+                    var engineOptions = Substitute.For<IOptionsMonitor<SmtTransferEngineOptions>>();
+                    engineOptions.CurrentValue.Returns(new SmtTransferEngineOptions());
+                    var buildOptions = Substitute.For<IOptionsMonitor<BuildJobOptions>>();
+                    buildOptions.CurrentValue.Returns(new BuildJobOptions());
                     return new SmtTransferPostprocessBuildJob(
                         _env.PlatformService,
                         _env.Engines,
@@ -715,7 +717,8 @@ public override object ActivateJob(Type jobType)
                         _env.TrainSegmentPairs,
                         _env.SmtModelFactory,
                         _env._truecaserFactory,
-                        options
+                        buildOptions,
+                        engineOptions
                     );
                 }
                 if (jobType == typeof(SmtTransferTrainBuildJob))