Add ability to use localizations for key terms (#454)

* Add ability to use localizations for key terms * Use language code from settings file * Adjust test to new terms inclusion
sillsdev · Aug 22, 2024 · 661eb1a · 661eb1a
1 parent 8732b40
commit 661eb1a
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 18 deletions.
diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj
@@ -36,9 +36,9 @@
 		<PackageReference Include="Hangfire.Mongo" Version="1.9.10" />
 		<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.16" />
 		<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.14" />
-		<PackageReference Include="SIL.Machine" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
-		<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
-		<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
+		<PackageReference Include="SIL.Machine" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+		<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
+		<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
 		<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
 		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 	</ItemGroup>
@@ -57,4 +57,4 @@
 		<ZipDirectory SourceDirectory="data\thot-new-model" DestinationFile="thot-new-model.zip" Overwrite="true" />
 	</Target>
 
-</Project>
+</Project>
diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs
@@ -50,13 +50,19 @@ protected override async Task DoWorkAsync(
         CancellationToken cancellationToken
     )
     {
+        TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken);
+        if (engine is null)
+            throw new OperationCanceledException($"Engine {engineId} does not exist.  Build canceled.");
+
+        bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang);
+        bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang);
+
         (int trainCount, int pretranslateCount) = await WriteDataFilesAsync(
             buildId,
             data,
             buildOptions,
             cancellationToken
         );
-
         // Log summary of build data
         JsonObject buildPreprocessSummary =
             new()
@@ -65,16 +71,10 @@ CancellationToken cancellationToken
                 { "EngineId", engineId },
                 { "BuildId", buildId },
                 { "NumTrainRows", trainCount },
-                { "NumPretranslateRows", pretranslateCount }
+                { "NumPretranslateRows", pretranslateCount },
+                { "SourceLanguageResolved", srcLang },
+                { "TargetLanguageResolved", trgLang }
             };
-        TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken);
-        if (engine is null)
-            throw new OperationCanceledException($"Engine {engineId} does not exist.  Build canceled.");
-
-        bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang);
-        buildPreprocessSummary.Add("SourceLanguageResolved", srcLang);
-        bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang);
-        buildPreprocessSummary.Add("TargetLanguageResolved", trgLang);
         Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString());
 
         if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel))

diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
@@ -104,7 +104,7 @@ public async Task RunAsync_EnableKeyTerms()
             Assert.That(src1Count, Is.EqualTo(0));
             Assert.That(src2Count, Is.EqualTo(0));
             Assert.That(trgCount, Is.EqualTo(0));
-            Assert.That(termCount, Is.EqualTo(1));
+            Assert.That(termCount, Is.EqualTo(5726));
         });
     }
 

diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj
@@ -19,7 +19,7 @@
 		<PackageReference Include="Grpc.Core.Api" Version="2.52.0" />
 		<PackageReference Include="Grpc.HealthCheck" Version="2.52.0" />
 		<PackageReference Include="Grpc.Net.ClientFactory" Version="2.52.0" />
-		<PackageReference Include="SIL.Machine" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+		<PackageReference Include="SIL.Machine" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 		<PackageReference Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
 	</ItemGroup>
 

diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs
@@ -619,7 +619,8 @@ public TestEnvironment()
                         fileNameSuffix: ".USFM",
                         biblicalTermsListType: "BiblicalTerms",
                         biblicalTermsProjectName: "",
-                        biblicalTermsFileName: "BiblicalTerms.xml"
+                        biblicalTermsFileName: "BiblicalTerms.xml",
+                        languageCode: "en"
                     )
                 );
 

diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs
@@ -402,7 +402,8 @@ private static ParatextProjectSettings CreateProjectSettings(string name)
                 fileNameSuffix: $"{name}.SFM",
                 biblicalTermsListType: "Major",
                 biblicalTermsProjectName: "",
-                biblicalTermsFileName: "BiblicalTerms.xml"
+                biblicalTermsFileName: "BiblicalTerms.xml",
+                languageCode: "en"
             );
         }
     }