Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add ability to use localizations for key terms #454

Merged
merged 17 commits into from
Aug 22, 2024
Merged

Add ability to use localizations for key terms #454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
0601c96
Add ability to use localizations for key terms
Enkidu93 Aug 15, 2024
0ab2ca7
Use language code from settings file
Enkidu93 Aug 16, 2024
c5841b6
Accommodate change in class name/structure
Enkidu93 Aug 16, 2024
7de44c4
Revert name of machine class
Enkidu93 Aug 20, 2024
20b0059
Revert to using filepath in constructor
Enkidu93 Aug 21, 2024
b998b8b
Add ability to use localizations for key terms
Enkidu93 Aug 15, 2024
4aaeedc
Use language code from settings file
Enkidu93 Aug 16, 2024
6f19694
Accommodate change in class name/structure
Enkidu93 Aug 16, 2024
66a0601
Revert name of machine class
Enkidu93 Aug 20, 2024
621bd57
Revert to using filepath in constructor
Enkidu93 Aug 21, 2024
2d9e750
Update Machine version
Enkidu93 Aug 22, 2024
a37a9aa
Merge branch 'key_terms' of https://github.com/sillsdev/serval into k…
Enkidu93 Aug 22, 2024
2e237c6
Merge branch 'key_terms' of https://github.com/sillsdev/serval into k…
Enkidu93 Aug 22, 2024
19ab4da
Fix project dependency mismerge
Enkidu93 Aug 22, 2024
286caea
Merge branch 'key_terms' of https://github.com/sillsdev/serval into k…
Enkidu93 Aug 22, 2024
c9d3784
Fix dependency mismerge
Enkidu93 Aug 22, 2024
e808ca0
Adjust test to new terms inclusion
Enkidu93 Aug 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@
<PackageReference Include="Hangfire.Mongo" Version="1.9.10" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.16" />
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.14" />
<PackageReference Include="SIL.Machine" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
Expand All @@ -57,4 +57,4 @@
<ZipDirectory SourceDirectory="data\thot-new-model" DestinationFile="thot-new-model.zip" Overwrite="true" />
</Target>

</Project>
</Project>
20 changes: 10 additions & 10 deletions src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,19 @@ protected override async Task DoWorkAsync(
CancellationToken cancellationToken
)
{
TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken);
if (engine is null)
throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled.");

bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang);
bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang);

(int trainCount, int pretranslateCount) = await WriteDataFilesAsync(
buildId,
data,
buildOptions,
cancellationToken
);

// Log summary of build data
JsonObject buildPreprocessSummary =
new()
Expand All @@ -65,16 +71,10 @@ CancellationToken cancellationToken
{ "EngineId", engineId },
{ "BuildId", buildId },
{ "NumTrainRows", trainCount },
{ "NumPretranslateRows", pretranslateCount }
{ "NumPretranslateRows", pretranslateCount },
{ "SourceLanguageResolved", srcLang },
{ "TargetLanguageResolved", trgLang }
};
TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken);
if (engine is null)
throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled.");

bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang);
buildPreprocessSummary.Add("SourceLanguageResolved", srcLang);
bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang);
buildPreprocessSummary.Add("TargetLanguageResolved", trgLang);
Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString());

if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel))
Expand Down
2 changes: 1 addition & 1 deletion src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public async Task RunAsync_EnableKeyTerms()
Assert.That(src1Count, Is.EqualTo(0));
Assert.That(src2Count, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(0));
Assert.That(termCount, Is.EqualTo(1));
Assert.That(termCount, Is.EqualTo(5726));
});
}

Expand Down
2 changes: 1 addition & 1 deletion src/Serval/src/Serval.Shared/Serval.Shared.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<PackageReference Include="Grpc.Core.Api" Version="2.52.0" />
<PackageReference Include="Grpc.HealthCheck" Version="2.52.0" />
<PackageReference Include="Grpc.Net.ClientFactory" Version="2.52.0" />
<PackageReference Include="SIL.Machine" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.2.6" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
</ItemGroup>

Expand Down
3 changes: 2 additions & 1 deletion src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -619,7 +619,8 @@ public TestEnvironment()
fileNameSuffix: ".USFM",
biblicalTermsListType: "BiblicalTerms",
biblicalTermsProjectName: "",
biblicalTermsFileName: "BiblicalTerms.xml"
biblicalTermsFileName: "BiblicalTerms.xml",
languageCode: "en"
)
);

Expand Down
3 changes: 2 additions & 1 deletion src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,8 @@ private static ParatextProjectSettings CreateProjectSettings(string name)
fileNameSuffix: $"{name}.SFM",
biblicalTermsListType: "Major",
biblicalTermsProjectName: "",
biblicalTermsFileName: "BiblicalTerms.xml"
biblicalTermsFileName: "BiblicalTerms.xml",
languageCode: "en"
);
}
}
Expand Down
Loading