Add SIL.ServiceToolkit library (#456)

sillsdev · Aug 19, 2024 · d9e928d · d9e928d
1 parent 0cbf32e
commit d9e928d
Show file tree

Hide file tree

Showing 19 changed files with 223 additions and 179 deletions.
diff --git a/Serval.sln b/Serval.sln
@@ -78,6 +78,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.JobServer",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared.Tests", "src\Machine\test\Serval.Machine.Shared.Tests\Serval.Machine.Shared.Tests.csproj", "{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ServiceToolkit", "ServiceToolkit", "{EA69B41C-49EF-4017-A687-44B9DF37FF98}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C3A14577-A654-4604-818C-4E683DD45A51}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit", "src\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj", "{0E40F959-C641-40A2-9750-B17A4F9F9E55}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -164,6 +170,10 @@ Global
 		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -196,6 +206,8 @@ Global
 		{C02494FB-663E-4430-9F2D-41F1A740B271} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}
 		{BC766753-E560-4ADF-9923-C7A96076EA47} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}
 		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB}
+		{C3A14577-A654-4604-818C-4E683DD45A51} = {EA69B41C-49EF-4017-A687-44B9DF37FF98}
+		{0E40F959-C641-40A2-9750-B17A4F9F9E55} = {C3A14577-A654-4604-818C-4E683DD45A51}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370}

diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs
@@ -192,7 +192,7 @@ public static IMachineBuilder AddMongoHangfireJobClient(
                 .UseMongoStorage(connectionString, GetMongoStorageOptions())
                 .UseFilter(new AutomaticRetryAttribute { Attempts = 0 })
         );
-        builder.Services.AddHealthChecks().AddCheck<HangfireHealthCheck>(name: "Hangfire");
+        builder.Services.AddHealthChecks().AddHangfire();
         return builder;
     }
 

diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj
@@ -32,15 +32,14 @@
 		<PackageReference Include="CommunityToolkit.HighPerformance" Version="8.2.2" />
 		<PackageReference Include="Grpc.AspNetCore" Version="2.57.0" />
 		<PackageReference Include="Grpc.AspNetCore.HealthChecks" Version="2.57.0" />
-		<PackageReference Include="HangFire" Version="1.8.5" />
+		<PackageReference Include="HangFire" Version="1.8.14" />
 		<PackageReference Include="Hangfire.Mongo" Version="1.9.10" />
 		<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.16" />
 		<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.14" />
-		<PackageReference Include="Python.Included" Version="3.11.4" />
 		<PackageReference Include="SIL.Machine" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 		<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
 		<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.2.5" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
-		<PackageReference Include="SIL.WritingSystems" Version="12.0.1" />
+		<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
 		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 	</ItemGroup>
 
@@ -50,6 +49,7 @@
 		<ProjectReference Include="..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj" Condition="Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 		<ProjectReference Include="..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj" Condition="Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
 		<ProjectReference Include="..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj" Condition="Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
+		<ProjectReference Include="..\..\..\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj" />
 		<EmbeddedResource Include="data\flores200languages.csv" />
 	</ItemGroup>
 

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs b/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs
@@ -2,113 +2,28 @@
 
 public class LanguageTagService : ILanguageTagService
 {
-    private static readonly Dictionary<string, string> StandardLanguages =
-        new()
-        {
-            { "ar", "arb" },
-            { "ms", "zsm" },
-            { "lv", "lvs" },
-            { "ne", "npi" },
-            { "sw", "swh" },
-            { "cmn", "zh" }
-        };
-
-    private static readonly Dictionary<string, string> StandardScripts = new() { { "Kore", "Hang" } };
-
-    private readonly Dictionary<string, string> _defaultScripts;
-
-    private readonly Dictionary<string, string> _flores200Languages;
-
-    private static readonly Regex LangTagPattern =
-        new("(?'language'[a-zA-Z]{2,8})([_-](?'script'[a-zA-Z]{4}))?", RegexOptions.ExplicitCapture);
-
-    public LanguageTagService()
-    {
-        // initialize SLDR language tags to retrieve latest langtags.json file
-        _defaultScripts = InitializeDefaultScripts();
-        _flores200Languages = InitializeFlores200Languages();
-    }
-
-    protected virtual void InitializeSldrLanguageTags()
-    {
-        Sldr.InitializeLanguageTags();
-    }
-
-    private Dictionary<string, string> InitializeDefaultScripts()
-    {
-        InitializeSldrLanguageTags();
-        var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json");
-        JsonNode? json;
-
-        if (!File.Exists(cachedAllTagsPath))
-        {
-            using HttpClient client = new();
-            using HttpResponseMessage response = client.Send(
-                new HttpRequestMessage(
-                    HttpMethod.Get,
-                    "https://raw.githubusercontent.com/silnrsi/langtags/master/pub/langtags.json"
-                )
-            );
-            response.EnsureSuccessStatusCode();
-            using Stream responseStream = response.Content.ReadAsStream();
-            using FileStream fileStream = new(cachedAllTagsPath, FileMode.Create);
-            responseStream.CopyTo(fileStream);
-        }
-        using FileStream stream = new(cachedAllTagsPath, FileMode.Open);
-        json = JsonNode.Parse(stream);
-
-        Dictionary<string, string> tempDefaultScripts = new();
-        foreach (JsonNode? entry in json!.AsArray())
-        {
-            if (entry is null)
-                continue;
-
-            var script = (string?)entry["script"];
-            if (script is null)
-                continue;
-
-            JsonNode? tags = entry["tags"];
-            if (tags is not null)
-            {
-                foreach (var t in tags.AsArray().Select(v => (string?)v))
-                {
-                    if (
-                        t is not null
-                        && IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _)
-                        && s is null
-                    )
-                    {
-                        tempDefaultScripts[t] = script;
-                    }
-                }
-            }
-
-            var tag = (string?)entry["tag"];
-            if (tag is not null)
-                tempDefaultScripts[tag] = script;
-        }
-        return tempDefaultScripts;
-    }
+    private readonly Dictionary<string, string> _flores200Languages = InitializeFlores200Languages();
+    private readonly LanguageTagParser _parser = new();
 
     private static Dictionary<string, string> InitializeFlores200Languages()
     {
-        var tempFlores200Languages = new Dictionary<string, string>();
+        Dictionary<string, string> flores200Languages = [];
         using var floresStream = Assembly
             .GetExecutingAssembly()
             .GetManifestResourceStream("Serval.Machine.Shared.data.flores200languages.csv");
         Debug.Assert(floresStream is not null);
-        var reader = new StreamReader(floresStream);
-        var firstLine = reader.ReadLine();
+        StreamReader reader = new(floresStream);
+        string? firstLine = reader.ReadLine();
         Debug.Assert(firstLine == "language, code");
         while (!reader.EndOfStream)
         {
             string? line = reader.ReadLine();
             if (line is null)
                 continue;
             string[] values = line.Split(',');
-            tempFlores200Languages[values[1].Trim()] = values[0].Trim();
+            flores200Languages[values[1].Trim()] = values[0].Trim();
         }
-        return tempFlores200Languages;
+        return flores200Languages;
     }
 
     /**
@@ -119,52 +34,10 @@ private static Dictionary<string, string> InitializeFlores200Languages()
      */
     public bool ConvertToFlores200Code(string languageTag, out string flores200Code)
     {
-        flores200Code = ResolveLanguageTag(languageTag);
-        return _flores200Languages.ContainsKey(flores200Code);
-    }
-
-    private string ResolveLanguageTag(string languageTag)
-    {
-        // Try to find a pattern of {language code}_{script}
-        Match langTagMatch = LangTagPattern.Match(languageTag);
-        if (!langTagMatch.Success)
-            return languageTag;
-        string parsedLanguage = langTagMatch.Groups["language"].Value;
-        string languageSubtag = parsedLanguage;
-        string iso639_3Code = parsedLanguage;
-
-        // Best attempt to convert language to a registered ISO 639-3 code
-        // Uses https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry for mapping
-
-        // If they gave us the ISO code, revert it to the 2 character code
-        if (StandardSubtags.TryGetLanguageFromIso3Code(languageSubtag, out LanguageSubtag tempSubtag))
-            languageSubtag = tempSubtag.Code;
-
-        // There are a few extra conversions not in SIL Writing Systems that we need to handle
-        if (StandardLanguages.TryGetValue(languageSubtag, out string? tempName))
-            languageSubtag = tempName;
-
-        if (StandardSubtags.RegisteredLanguages.TryGet(languageSubtag, out LanguageSubtag? languageSubtagObj))
-            iso639_3Code = languageSubtagObj.Iso3Code;
-
-        // Use default script unless there is one parsed out of the language tag
-        Group scriptGroup = langTagMatch.Groups["script"];
-        string? script = null;
-
-        if (scriptGroup.Success)
-            script = scriptGroup.Value;
-        else if (_defaultScripts.TryGetValue(languageTag, out string? tempScript2))
-            script = tempScript2;
-        else if (_defaultScripts.TryGetValue(languageSubtag, out string? tempScript))
-            script = tempScript;
-
-        // There are a few extra conversions not in SIL Writing Systems that we need to handle
-        if (script is not null && StandardScripts.TryGetValue(script, out string? tempScript3))
-            script = tempScript3;
-
-        if (script is not null)
-            return $"{iso639_3Code}_{script}";
+        if (_parser.TryParse(languageTag, out string? languageCode, out string? scriptCode))
+            flores200Code = $"{languageCode}_{scriptCode}";
         else
-            return languageTag;
+            flores200Code = languageTag;
+        return _flores200Languages.ContainsKey(flores200Code);
     }
 }
diff --git a/src/Machine/src/Serval.Machine.Shared/Usings.cs b/src/Machine/src/Serval.Machine.Shared/Usings.cs
@@ -16,7 +16,6 @@
 global using System.Text.Json;
 global using System.Text.Json.Nodes;
 global using System.Text.Json.Serialization;
-global using System.Text.RegularExpressions;
 global using Amazon;
 global using Amazon.Runtime;
 global using Amazon.S3;
@@ -56,4 +55,6 @@
 global using SIL.Machine.Translation.Thot;
 global using SIL.Machine.Utils;
 global using SIL.Scripture;
+global using SIL.ServiceToolkit.Services;
+global using SIL.ServiceToolkit.Utils;
 global using SIL.WritingSystems;
diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs
@@ -3,6 +3,13 @@
 [TestFixture]
 public class LanguageTagServiceTests
 {
+    [OneTimeSetUp]
+    public void OneTimeSetUp()
+    {
+        if (!Sldr.IsInitialized)
+            Sldr.Initialize();
+    }
+
     [Test]
     [TestCase("es", "spa_Latn", Description = "Iso639_1Code")]
     [TestCase("hne", "hne_Deva", Description = "Iso639_3Code")]
@@ -21,8 +28,6 @@ public class LanguageTagServiceTests
     [TestCase("kor_Kore", "kor_Hang", Description = "KoreanScriptCorrection")]
     public void ConvertToFlores200CodeTest(string language, string internalCodeTruth)
     {
-        if (!Sldr.IsInitialized)
-            Sldr.Initialize();
         new LanguageTagService().ConvertToFlores200Code(language, out string internalCode);
         Assert.That(internalCode, Is.EqualTo(internalCodeTruth));
     }
@@ -34,36 +39,11 @@ public void ConvertToFlores200CodeTest(string language, string internalCodeTruth
     [TestCase("xyz", "xyz", false)]
     public void GetLanguageInfoAsync(string languageCode, string? resolvedLanguageCode, bool nativeLanguageSupport)
     {
-        if (!Sldr.IsInitialized)
-            Sldr.Initialize();
         bool isNative = new LanguageTagService().ConvertToFlores200Code(languageCode, out string internalCode);
         Assert.Multiple(() =>
         {
             Assert.That(internalCode, Is.EqualTo(resolvedLanguageCode));
             Assert.That(isNative, Is.EqualTo(nativeLanguageSupport));
         });
     }
-
-    public class TestLanguageTagService : LanguageTagService
-    {
-        // Don't call Sldr initialize to call
-        protected override void InitializeSldrLanguageTags()
-        {
-            // remove langtags.json to force download
-            var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json");
-            if (File.Exists(cachedAllTagsPath))
-                File.Delete(cachedAllTagsPath);
-            Directory.CreateDirectory(Sldr.SldrCachePath);
-        }
-    }
-
-    [Test]
-    public void BackupLangtagsJsonTest()
-    {
-        if (!Sldr.IsInitialized)
-            Sldr.Initialize();
-        var service = new TestLanguageTagService();
-        service.ConvertToFlores200Code("en", out string internalCode);
-        Assert.That(internalCode, Is.EqualTo("eng_Latn"));
-    }
 }
diff --git a/src/Serval/src/Serval.ApiServer/Usings.cs b/src/Serval/src/Serval.ApiServer/Usings.cs
@@ -12,7 +12,6 @@
 global using Microsoft.AspNetCore.Authorization;
 global using Microsoft.AspNetCore.Mvc;
 global using Microsoft.AspNetCore.OutputCaching;
-global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.Extensions.Diagnostics.HealthChecks;
 global using Microsoft.IdentityModel.Tokens;
 global using NJsonSchema;

diff --git a/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs
@@ -19,7 +19,7 @@ public void GetZipParatextProjectTextUpdater()
         Assert.That(
             updater.UpdateUsfm("MAT", [], preferExistingText: true),
             Is.EqualTo(
-                $@"\id MAT - PROJ
+                    $@"\id MAT - PROJ
 \h {Canon.BookIdToEnglishName("MAT")}
 \c 1
 \p
@@ -29,8 +29,9 @@ public void GetZipParatextProjectTextUpdater()
 \p
 \v 1 Chapter two, verse one.
 \v 2 Chapter two, verse two.
-".Replace("\n", "\r\n")
-            )
+"
+                )
+                .IgnoreLineEndings()
         );
     }
 

diff --git a/src/Serval/test/Serval.Shared.Tests/Usings.cs b/src/Serval/test/Serval.Shared.Tests/Usings.cs
@@ -4,6 +4,8 @@
 global using Microsoft.Extensions.Options;
 global using NSubstitute;
 global using NUnit.Framework;
+global using NUnit.Framework.Constraints;
 global using Serval.Shared.Configuration;
+global using Serval.Shared.Utils;
 global using SIL.Machine.Corpora;
 global using SIL.Scripture;
diff --git a/...slation.Tests/Services/NUnitExtensions.cs → ...val.Shared.Tests/Utils/NUnitExtensions.cs b/...slation.Tests/Services/NUnitExtensions.cs → ...val.Shared.Tests/Utils/NUnitExtensions.cs
@@ -1,4 +1,4 @@
-namespace Serval.Translation.Services;
+namespace Serval.Shared.Utils;
 
 public static class NUnitExtensions
 {

diff --git a/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj b/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj
@@ -33,6 +33,7 @@
 
 	<ItemGroup>
 		<ProjectReference Include="..\..\src\Serval.Translation\Serval.Translation.csproj" />
+		<ProjectReference Include="..\Serval.Shared.Tests\Serval.Shared.Tests.csproj" />
 	</ItemGroup>
 
 </Project>
diff --git a/src/Serval/test/Serval.Translation.Tests/Usings.cs b/src/Serval/test/Serval.Translation.Tests/Usings.cs
@@ -6,7 +6,6 @@
 global using Microsoft.Extensions.Options;
 global using NSubstitute;
 global using NUnit.Framework;
-global using NUnit.Framework.Constraints;
 global using Serval.Shared.Configuration;
 global using Serval.Shared.Services;
 global using Serval.Shared.Utils;

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs
@@ -0,0 +1,12 @@
+using SIL.ServiceToolkit.Services;
+
+namespace Microsoft.Extensions.DependencyInjection;
+
+public static class IHealthChecksBuilderExtensions
+{
+    public static IHealthChecksBuilder AddHangfire(this IHealthChecksBuilder builder, string name = "Hangfire")
+    {
+        builder.AddCheck<HangfireHealthCheck>(name);
+        return builder;
+    }
+}