[wasm] Enable ICU sharding

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_static.c

@@ -47,7 +47,7 @@ static void U_CALLCONV icu_trace_data(const void* context, int32_t fnNumber, int
 #ifdef __EMSCRIPTEN__
 #include <emscripten.h>
 
-static int32_t load_icu_data(void* pData);
+static int32_t load_icu_data(void* pData, int32_t type);
 
 EMSCRIPTEN_KEEPALIVE const char* mono_wasm_get_icudt_name(const char* culture);
 
@@ -56,11 +56,11 @@ EMSCRIPTEN_KEEPALIVE const char* mono_wasm_get_icudt_name(const char* culture)
     return GlobalizationNative_GetICUDTName(culture);
 }
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData);
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData, int32_t type);
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData)
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData, int32_t type)
 {
-    return load_icu_data(pData);
+    return load_icu_data(pData, type);
 }
 
 
@@ -77,14 +77,22 @@ void mono_wasm_link_icu_shim(void)
 
 #endif
 
-static int32_t load_icu_data(void* pData)
+static int32_t load_icu_data(void* pData, int32_t type)
 {
 
     UErrorCode status = 0;
-    udata_setCommonData(pData, &status);
+    if (type == 0) {
+        udata_setAppData(NULL, pData, &status);
+    }
+    if (type == 1) {
+        udata_setCommonData(pData, &status);
+    }
 
     if (U_FAILURE(status)) {
-        log_icu_error("udata_setCommonData", status);
+        if (type)
+            log_icu_error("udata_setCommonData", status);
+        else
+            log_icu_error("udata_setAppData", status);
         return 0;
     } else {
 
@@ -146,7 +154,7 @@ int32_t GlobalizationNative_LoadICUData(const char* path)
 
     fclose(fp);
 
-    if (load_icu_data(icu_data) == 0) {
+    if (load_icu_data(icu_data, strstr(path, "app") == NULL) == 0) {
         log_shim_error("ICU BAD EXIT %d.", ret);
         return ret;
     }

src/mono/sample/wasm/console/Makefile

@@ -1,4 +1,5 @@
 TOP=../../../../..
+CULTURE=en-us
 
 include ../wasm.mk
 
@@ -10,6 +11,10 @@ ifneq ($(V),)
 DOTNET_MONO_LOG_LEVEL=--setenv=MONO_LOG_LEVEL=debug
 endif
 
+ifneq ($(SHARD),)
+override MSBUILD_ARGS+=/p:EnableSharding=true /p:ICUDefaultCulture=$(CULTURE)
+endif
+
 PROJECT_NAME=Wasm.Console.Sample.csproj
 
 run: run-console

src/mono/sample/wasm/console/Program.cs

@@ -3,12 +3,15 @@
 
 using System;
 using System.Threading.Tasks;
+using System.Globalization;
 
 public class Test
 {
     public static async Task<int> Main(string[] args)
     {
         await Task.Delay(1);
+        CultureInfo cr = CultureInfo.GetCultureInfo("zh");
+        Console.WriteLine(cr.CompareInfo);
         Console.WriteLine("Hello World!");
         for (int i = 0; i < args.Length; i++) {
             Console.WriteLine($"args[{i}] = {args[i]}");

src/mono/wasm/Makefile

@@ -178,7 +178,7 @@ run-tests-%:
 	EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS)
 
 run-build-tests:
-	PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/tests/BuildWasmApps/Wasm.Build.Tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS)
+	PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/tests/BuildWasmApps/Wasm.Build.Tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS) /p:XUnitMethodName=Wasm.Build.Tests.InvariantGlobalizationTests.Invariant_WithSharding
 
 run-browser-tests-%:
 	PATH="$(GECKODRIVER):$(CHROMEDRIVER):$(PATH)" XHARNESS_COMMAND="test-browser --browser=$(XHARNESS_BROWSER)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS)

src/mono/wasm/build/WasmApp.targets

@@ -122,8 +122,6 @@
     <Error Condition="'$(WasmMainJSPath)' == ''" Text="%24(WasmMainJSPath) property needs to be set" />
 
     <PropertyGroup>
-      <WasmIcuDataFileName Condition="'$(InvariantGlobalization)' != 'true'">icudt.dat</WasmIcuDataFileName>
-
       <_HasDotnetWasm Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.wasm'">true</_HasDotnetWasm>
       <_HasDotnetJs Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js'">true</_HasDotnetJs>
     </PropertyGroup>
@@ -132,19 +130,24 @@
       <!-- If dotnet.{wasm,js} weren't added already (eg. AOT can add them), then add the default ones -->
       <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.wasm" Condition="'$(_HasDotnetWasm)' != 'true'" />
       <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.js" Condition="'$(_HasDotnetJs)' != 'true'" />
-
-      <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)$(WasmIcuDataFileName)" Condition="'$(InvariantGlobalization)' != 'true'" />
+
+      <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)icu_dictionary.js" Condition="'$(InvariantGlobalization)' != 'true'" />
+      <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)*.dat" Condition="'$(InvariantGlobalization)' != 'true'" />
       <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.timezones.blat" />
     </ItemGroup>
 
+    <ItemGroup Condition="'$(InvariantGlobalization)' != 'true'">
+      <WasmExtraConfig Include="enable_sharding" Value="true" Condition="'$(EnableSharding)' == 'true'" />
+      <WasmExtraConfig Include="default_culture" Value="&quot;$(ICUDefaultCulture)&quot;" Condition="'$(EnableSharding)' == 'true'" />
+    </ItemGroup>
+
     <WasmAppBuilder
       AppDir="$(WasmAppDir)"
       MainJS="$(WasmMainJSPath)"
       Assemblies="@(_WasmAssembliesInternal)"
       InvariantGlobalization="$(InvariantGlobalization)"
       SatelliteAssemblies="@(WasmSatelliteAssemblies)"
       FilesToIncludeInFileSystem="@(WasmFilesToIncludeInFileSystem)"
-      IcuDataFileName="$(WasmIcuDataFileName)"
       RemoteSources="@(WasmRemoteSources)"
       ExtraFilesToDeploy="@(WasmExtraFilesToDeploy)"
       ExtraConfig="@(WasmExtraConfig)"

src/mono/wasm/runtime-test.js

@@ -2,7 +2,6 @@
 //
 // Run runtime tests under a JS shell or a browser
 //
-
 //glue code to deal with the differences between chrome, ch, d8, jsc and sm.
 var is_browser = typeof window != "undefined";
 
@@ -205,7 +204,7 @@ function loadScript (url)
 }
 
 loadScript ("mono-config.js");
-
+loadScript ("icu_dictionary.js");
 var Module = {
 	mainScriptUrlOrBlob: "dotnet.js",
 
@@ -279,7 +278,14 @@ var Module = {
 				})
 			}
 		};
-
+		config.icu_dictionary = dictionary;
+		if (config.enable_sharding) {
+			if (config.default_culture != null) {
+				config.application_culture = config.default_culture;
+			} else {
+				config.application_culture = Intl.DateTimeFormat().resolvedOptions().locale;
+			}
+		}
 		MONO.mono_load_runtime_and_bcl_args (config);
 	},
 };

src/mono/wasm/runtime/library_mono.js

@@ -1143,6 +1143,59 @@ var MonoSupportLib = {
 			return res;
 		},
 
+		_get_shard_name: function (shards, culture) {
+			// Get shard name that culture belongs to
+			var parent_culture = culture.includes('-') ? culture.split('-')[0] : culture;
+			for (var name in shards) {
+				if (parent_culture.match(shards[name]))
+					return name
+			}
+		},
+
+		_get_list_of_icu_files: function (dictionary, culture, feature_shards=true, features="") {
+			var icu_files = []
+			if (dictionary === undefined)
+				return null;
+			if (culture === undefined || culture.length < 2 ) {
+				icu_files = [dictionary.packs.full];
+			} else {
+				var shard_name = this._get_shard_name(dictionary.shards, culture)
+				var packs = dictionary.packs;
+				var files = packs[shard_name];
+				if (!feature_shards) {
+					icu_files = files.full;
+				} else {
+					// Get base files first
+					for (var feature in packs[files.extends]) {
+						icu_files.push(...packs[files.extends][feature]);
+					}
+					// Adding shard specific core files such as collation and locales
+					icu_files.push(...files["core"])
+
+					//	Add any additional features
+					if (features != "") {
+						features = features.split(",");
+						features.forEach(feat => {
+							icu_files.push(...files[feat])
+						})
+					}
+				}
+			}
+			icu_assets = [];
+			icu_files.forEach(file => {
+				var type = "common";
+				// if (file.includes("locales"))
+				// 	type = "app";
+				icu_assets.push({
+									"behavior": "icu",
+									"name": file,
+									"load_remote": false,
+									"data_type": type
+								})
+				});
+			return icu_assets;
+		},
+
 		mono_wasm_get_details: function (objectId, args={}) {
 			let id = this._parse_object_id (objectId, true);
 
@@ -1620,7 +1673,7 @@ var MonoSupportLib = {
 				}
 			}
 			else if (asset.behavior === "icu") {
-				if (this.mono_wasm_load_icu_data (offset))
+				if (this.mono_wasm_load_icu_data (offset, +(asset.data_type == "common")))
 					ctx.num_icu_assets_loaded_successfully += 1;
 				else
 					console.error ("Error loading ICU asset", asset.name);
@@ -1699,6 +1752,7 @@ var MonoSupportLib = {
 		//      "icu": load ICU globalization data from any runtime assets with behavior "icu".
 		//      "invariant": operate in invariant globalization mode.
 		//      "auto" (default): if "icu" behavior assets are present, use ICU, otherwise invariant.
+		//		application_culture: (optional) current browser culture
 		//    diagnostic_tracing: (optional) enables diagnostic log messages during startup
 		mono_load_runtime_and_bcl_args: function (args) {
 			try {
@@ -1722,9 +1776,9 @@ var MonoSupportLib = {
 
 		// @offset must be the address of an ICU data archive in the native heap.
 		// returns true on success.
-		mono_wasm_load_icu_data: function (offset) {
-			var fn = Module.cwrap ('mono_wasm_load_icu_data', 'number', ['number']);
-			var ok = (fn (offset)) === 1;
+		mono_wasm_load_icu_data: function (offset, type) {
+			var fn = Module.cwrap ('mono_wasm_load_icu_data', 'number', ['number', 'number']);
+			var ok = (fn (offset, type)) === 1;
 			if (ok)
 				this.num_icu_assets_loaded_successfully++;
 			return ok;
@@ -1790,7 +1844,10 @@ var MonoSupportLib = {
 				throw new Error ("Invalid args (runtime_asset_sources was replaced by remote_sources)");
 			if (!args.loaded_cb)
 				throw new Error ("loaded_cb not provided");
-
+
+			var icu_files = this._get_list_of_icu_files(args.icu_dictionary, args.application_culture);
+			if (icu_files != null)
+				args.assets = args.assets.concat(icu_files);
 			var ctx = {
 				tracing: args.diagnostic_tracing || false,
 				pending_count: args.assets.length,
@@ -1838,7 +1895,6 @@ var MonoSupportLib = {
 				var attemptNextSource;
 				var sourceIndex = 0;
 				var sourcesList = asset.load_remote ? args.remote_sources : [""];
-
 				var handleFetchResponse = function (response) {
 					if (!response.ok) {
 						try {

src/mono/wasm/wasm.proj

@@ -197,6 +197,7 @@
       <MonoIncludeDir>$(MonoArtifactsPath)include/mono-2.0</MonoIncludeDir>
       <SystemNativeDir>$(RepoRoot)src\libraries\Native\Unix\System.Native</SystemNativeDir>
     </PropertyGroup>
+    <Import Project="$(IcuLibDir)ICU.DataFiles.props" Condition="'$(InvariantGlobalization)' != 'true'" />
     <ItemGroup>
       <ICULibNativeFiles Include="$(ICULibDir)/libicuuc.a;
                                   $(ICULibDir)/libicui18n.a" />

src/tasks/WasmAppBuilder/WasmAppBuilder.cs

@@ -41,7 +41,6 @@ public class WasmAppBuilder : Task
 
     // full list of ICU data files we produce can be found here:
     // https://github.com/dotnet/icu/tree/maint/maint-67/icu-filters
-    public string? IcuDataFileName { get; set; }
 
     public int DebugLevel { get; set; }
     public ITaskItem[]? SatelliteAssemblies { get; set; }
@@ -119,14 +118,22 @@ private class IcuData : AssetEntry
         public IcuData(string name) : base(name, "icu") {}
         [JsonPropertyName("load_remote")]
         public bool LoadRemote { get; set; }
+        [JsonPropertyName("data_type")]
+        public string? DataType { get; set; }
     }
 
     public override bool Execute ()
     {
-        if (!File.Exists(MainJS))
-            throw new ArgumentException($"File MainJS='{MainJS}' doesn't exist.");
-        if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDataFileName))
-            throw new ArgumentException("IcuDataFileName property shouldn't be empty if InvariantGlobalization=false");
+        if (!File.Exists(MainJS)) {
+            Log.LogError($"File MainJS='{MainJS}' doesn't exist.");
+            return false;
+        }
+
+        // if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDictionary))
+        // {
+        //     Log.LogError("IcuDictionary property shouldn't be empty if InvariantGlobalization=false");
+        //     return false;
+        // }
 
         if (Assemblies?.Length == 0)
         {
@@ -225,9 +232,6 @@ public override bool Execute ()
             }
         }
 
-        if (!InvariantGlobalization)
-            config.Assets.Add(new IcuData(IcuDataFileName!) { LoadRemote = RemoteSources?.Length > 0 });
-
         config.Assets.Add(new VfsEntry ("dotnet.timezones.blat") { VirtualPath = "/usr/share/zoneinfo/"});
 
         if (RemoteSources?.Length > 0)