diff --git a/src/Microsoft.DocAsCode.Build.Engine/ManifestProcessor.cs b/src/Microsoft.DocAsCode.Build.Engine/ManifestProcessor.cs index 4ea08f40e29..0e2f9b0e7d3 100644 --- a/src/Microsoft.DocAsCode.Build.Engine/ManifestProcessor.cs +++ b/src/Microsoft.DocAsCode.Build.Engine/ManifestProcessor.cs @@ -6,26 +6,29 @@ namespace Microsoft.DocAsCode.Build.Engine using System; using System.Collections.Concurrent; using System.Collections.Generic; - using System.Collections.Immutable; - using System.IO; using System.Linq; - using Newtonsoft.Json.Linq; using Microsoft.DocAsCode.Common; using Microsoft.DocAsCode.Plugins; internal class ManifestProcessor { - private List _manifestWithContext; - private DocumentBuildContext _context; - private TemplateProcessor _templateProcessor; + private readonly List _manifestWithContext; + private readonly DocumentBuildContext _context; + private readonly TemplateProcessor _templateProcessor; + private readonly IDictionary _globalMetadata; public ManifestProcessor(List manifestWithContext, DocumentBuildContext context, TemplateProcessor templateProcessor) { _context = context ?? throw new ArgumentNullException(nameof(context)); _templateProcessor = templateProcessor ?? throw new ArgumentNullException(nameof(templateProcessor)); _manifestWithContext = manifestWithContext ?? throw new ArgumentNullException(nameof(manifestWithContext)); + + // E.g. we can set TOC model to be globally shared by every data model + // Make sure it is single thread + _globalMetadata = _templateProcessor.Tokens?.ToDictionary(pair => pair.Key, pair => (object)pair.Value) + ?? new Dictionary(); } public void Process() @@ -35,16 +38,17 @@ public void Process() UpdateContext(); } - // Run getOptions from Template - using (new LoggerPhaseScope("FeedOptions", LogLevel.Verbose)) + // Afterwards, m.Item.Model.Content is always IDictionary + using (new LoggerPhaseScope("NormalizeToObject", LogLevel.Verbose)) { - FeedOptions(); + NormalizeToObject(); } + // Run getOptions from Template and feed options back to context // Template can feed back xref map, actually, the anchor # location can only be determined in template - using (new LoggerPhaseScope("FeedXRefMap", LogLevel.Verbose)) + using (new LoggerPhaseScope("FeedOptions", LogLevel.Verbose)) { - FeedXRefMap(); + FeedOptions(); } using (new LoggerPhaseScope("UpdateHref", LogLevel.Verbose)) @@ -71,28 +75,42 @@ private void UpdateContext() _context.ResolveExternalXRefSpec(); } - private void FeedOptions() + private void NormalizeToObject() { - Logger.LogVerbose("Feeding options from template..."); + Logger.LogVerbose("Normalizing all the object to week type"); + _manifestWithContext.RunAll(m => { - if (m.TemplateBundle == null) + if (m.FileModel.Type == DocumentType.Resource) { return; } - using (new LoggerFileScope(m.FileModel.LocalPathFromRoot)) { - Logger.LogDiagnostic($"Feed options from template for {m.Item.DocumentType}..."); - m.Options = m.TemplateBundle.GetOptions(m.Item, _context); + var model = m.Item.Model.Content; + // Change file model to weak type + // Go through the convert even if it is IDictionary as the inner object might be of strong type + var modelAsObject = ConvertToObjectHelper.ConvertStrongTypeToObject(model); + if (modelAsObject is IDictionary) + { + m.Item.Model.Content = modelAsObject; + } + else + { + Logger.LogWarning("Input model is not an Object model, it will be wrapped into an Object model. Please use --exportRawModel to view the wrapped model"); + m.Item.Model.Content = new Dictionary + { + ["model"] = modelAsObject + }; + } } }, _context.MaxParallelism); } - private void FeedXRefMap() + private void FeedOptions() { - Logger.LogVerbose("Feeding xref map..."); + Logger.LogVerbose("Feeding options from template..."); _manifestWithContext.RunAll(m => { if (m.TemplateBundle == null) @@ -102,11 +120,14 @@ private void FeedXRefMap() using (new LoggerFileScope(m.FileModel.LocalPathFromRoot)) { - Logger.LogDiagnostic($"Feed xref map from template for {m.Item.DocumentType}..."); - if (m.Options.Bookmarks == null) return; - foreach (var pair in m.Options.Bookmarks) + Logger.LogDiagnostic($"Feed options from template for {m.Item.DocumentType}..."); + m.Options = m.TemplateBundle.GetOptions(m.Item, _context); + if (m.Options?.Bookmarks != null) { - _context.RegisterInternalXrefSpecBookmark(pair.Key, pair.Value); + foreach (var pair in m.Options.Bookmarks) + { + _context.RegisterInternalXrefSpecBookmark(pair.Key, pair.Value); + } } } }, @@ -137,6 +158,8 @@ private void ApplySystemMetadata() // Add system attributes var systemMetadataGenerator = new SystemMetadataGenerator(_context); + var sharedObjects = new ConcurrentDictionary(); + _manifestWithContext.RunAll(m => { if (m.FileModel.Type == DocumentType.Resource) @@ -149,76 +172,36 @@ private void ApplySystemMetadata() // TODO: use weak type for system attributes from the beginning var systemAttrs = systemMetadataGenerator.Generate(m.Item); - var metadata = (JObject)ConvertToObjectHelper.ConvertStrongTypeToJObject(systemAttrs); - // Change file model to weak type - var model = m.Item.Model.Content; - var modelAsObject = (JToken)ConvertToObjectHelper.ConvertStrongTypeToJObject(model); - if (modelAsObject is JObject) + var metadata = (IDictionary)ConvertToObjectHelper.ConvertStrongTypeToObject(systemAttrs); + + var model = (IDictionary)m.Item.Model.Content; + + foreach (var pair in metadata) { - foreach (var pair in (JObject)modelAsObject) + if (!model.ContainsKey(pair.Key)) { - // Overwrites the existing system metadata if the same key is defined in document model - metadata[pair.Key] = pair.Value; + model[pair.Key] = pair.Value; } } - else - { - Logger.LogWarning("Input model is not an Object model, it will be wrapped into an Object model. Please use --exportRawModel to view the wrapped model"); - metadata["model"] = modelAsObject; - } - - // Append system metadata to model - m.Item.Model.Serializer = null; - m.Item.Model.Content = metadata; - } - }, - _context.MaxParallelism); - } - - private IDictionary FeedGlobalVariables() - { - Logger.LogVerbose("Feeding global variables from template..."); - // E.g. we can set TOC model to be globally shared by every data model - // Make sure it is single thread - var initialGlobalVariables = _templateProcessor.Tokens; - IDictionary metadata = initialGlobalVariables == null ? - new Dictionary() : - initialGlobalVariables.ToDictionary(pair => pair.Key, pair => (object)pair.Value); - var sharedObjects = new ConcurrentDictionary(); - _manifestWithContext.RunAll(m => - { - if (m.TemplateBundle == null) - { - return; - } - - using (new LoggerFileScope(m.FileModel.LocalPathFromRoot)) - { Logger.LogDiagnostic($"Load shared model from template for {m.Item.DocumentType}..."); - if (m.Options.IsShared) + if (m.Options?.IsShared == true) { - sharedObjects[m.Item.Key] = m.Item.Model.Content; + // Take a snapshot of current model as shared object + sharedObjects[m.Item.Key] = new Dictionary(model); } + } }, _context.MaxParallelism); - metadata["_shared"] = sharedObjects; - return metadata; + _globalMetadata["_shared"] = sharedObjects; } private List ProcessTemplate() { - // Register global variables after href are all updated - IDictionary globalVariables; - using (new LoggerPhaseScope("FeedGlobalVariables", LogLevel.Verbose)) - { - globalVariables = FeedGlobalVariables(); - } - // processor to add global variable to the model - return _templateProcessor.Process(_manifestWithContext.Select(s => s.Item).ToList(), _context.ApplyTemplateSettings, globalVariables); + return _templateProcessor.Process(_manifestWithContext.Select(s => s.Item).ToList(), _context.ApplyTemplateSettings, _globalMetadata); } #endregion diff --git a/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/JintProcessorHelper.cs b/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/JintProcessorHelper.cs index 8a8ab383d2c..f7f1b79ae4c 100644 --- a/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/JintProcessorHelper.cs +++ b/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/JintProcessorHelper.cs @@ -3,115 +3,37 @@ namespace Microsoft.DocAsCode.Build.Engine { - using System; - using System.IO; - using System.Threading; + using System.Collections.Generic; using Jint; - using Microsoft.DocAsCode.Common; - - using Newtonsoft.Json; - using Newtonsoft.Json.Linq; - public static class JintProcessorHelper { private static readonly Engine DefaultEngine = new Engine(); - private static readonly ThreadLocal _toJsValueSerializer = new ThreadLocal( - () => - { - var jsonSerializer = new JsonSerializer(); - jsonSerializer.NullValueHandling = NullValueHandling.Ignore; - jsonSerializer.ReferenceLoopHandling = ReferenceLoopHandling.Serialize; - jsonSerializer.Converters.Add(new JObjectToJsValueConverter()); - return jsonSerializer; - }); - - public static Jint.Native.JsValue ConvertStrongTypeToJsValue(object raw) - { - var token = raw as JToken; - if (token != null) - { - return ConvertJTokenToJsValue(token); - } - - using (MemoryStream ms = new MemoryStream()) - { - using (StreamWriter sw = new StreamWriter(ms)) - { - JsonUtility.Serialize(sw, raw); - sw.Flush(); - ms.Seek(0, SeekOrigin.Begin); - using (StreamReader sr = new StreamReader(ms)) - { - return JsonUtility.Deserialize(sr, _toJsValueSerializer.Value); - } - } - } - } - public static Jint.Native.JsValue ConvertJTokenToJsValue(JToken raw) + public static Jint.Native.JsValue ConvertObjectToJsValue(object raw) { - var jArray = raw as JArray; - if (jArray != null) - { - var jsArray = DefaultEngine.Array.Construct(Jint.Runtime.Arguments.Empty); - foreach (var item in jArray) - { - DefaultEngine.Array.PrototypeObject.Push(jsArray, Jint.Runtime.Arguments.From(ConvertJTokenToJsValue(item))); - } - return jsArray; - } - var jObject = raw as JObject; - if (jObject != null) + if (raw is IDictionary idict) { var jsObject = DefaultEngine.Object.Construct(Jint.Runtime.Arguments.Empty); - foreach (var pair in jObject) + foreach (var pair in idict) { - jsObject.Put(pair.Key, ConvertJTokenToJsValue(pair.Value), true); + jsObject.Put(pair.Key, ConvertObjectToJsValue(pair.Value), true); } return jsObject; } - - var jValue = raw as JValue; - if (jValue != null) - { - return Jint.Native.JsValue.FromObject(DefaultEngine, jValue.Value); - } - - return Jint.Native.JsValue.FromObject(DefaultEngine, raw); - } - - private sealed class JObjectToJsValueConverter : JsonConverter - { - public override bool CanConvert(Type objectType) - { - return objectType == typeof(Jint.Native.JsValue); - } - - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + else if (raw is IList list) { - if (reader.TokenType == JsonToken.StartArray) - { - var jArray = JArray.Load(reader); - return ConvertJTokenToJsValue(jArray); - } - else if (reader.TokenType == JsonToken.StartObject) - { - var jObject = JObject.Load(reader); - var converted = ConvertJTokenToJsValue(jObject); - return converted; - } - else + var jsArray = DefaultEngine.Array.Construct(Jint.Runtime.Arguments.Empty); + foreach (var item in list) { - var jValue = JValue.Load(reader); - return ConvertJTokenToJsValue(jValue); + DefaultEngine.Array.PrototypeObject.Push(jsArray, Jint.Runtime.Arguments.From(ConvertObjectToJsValue(item))); } + return jsArray; } - - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + else { - throw new NotImplementedException(); + return Jint.Native.JsValue.FromObject(DefaultEngine, raw); } } } diff --git a/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/TemplateJintPreprocessor.cs b/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/TemplateJintPreprocessor.cs index 004fa9ff07d..b31899bb027 100644 --- a/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/TemplateJintPreprocessor.cs +++ b/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/TemplateJintPreprocessor.cs @@ -217,7 +217,7 @@ private static Func GetFunc(string funcName, ObjectInstance expo { return s => { - var model = JintProcessorHelper.ConvertStrongTypeToJsValue(s); + var model = JintProcessorHelper.ConvertObjectToJsValue(s); return func.Invoke(model).ToObject(); }; } diff --git a/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/TemplateModelTransformer.cs b/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/TemplateModelTransformer.cs index 59da0a780b3..c0a854cfb17 100644 --- a/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/TemplateModelTransformer.cs +++ b/src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/TemplateModelTransformer.cs @@ -46,8 +46,10 @@ internal ManifestItem Transform(InternalManifestItem item) { throw new ArgumentNullException("Content for item.Model should not be null!"); } + var model = ConvertObjectToDictionary(item.Model.Content); - model = AppendGlobalMetadata(model); + AppendGlobalMetadata(model); + if (_settings.Options.HasFlag(ApplyTemplateOptions.ExportRawModel)) { ExportModel(model, item.FileWithoutExtension, _settings.RawModelExportSettings); @@ -176,6 +178,8 @@ internal ManifestItem Transform(InternalManifestItem item) } } + item.Model = null; + LogInvalidXRefs(unresolvedXRefs); return manifestItem; @@ -237,11 +241,11 @@ private string GetLinkToPath(string fileName) } } - private IDictionary AppendGlobalMetadata(IDictionary model) + private void AppendGlobalMetadata(IDictionary model) { if (_globalVariables == null) { - return model; + return; } if (model.ContainsKey(GlobalVariableKey)) @@ -249,12 +253,7 @@ private IDictionary AppendGlobalMetadata(IDictionary(model) - { - [GlobalVariableKey] = _globalVariables - }; - return appended; + model[GlobalVariableKey] = new Dictionary(_globalVariables); } private static IDictionary ConvertObjectToDictionary(object model) diff --git a/src/Microsoft.DocAsCode.Build.TableOfContents/TocDocumentProcessorBase.cs b/src/Microsoft.DocAsCode.Build.TableOfContents/TocDocumentProcessorBase.cs index ea6c3595bc7..ecd6cffdf83 100644 --- a/src/Microsoft.DocAsCode.Build.TableOfContents/TocDocumentProcessorBase.cs +++ b/src/Microsoft.DocAsCode.Build.TableOfContents/TocDocumentProcessorBase.cs @@ -13,6 +13,8 @@ namespace Microsoft.DocAsCode.Build.TableOfContents using Microsoft.DocAsCode.DataContracts.Common; using Microsoft.DocAsCode.Plugins; + using Newtonsoft.Json; + /// /// Base document processor for table of contents. /// @@ -59,10 +61,11 @@ public override SaveResult Save(FileModel model) public override void UpdateHref(FileModel model, IDocumentBuildContext context) { - var toc = (TocItemViewModel)model.Content; + var toc = ConvertFromObject(model.Content); UpdateTocItemHref(toc, model, context); RegisterTocToContext(toc, model, context); + model.Content = ConvertToObject(toc); } #endregion @@ -77,6 +80,19 @@ public override void UpdateHref(FileModel model, IDocumentBuildContext context) #region Private methods + private TocItemViewModel ConvertFromObject(object model) + { + using (var jr = new ObjectJsonReader(model)) + { + return JsonUtility.DefaultSerializer.Value.Deserialize(jr); + } + } + + private object ConvertToObject(TocItemViewModel model) + { + return ConvertToObjectHelper.ConvertStrongTypeToObject(model); + } + private void UpdateTocItemHref(TocItemViewModel toc, FileModel model, IDocumentBuildContext context) { if (toc.IsHrefUpdated) return; @@ -87,9 +103,13 @@ private void UpdateTocItemHref(TocItemViewModel toc, FileModel model, IDocumentB RegisterTocMapToContext(toc, model, context); toc.Homepage = ResolveHref(toc.Homepage, toc.OriginalHomepage, model, context, nameof(toc.Homepage)); + toc.OriginalHomepage = null; toc.Href = ResolveHref(toc.Href, toc.OriginalHref, model, context, nameof(toc.Href)); + toc.OriginalHref = null; toc.TocHref = ResolveHref(toc.TocHref, toc.OriginalTocHref, model, context, nameof(toc.TocHref)); + toc.OriginalTocHref = null; toc.TopicHref = ResolveHref(toc.TopicHref, toc.OriginalTopicHref, model, context, nameof(toc.TopicHref)); + toc.OriginalTopicHref = null; if (toc.Items != null && toc.Items.Count > 0) { diff --git a/src/Microsoft.DocAsCode.DataContracts.Common/TocItemViewModel.cs b/src/Microsoft.DocAsCode.DataContracts.Common/TocItemViewModel.cs index 943a66101a2..7579f963cae 100644 --- a/src/Microsoft.DocAsCode.DataContracts.Common/TocItemViewModel.cs +++ b/src/Microsoft.DocAsCode.DataContracts.Common/TocItemViewModel.cs @@ -10,8 +10,8 @@ namespace Microsoft.DocAsCode.DataContracts.Common using Newtonsoft.Json; using YamlDotNet.Serialization; + using Microsoft.DocAsCode.Common; using Microsoft.DocAsCode.YamlSerialization; - using DocAsCode.Common; [Serializable] public class TocItemViewModel @@ -60,24 +60,24 @@ public string NameForVB [JsonProperty(Constants.PropertyName.Href)] public string Href { get; set; } - [YamlIgnore] - [JsonIgnore] + [YamlMember(Alias = "originalHref")] + [JsonProperty("originalHref")] public string OriginalHref { get; set; } [YamlMember(Alias = Constants.PropertyName.TocHref)] [JsonProperty(Constants.PropertyName.TocHref)] public string TocHref { get; set; } - [YamlIgnore] - [JsonIgnore] + [YamlMember(Alias = "originalTocHref")] + [JsonProperty("originalTocHref")] public string OriginalTocHref { get; set; } [YamlMember(Alias = Constants.PropertyName.TopicHref)] [JsonProperty(Constants.PropertyName.TopicHref)] public string TopicHref { get; set; } - [YamlIgnore] - [JsonIgnore] + [YamlMember(Alias = "originalTopicHref")] + [JsonProperty("originalTopicHref")] public string OriginalTopicHref { get; set; } [YamlIgnore] @@ -88,8 +88,8 @@ public string NameForVB [JsonProperty("homepage")] public string Homepage { get; set; } - [YamlIgnore] - [JsonIgnore] + [YamlMember(Alias = "originallHomepage")] + [JsonProperty("originallHomepage")] public string OriginalHomepage { get; set; } [YamlMember(Alias = "homepageUid")] diff --git a/test/Microsoft.DocAsCode.Build.Engine.Tests/ConvertStrongTypeToJsValueTest.cs b/test/Microsoft.DocAsCode.Build.Engine.Tests/JintProcessorHelperTest.cs similarity index 80% rename from test/Microsoft.DocAsCode.Build.Engine.Tests/ConvertStrongTypeToJsValueTest.cs rename to test/Microsoft.DocAsCode.Build.Engine.Tests/JintProcessorHelperTest.cs index 28cd5288245..f1a6b826b4b 100644 --- a/test/Microsoft.DocAsCode.Build.Engine.Tests/ConvertStrongTypeToJsValueTest.cs +++ b/test/Microsoft.DocAsCode.Build.Engine.Tests/JintProcessorHelperTest.cs @@ -9,17 +9,15 @@ namespace Microsoft.DocAsCode.Build.Engine.Tests using Xunit; [Trait("Owner", "lianwei")] - public class ConvertStrongTypeToJsValueTest + public class JintProcessorHelperTest { [Trait("Related", "JintProcessor")] [Fact] public void TestJObjectConvertWithJToken() { - var testDataJson = JsonUtility.Serialize(new TestData()); - using (var sr = new StringReader(testDataJson)) + var testData = ConvertToObjectHelper.ConvertStrongTypeToObject(new TestData()); { - var jObject = JsonUtility.Deserialize(sr); - var jsValue = JintProcessorHelper.ConvertStrongTypeToJsValue(jObject); + var jsValue = JintProcessorHelper.ConvertObjectToJsValue(testData); Assert.True(jsValue.IsObject()); dynamic value = jsValue.ToObject(); Assert.Equal(2, value.ValueA); @@ -42,7 +40,7 @@ public void TestJObjectConvertWithJToken() [InlineData(true, true)] public void TestJObjectConvertWithPrimaryType(object input, object expected) { - var jsValue = JintProcessorHelper.ConvertStrongTypeToJsValue(input); + var jsValue = JintProcessorHelper.ConvertObjectToJsValue(input); Assert.Equal(expected, jsValue.ToObject()); }