Skip to content

Memory consumption improvement 1: normalize object to week type befor… #2082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vicancy merged 2 commits into from
Sep 27, 2017
Merged

Memory consumption improvement 1: normalize object to week type befor… #2082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
28e9eb1
Memory consumption improvement 1: normalize object to week type befor…
vicancy Sep 26, 2017
320368c
fix test failure
vicancy Sep 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 59 additions & 76 deletions src/Microsoft.DocAsCode.Build.Engine/ManifestProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,29 @@ namespace Microsoft.DocAsCode.Build.Engine
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;

using Newtonsoft.Json.Linq;

using Microsoft.DocAsCode.Common;
using Microsoft.DocAsCode.Plugins;

internal class ManifestProcessor
{
private List<ManifestItemWithContext> _manifestWithContext;
private DocumentBuildContext _context;
private TemplateProcessor _templateProcessor;
private readonly List<ManifestItemWithContext> _manifestWithContext;
private readonly DocumentBuildContext _context;
private readonly TemplateProcessor _templateProcessor;
private readonly IDictionary<string, object> _globalMetadata;

public ManifestProcessor(List<ManifestItemWithContext> manifestWithContext, DocumentBuildContext context, TemplateProcessor templateProcessor)
{
_context = context ?? throw new ArgumentNullException(nameof(context));
_templateProcessor = templateProcessor ?? throw new ArgumentNullException(nameof(templateProcessor));
_manifestWithContext = manifestWithContext ?? throw new ArgumentNullException(nameof(manifestWithContext));

// E.g. we can set TOC model to be globally shared by every data model
// Make sure it is single thread
_globalMetadata = _templateProcessor.Tokens?.ToDictionary(pair => pair.Key, pair => (object)pair.Value)
?? new Dictionary<string, object>();
}

public void Process()
Expand All @@ -35,16 +38,17 @@ public void Process()
UpdateContext();
}

// Run getOptions from Template
using (new LoggerPhaseScope("FeedOptions", LogLevel.Verbose))
// Afterwards, m.Item.Model.Content is always IDictionary
using (new LoggerPhaseScope("NormalizeToObject", LogLevel.Verbose))
{
FeedOptions();
NormalizeToObject();
}

// Run getOptions from Template and feed options back to context
// Template can feed back xref map, actually, the anchor # location can only be determined in template
using (new LoggerPhaseScope("FeedXRefMap", LogLevel.Verbose))
using (new LoggerPhaseScope("FeedOptions", LogLevel.Verbose))
{
FeedXRefMap();
FeedOptions();
}

using (new LoggerPhaseScope("UpdateHref", LogLevel.Verbose))
Expand All @@ -71,28 +75,42 @@ private void UpdateContext()
_context.ResolveExternalXRefSpec();
}

private void FeedOptions()
private void NormalizeToObject()
{
Logger.LogVerbose("Feeding options from template...");
Logger.LogVerbose("Normalizing all the object to week type");

_manifestWithContext.RunAll(m =>
{
if (m.TemplateBundle == null)
if (m.FileModel.Type == DocumentType.Resource)
{
return;
}

using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic($"Feed options from template for {m.Item.DocumentType}...");
m.Options = m.TemplateBundle.GetOptions(m.Item, _context);
var model = m.Item.Model.Content;
// Change file model to weak type
// Go through the convert even if it is IDictionary as the inner object might be of strong type
var modelAsObject = ConvertToObjectHelper.ConvertStrongTypeToObject(model);
if (modelAsObject is IDictionary<string, object>)
{
m.Item.Model.Content = modelAsObject;
}
else
{
Logger.LogWarning("Input model is not an Object model, it will be wrapped into an Object model. Please use --exportRawModel to view the wrapped model");
m.Item.Model.Content = new Dictionary<string, object>
{
["model"] = modelAsObject
};
}
}
},
_context.MaxParallelism);
}

private void FeedXRefMap()
private void FeedOptions()
{
Logger.LogVerbose("Feeding xref map...");
Logger.LogVerbose("Feeding options from template...");
_manifestWithContext.RunAll(m =>
{
if (m.TemplateBundle == null)
Expand All @@ -102,11 +120,14 @@ private void FeedXRefMap()

using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic($"Feed xref map from template for {m.Item.DocumentType}...");
if (m.Options.Bookmarks == null) return;
foreach (var pair in m.Options.Bookmarks)
Logger.LogDiagnostic($"Feed options from template for {m.Item.DocumentType}...");
m.Options = m.TemplateBundle.GetOptions(m.Item, _context);
if (m.Options?.Bookmarks != null)
{
_context.RegisterInternalXrefSpecBookmark(pair.Key, pair.Value);
foreach (var pair in m.Options.Bookmarks)
{
_context.RegisterInternalXrefSpecBookmark(pair.Key, pair.Value);
}
}
}
},
Expand Down Expand Up @@ -137,6 +158,8 @@ private void ApplySystemMetadata()
// Add system attributes
var systemMetadataGenerator = new SystemMetadataGenerator(_context);

var sharedObjects = new ConcurrentDictionary<string, object>();

_manifestWithContext.RunAll(m =>
{
if (m.FileModel.Type == DocumentType.Resource)
Expand All @@ -149,76 +172,36 @@ private void ApplySystemMetadata()

// TODO: use weak type for system attributes from the beginning
var systemAttrs = systemMetadataGenerator.Generate(m.Item);
var metadata = (JObject)ConvertToObjectHelper.ConvertStrongTypeToJObject(systemAttrs);
// Change file model to weak type
var model = m.Item.Model.Content;
var modelAsObject = (JToken)ConvertToObjectHelper.ConvertStrongTypeToJObject(model);
if (modelAsObject is JObject)
var metadata = (IDictionary<string, object>)ConvertToObjectHelper.ConvertStrongTypeToObject(systemAttrs);

var model = (IDictionary<string, object>)m.Item.Model.Content;

foreach (var pair in metadata)
{
foreach (var pair in (JObject)modelAsObject)
if (!model.ContainsKey(pair.Key))
{
// Overwrites the existing system metadata if the same key is defined in document model
metadata[pair.Key] = pair.Value;
model[pair.Key] = pair.Value;
}
}
else
{
Logger.LogWarning("Input model is not an Object model, it will be wrapped into an Object model. Please use --exportRawModel to view the wrapped model");
metadata["model"] = modelAsObject;
}

// Append system metadata to model
m.Item.Model.Serializer = null;
m.Item.Model.Content = metadata;
}
},
_context.MaxParallelism);
}

private IDictionary<string, object> FeedGlobalVariables()
{
Logger.LogVerbose("Feeding global variables from template...");

// E.g. we can set TOC model to be globally shared by every data model
// Make sure it is single thread
var initialGlobalVariables = _templateProcessor.Tokens;
IDictionary<string, object> metadata = initialGlobalVariables == null ?
new Dictionary<string, object>() :
initialGlobalVariables.ToDictionary(pair => pair.Key, pair => (object)pair.Value);
var sharedObjects = new ConcurrentDictionary<string, object>();
_manifestWithContext.RunAll(m =>
{
if (m.TemplateBundle == null)
{
return;
}

using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic($"Load shared model from template for {m.Item.DocumentType}...");
if (m.Options.IsShared)
if (m.Options?.IsShared == true)
{
sharedObjects[m.Item.Key] = m.Item.Model.Content;
// Take a snapshot of current model as shared object
sharedObjects[m.Item.Key] = new Dictionary<string, object>(model);
}

}
},
_context.MaxParallelism);

metadata["_shared"] = sharedObjects;
return metadata;
_globalMetadata["_shared"] = sharedObjects;
}

private List<ManifestItem> ProcessTemplate()
{
// Register global variables after href are all updated
IDictionary<string, object> globalVariables;
using (new LoggerPhaseScope("FeedGlobalVariables", LogLevel.Verbose))
{
globalVariables = FeedGlobalVariables();
}

// processor to add global variable to the model
return _templateProcessor.Process(_manifestWithContext.Select(s => s.Item).ToList(), _context.ApplyTemplateSettings, globalVariables);
return _templateProcessor.Process(_manifestWithContext.Select(s => s.Item).ToList(), _context.ApplyTemplateSettings, _globalMetadata);
}

#endregion
Expand Down
102 changes: 12 additions & 90 deletions src/Microsoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/JintProcessorHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,115 +3,37 @@

namespace Microsoft.DocAsCode.Build.Engine
{
using System;
using System.IO;
using System.Threading;
using System.Collections.Generic;

using Jint;

using Microsoft.DocAsCode.Common;

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

public static class JintProcessorHelper
{
private static readonly Engine DefaultEngine = new Engine();
private static readonly ThreadLocal<JsonSerializer> _toJsValueSerializer = new ThreadLocal<JsonSerializer>(
() =>
{
var jsonSerializer = new JsonSerializer();
jsonSerializer.NullValueHandling = NullValueHandling.Ignore;
jsonSerializer.ReferenceLoopHandling = ReferenceLoopHandling.Serialize;
jsonSerializer.Converters.Add(new JObjectToJsValueConverter());
return jsonSerializer;
});

public static Jint.Native.JsValue ConvertStrongTypeToJsValue(object raw)
{
var token = raw as JToken;
if (token != null)
{
return ConvertJTokenToJsValue(token);
}

using (MemoryStream ms = new MemoryStream())
{
using (StreamWriter sw = new StreamWriter(ms))
{
JsonUtility.Serialize(sw, raw);
sw.Flush();
ms.Seek(0, SeekOrigin.Begin);
using (StreamReader sr = new StreamReader(ms))
{
return JsonUtility.Deserialize<Jint.Native.JsValue>(sr, _toJsValueSerializer.Value);
}
}
}
}

public static Jint.Native.JsValue ConvertJTokenToJsValue(JToken raw)
public static Jint.Native.JsValue ConvertObjectToJsValue(object raw)
{
var jArray = raw as JArray;
if (jArray != null)
{
var jsArray = DefaultEngine.Array.Construct(Jint.Runtime.Arguments.Empty);
foreach (var item in jArray)
{
DefaultEngine.Array.PrototypeObject.Push(jsArray, Jint.Runtime.Arguments.From(ConvertJTokenToJsValue(item)));
}
return jsArray;
}
var jObject = raw as JObject;
if (jObject != null)
if (raw is IDictionary<string, object> idict)
{
var jsObject = DefaultEngine.Object.Construct(Jint.Runtime.Arguments.Empty);
foreach (var pair in jObject)
foreach (var pair in idict)
{
jsObject.Put(pair.Key, ConvertJTokenToJsValue(pair.Value), true);
jsObject.Put(pair.Key, ConvertObjectToJsValue(pair.Value), true);
}
return jsObject;
}

var jValue = raw as JValue;
if (jValue != null)
{
return Jint.Native.JsValue.FromObject(DefaultEngine, jValue.Value);
}

return Jint.Native.JsValue.FromObject(DefaultEngine, raw);
}

private sealed class JObjectToJsValueConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
{
return objectType == typeof(Jint.Native.JsValue);
}

public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
else if (raw is IList<object> list)
{
if (reader.TokenType == JsonToken.StartArray)
{
var jArray = JArray.Load(reader);
return ConvertJTokenToJsValue(jArray);
}
else if (reader.TokenType == JsonToken.StartObject)
{
var jObject = JObject.Load(reader);
var converted = ConvertJTokenToJsValue(jObject);
return converted;
}
else
var jsArray = DefaultEngine.Array.Construct(Jint.Runtime.Arguments.Empty);
foreach (var item in list)
{
var jValue = JValue.Load(reader);
return ConvertJTokenToJsValue(jValue);
DefaultEngine.Array.PrototypeObject.Push(jsArray, Jint.Runtime.Arguments.From(ConvertObjectToJsValue(item)));
}
return jsArray;
}

public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
else
{
throw new NotImplementedException();
return Jint.Native.JsValue.FromObject(DefaultEngine, raw);
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion ...osoft.DocAsCode.Build.Engine/TemplateProcessors/Preprocessors/TemplateJintPreprocessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ private static Func<object, object> GetFunc(string funcName, ObjectInstance expo
{
return s =>
{
var model = JintProcessorHelper.ConvertStrongTypeToJsValue(s);
var model = JintProcessorHelper.ConvertObjectToJsValue(s);
return func.Invoke(model).ToObject();
};
}
Expand Down
Loading