Skip to content

[Don't merge] RAR cache file optimization #8901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

[Don't merge] RAR cache file optimization #8901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e230803
Split SystemState into ResolveAssemblyReferenceCache (on-disk cache) …
ladipro May 22, 2023
576ae20
Implement lazy loading of RAR on-disk cache
ladipro Jun 9, 2023
45f704e
Make on-disk cache contents deterministic
ladipro Jun 12, 2023
d876eee
Various fixes (incomplete)
ladipro Jun 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,85 +39,85 @@ public void Dispose()
}

[Fact]
public void RoundTripEmptyState()
public void RoundTripEmptyCache()
{
SystemState systemState = new();
ResolveAssemblyReferenceCache rarCache = new();

systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper);
rarCache.SerializeCache(_rarCacheFile, _taskLoggingHelper);

var deserialized = StateFileBase.DeserializeCache<SystemState>(_rarCacheFile, _taskLoggingHelper);
var deserialized = StateFileBase.DeserializeCache<ResolveAssemblyReferenceCache>(_rarCacheFile, _taskLoggingHelper);

deserialized.ShouldNotBeNull();
}

[Fact]
public void CorrectFileVersion()
{
SystemState systemState = new();
ResolveAssemblyReferenceCache rarCache = new();

systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper);
rarCache.SerializeCache(_rarCacheFile, _taskLoggingHelper);
using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite))
{
cacheStream.Seek(0, SeekOrigin.Begin);
cacheStream.WriteByte(StateFileBase.CurrentSerializationVersion);
cacheStream.Close();
}

var deserialized = StateFileBase.DeserializeCache<SystemState>(_rarCacheFile, _taskLoggingHelper);
var deserialized = StateFileBase.DeserializeCache<ResolveAssemblyReferenceCache>(_rarCacheFile, _taskLoggingHelper);

deserialized.ShouldNotBeNull();
}

[Fact]
public void WrongFileVersion()
{
SystemState systemState = new();
ResolveAssemblyReferenceCache rarCache = new();

systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper);
rarCache.SerializeCache(_rarCacheFile, _taskLoggingHelper);
using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite))
{
cacheStream.Seek(0, SeekOrigin.Begin);
cacheStream.WriteByte(StateFileBase.CurrentSerializationVersion - 1);
cacheStream.Close();
}

var deserialized = StateFileBase.DeserializeCache<SystemState>(_rarCacheFile, _taskLoggingHelper);
var deserialized = StateFileBase.DeserializeCache<ResolveAssemblyReferenceCache>(_rarCacheFile, _taskLoggingHelper);

deserialized.ShouldBeNull();
}

[Fact]
public void ValidateSerializationAndDeserialization()
{
Dictionary<string, SystemState.FileState> cache = new() {
{ "path1", new SystemState.FileState(DateTime.Now) },
{ "path2", new SystemState.FileState(DateTime.Now) { Assembly = new AssemblyNameExtension("hi") } },
{ "dllName", new SystemState.FileState(DateTime.Now.AddSeconds(-10)) {
Dictionary<string, ResolveAssemblyReferenceCache.FileState> cache = new() {
{ "path1", new ResolveAssemblyReferenceCache.FileState(DateTime.Now) },
{ "path2", new ResolveAssemblyReferenceCache.FileState(DateTime.Now) { Assembly = new AssemblyNameExtension("hi") } },
{ "dllName", new ResolveAssemblyReferenceCache.FileState(DateTime.Now.AddSeconds(-10)) {
Assembly = null,
RuntimeVersion = "v4.0.30319",
FrameworkNameAttribute = new FrameworkName(".NETFramework", Version.Parse("4.7.2"), "Profile"),
scatterFiles = new string[] { "first", "second" } } } };
SystemState sysState = new();
sysState.instanceLocalFileStateCache = cache;
SystemState sysState2 = null;
ScatterFiles = new string[] { "first", "second" } } } };
ResolveAssemblyReferenceCache rarCache = new();
rarCache.instanceLocalFileStateCache = cache;
ResolveAssemblyReferenceCache rarCache2 = null;
using (TestEnvironment env = TestEnvironment.Create())
{
TransientTestFile file = env.CreateFile();
sysState.SerializeCache(file.Path, null);
sysState2 = StateFileBase.DeserializeCache<SystemState>(file.Path, null);
rarCache.SerializeCache(file.Path, null);
rarCache2 = StateFileBase.DeserializeCache<ResolveAssemblyReferenceCache>(file.Path, null);
}

Dictionary<string, SystemState.FileState> cache2 = sysState2.instanceLocalFileStateCache;
Dictionary<string, ResolveAssemblyReferenceCache.FileState> cache2 = rarCache2.instanceLocalFileStateCache;
cache2.Count.ShouldBe(cache.Count);
cache2["path2"].Assembly.Name.ShouldBe(cache["path2"].Assembly.Name);
SystemState.FileState dll = cache["dllName"];
SystemState.FileState dll2 = cache2["dllName"];
ResolveAssemblyReferenceCache.FileState dll = cache["dllName"];
ResolveAssemblyReferenceCache.FileState dll2 = cache2["dllName"];
dll2.Assembly.ShouldBe(dll.Assembly);
dll2.FrameworkNameAttribute.FullName.ShouldBe(dll.FrameworkNameAttribute.FullName);
dll2.LastModified.ShouldBe(dll.LastModified);
dll2.RuntimeVersion.ShouldBe(dll.RuntimeVersion);
dll2.scatterFiles.Length.ShouldBe(dll.scatterFiles.Length);
dll2.scatterFiles[1].ShouldBe(dll.scatterFiles[1]);
dll2.ScatterFiles.Length.ShouldBe(dll.ScatterFiles.Length);
dll2.ScatterFiles[1].ShouldBe(dll.ScatterFiles[1]);
}
}
}
44 changes: 22 additions & 22 deletions src/Tasks.UnitTests/RARPrecomputedCache_Tests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,20 @@ public void TestPrecomputedCacheOutput()
{
_cache = new SystemState()
};
t._cache.instanceLocalFileStateCache = new Dictionary<string, SystemState.FileState>() {
{ Path.Combine(standardCache.Path, "assembly1"), new SystemState.FileState(DateTime.Now) },
{ Path.Combine(standardCache.Path, "assembly2"), new SystemState.FileState(DateTime.Now) { Assembly = new Shared.AssemblyNameExtension("hi") } } };
t._cache.instanceLocalFileStateCache = new Dictionary<string, ResolveAssemblyReferenceCache.FileState>() {
{ Path.Combine(standardCache.Path, "assembly1"), new ResolveAssemblyReferenceCache.FileState(DateTime.Now) },
{ Path.Combine(standardCache.Path, "assembly2"), new ResolveAssemblyReferenceCache.FileState(DateTime.Now) { Assembly = new Shared.AssemblyNameExtension("hi") } } };
t._cache.IsDirty = true;
t.StateFile = standardCache.Path;
t.WriteStateFile();
t.WriteStateFile(p => DateTime.Now);
int standardLen = File.ReadAllText(standardCache.Path).Length;
File.Delete(standardCache.Path);
standardLen.ShouldBeGreaterThan(0);

string precomputedPath = standardCache.Path + ".cache";
t._cache.IsDirty = true;
t.AssemblyInformationCacheOutputPath = precomputedPath;
t.WriteStateFile();
t.WriteStateFile(p => DateTime.Now);
File.Exists(standardCache.Path).ShouldBeFalse();
int preLen = File.ReadAllText(precomputedPath).Length;
preLen.ShouldBeGreaterThan(0);
Expand All @@ -57,26 +57,26 @@ public void StandardCacheTakesPrecedence()
{
_cache = new SystemState()
};
rarWriterTask._cache.instanceLocalFileStateCache = new Dictionary<string, SystemState.FileState>();
rarWriterTask._cache.instanceLocalFileStateCache = new Dictionary<string, ResolveAssemblyReferenceCache.FileState>();
rarWriterTask.StateFile = standardCache.Path;
rarWriterTask._cache.IsDirty = true;
// Write standard cache
rarWriterTask.WriteStateFile();
rarWriterTask.WriteStateFile(p => DateTime.Now);

string dllName = Path.Combine(Path.GetDirectoryName(standardCache.Path), "randomFolder", "dll.dll");
rarWriterTask._cache.instanceLocalFileStateCache.Add(dllName,
new SystemState.FileState(DateTime.Now)
new ResolveAssemblyReferenceCache.FileState(DateTime.Now)
{
Assembly = null,
RuntimeVersion = "v4.0.30319",
FrameworkNameAttribute = new System.Runtime.Versioning.FrameworkName(".NETFramework", Version.Parse("4.7.2"), "Profile"),
scatterFiles = new string[] { "first", "second" }
ScatterFiles = new string[] { "first", "second" }
});
string precomputedCachePath = standardCache.Path + ".cache";
rarWriterTask.AssemblyInformationCacheOutputPath = precomputedCachePath;
rarWriterTask._cache.IsDirty = true;
// Write precomputed cache
rarWriterTask.WriteStateFile();
rarWriterTask.WriteStateFile(p => DateTime.Now);

ResolveAssemblyReference rarReaderTask = new ResolveAssemblyReference();
rarReaderTask.StateFile = standardCache.Path;
Expand All @@ -89,7 +89,7 @@ public void StandardCacheTakesPrecedence()
// When we read the state file, it should read from the caches produced in a normal build. In this case,
// the normal cache does not have dll.dll, whereas the precomputed cache does, so it should not be
// present when we read it.
rarReaderTask.ReadStateFile(p => true);
rarReaderTask.ReadStateFile(p => true, p => DateTime.Now);
rarReaderTask._cache.instanceLocalFileStateCache.ShouldNotContainKey(dllName);
}
}
Expand All @@ -105,22 +105,22 @@ public void TestPreComputedCacheInputMatchesOutput()
_cache = new SystemState()
};
string dllName = Path.Combine(Path.GetDirectoryName(precomputedCache.Path), "randomFolder", "dll.dll");
rarWriterTask._cache.instanceLocalFileStateCache = new Dictionary<string, SystemState.FileState>() {
{ Path.Combine(precomputedCache.Path, "..", "assembly1", "assembly1"), new SystemState.FileState(DateTime.Now) },
{ Path.Combine(precomputedCache.Path, "assembly2"), new SystemState.FileState(DateTime.Now) { Assembly = new Shared.AssemblyNameExtension("hi") } },
{ dllName, new SystemState.FileState(DateTime.Now) {
rarWriterTask._cache.instanceLocalFileStateCache = new Dictionary<string, ResolveAssemblyReferenceCache.FileState>() {
{ Path.Combine(precomputedCache.Path, "..", "assembly1", "assembly1"), new ResolveAssemblyReferenceCache.FileState(DateTime.Now) },
{ Path.Combine(precomputedCache.Path, "assembly2"), new ResolveAssemblyReferenceCache.FileState(DateTime.Now) { Assembly = new Shared.AssemblyNameExtension("hi") } },
{ dllName, new ResolveAssemblyReferenceCache.FileState(DateTime.Now) {
Assembly = null,
RuntimeVersion = "v4.0.30319",
FrameworkNameAttribute = new System.Runtime.Versioning.FrameworkName(".NETFramework", Version.Parse("4.7.2"), "Profile"),
scatterFiles = new string[] { "first", "second" } } } };
ScatterFiles = new string[] { "first", "second" } } } };

rarWriterTask.AssemblyInformationCacheOutputPath = precomputedCache.Path;
rarWriterTask._cache.IsDirty = true;

// Throws an exception because precomputedCache.Path already exists.
Should.Throw<InvalidOperationException>(() => rarWriterTask.WriteStateFile());
Should.Throw<InvalidOperationException>(() => rarWriterTask.WriteStateFile(p => DateTime.Now));
File.Delete(precomputedCache.Path);
rarWriterTask.WriteStateFile();
rarWriterTask.WriteStateFile(p => DateTime.Now);

ResolveAssemblyReference rarReaderTask = new ResolveAssemblyReference();
rarReaderTask.StateFile = precomputedCache.Path.Substring(0, precomputedCache.Path.Length - 6); // Not a real path; should not be used.
Expand All @@ -131,14 +131,14 @@ public void TestPreComputedCacheInputMatchesOutput()

// At this point, the standard cache does not exist, so it defaults to reading the "precomputed" cache.
// Then we verify that the information contained in that cache matches what we'd expect.
rarReaderTask.ReadStateFile(p => true);
rarReaderTask.ReadStateFile(p => true, p => DateTime.Now);
rarReaderTask._cache.instanceLocalFileStateCache.ShouldContainKey(dllName);
SystemState.FileState assembly3 = rarReaderTask._cache.instanceLocalFileStateCache[dllName];
ResolveAssemblyReferenceCache.FileState assembly3 = rarReaderTask._cache.instanceLocalFileStateCache[dllName];
assembly3.Assembly.ShouldBeNull();
assembly3.RuntimeVersion.ShouldBe("v4.0.30319");
assembly3.FrameworkNameAttribute.Version.ShouldBe(Version.Parse("4.7.2"));
assembly3.scatterFiles.Length.ShouldBe(2);
assembly3.scatterFiles[1].ShouldBe("second");
assembly3.ScatterFiles.Length.ShouldBe(2);
assembly3.ScatterFiles[1].ShouldBe("second");
}
}
}
Expand Down
Loading