diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitReferenceResolverTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitReferenceResolverTests.cs index c61fb555..8a8295dd 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitReferenceResolverTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitReferenceResolverTests.cs @@ -131,5 +131,136 @@ public void ReadPackedReferences_Errors(string content) { Assert.Throws(() => GitReferenceResolver.ReadPackedReferences(new StringReader(content), "")); } + + [Fact] + public void ResolveReference_Reftable() + { + using var temp = new TempRoot(); + + var gitDir = temp.CreateDirectory(); + var reftableDir = gitDir.CreateDirectory("reftable"); + + // Create a minimal reftable file with a reference + // This is a simplified test - in reality, we'd need to create a proper binary reftable file + // For now, we'll test that the resolver falls back correctly when reftable is empty + + var commonDir = temp.CreateDirectory(); + var refsHeadsDir = commonDir.CreateDirectory("refs").CreateDirectory("heads"); + + refsHeadsDir.CreateFile("master").WriteAllText("1111111111111111111111111111111111111111"); + + var resolver = new GitReferenceResolver(gitDir.Path, commonDir.Path); + + // Should still resolve refs from files even when reftable directory exists but is empty + Assert.Equal("1111111111111111111111111111111111111111", resolver.ResolveReference("ref: refs/heads/master")); + } + + [Fact] + public void ResolveReference_ReftableWithBinaryFile() + { + using var temp = new TempRoot(); + + var gitDir = temp.CreateDirectory(); + var reftableDir = gitDir.CreateDirectory("reftable"); + + // Create a minimal valid reftable file + var reftableFile = reftableDir.CreateFile("test.ref"); + using (var stream = new FileStream(reftableFile.Path, FileMode.Create)) + using (var writer = new BinaryWriter(stream)) + { + // Write reftable header (24 bytes) + WriteUInt32BE(writer, 0x52454654); // Magic: 'REFT' + WriteUInt32BE(writer, 1); // Version: 1 + WriteUInt64BE(writer, 1); // Min update index + WriteUInt64BE(writer, 1); // Max update index + + // Write a ref block + long blockStart = stream.Position; + writer.Write((byte)0x72); // Block type: 'r' (ref) + + // Placeholder for block size (will be updated later) + long sizePos = stream.Position; + writer.Write((byte)0); + writer.Write((byte)0); + writer.Write((byte)0); + + // Write a ref record for refs/heads/test + byte[] refName = System.Text.Encoding.UTF8.GetBytes("refs/heads/test"); + WriteVarint(writer, 0); // Prefix length + WriteVarint(writer, refName.Length); // Suffix length + writer.Write(refName); + + // Value type: 0x1 (has object ID) + writer.Write((byte)0x01); + + // Object ID (20 bytes SHA-1) - all 0x22 for testing + writer.Write(new byte[] { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22 }); + + // Restart points (2 bytes) + writer.Write((byte)0); + writer.Write((byte)0); + + // Calculate and write block size + long blockEnd = stream.Position; + int blockSize = (int)(blockEnd - blockStart); + + // Pad to 256 bytes + int padding = 256 - (blockSize % 256); + if (padding < 256) + { + writer.Write(new byte[padding]); + blockSize += padding; + } + + // Update block size in header + long currentPos = stream.Position; + stream.Seek(sizePos, SeekOrigin.Begin); + writer.Write((byte)((blockSize >> 16) & 0xFF)); + writer.Write((byte)((blockSize >> 8) & 0xFF)); + writer.Write((byte)(blockSize & 0xFF)); + stream.Seek(currentPos, SeekOrigin.Begin); + + // Write footer (68 bytes of zeros for simplicity) + writer.Write(new byte[68]); + } + + var commonDir = temp.CreateDirectory(); + var resolver = new GitReferenceResolver(gitDir.Path, commonDir.Path); + + // Should be able to resolve the ref from reftable + var result = resolver.ResolveReference("ref: refs/heads/test"); + Assert.Equal("2222222222222222222222222222222222222222", result); + } + + private static void WriteUInt32BE(BinaryWriter writer, uint value) + { + writer.Write((byte)((value >> 24) & 0xFF)); + writer.Write((byte)((value >> 16) & 0xFF)); + writer.Write((byte)((value >> 8) & 0xFF)); + writer.Write((byte)(value & 0xFF)); + } + + private static void WriteUInt64BE(BinaryWriter writer, ulong value) + { + writer.Write((byte)((value >> 56) & 0xFF)); + writer.Write((byte)((value >> 48) & 0xFF)); + writer.Write((byte)((value >> 40) & 0xFF)); + writer.Write((byte)((value >> 32) & 0xFF)); + writer.Write((byte)((value >> 24) & 0xFF)); + writer.Write((byte)((value >> 16) & 0xFF)); + writer.Write((byte)((value >> 8) & 0xFF)); + writer.Write((byte)(value & 0xFF)); + } + + private static void WriteVarint(BinaryWriter writer, int value) + { + while (value > 0x7F) + { + writer.Write((byte)(0x80 | (value & 0x7F))); + value >>= 7; + } + writer.Write((byte)(value & 0x7F)); + } } } diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs index 7ba9993d..fb85193c 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs @@ -239,6 +239,39 @@ public void OpenRepository_Version1_Extensions() Assert.Null(repository.WorkingDirectory); } + [Fact] + public void OpenRepository_Version1_ReftableExtension() + { + using var temp = new TempRoot(); + + var homeDir = temp.CreateDirectory(); + + var workingDir = temp.CreateDirectory(); + var gitDir = workingDir.CreateDirectory(".git"); + + gitDir.CreateFile("HEAD").WriteAllText("ref: refs/heads/main"); + gitDir.CreateDirectory("reftable"); + gitDir.CreateDirectory("objects"); + + gitDir.CreateFile("config").WriteAllText(@" +[core] + repositoryformatversion = 1 +[extensions] + refstorage = reftable +"); + + Assert.True(GitRepository.TryFindRepository(gitDir.Path, out var location)); + Assert.Equal(gitDir.Path, location.CommonDirectory); + Assert.Equal(gitDir.Path, location.GitDirectory); + Assert.Null(location.WorkingDirectory); + + // Should not throw - refstorage is a known extension + var repository = GitRepository.OpenRepository(location, GitEnvironment.Empty); + Assert.Equal(gitDir.Path, repository.CommonDirectory); + Assert.Equal(gitDir.Path, repository.GitDirectory); + Assert.Null(repository.WorkingDirectory); + } + [Fact] public void OpenRepository_Version1_UnknownExtension() { diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs index 7d970c09..56d64ab1 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs @@ -17,6 +17,7 @@ internal sealed class GitReferenceResolver // See https://git-scm.com/docs/gitrepository-layout#Documentation/gitrepository-layout.txt-HEAD private const string PackedRefsFileName = "packed-refs"; + private const string ReftableDirectoryName = "reftable"; private const string RefsPrefix = "refs/"; private readonly string _commonDirectory; @@ -39,6 +40,25 @@ private static ImmutableDictionary ReadPackedReferences(string g { // https://git-scm.com/docs/git-pack-refs + // First try to read from reftable + var reftableDirectory = Path.Combine(gitDirectory, ReftableDirectoryName); + if (Directory.Exists(reftableDirectory)) + { + try + { + var reftableRefs = GitReftableReader.ReadReftableReferences(reftableDirectory); + if (reftableRefs.Count > 0) + { + return reftableRefs; + } + } + catch + { + // If reftable reading fails, fall through to try packed-refs + } + } + + // Fall back to packed-refs var packedRefsPath = Path.Combine(gitDirectory, PackedRefsFileName); if (!File.Exists(packedRefsPath)) { diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReftableReader.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReftableReader.cs new file mode 100644 index 00000000..94e94d44 --- /dev/null +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReftableReader.cs @@ -0,0 +1,339 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Text; + +namespace Microsoft.Build.Tasks.Git +{ + internal static class GitReftableReader + { + // See https://git-scm.com/docs/reftable + private const uint ReftableMagic = 0x52454654; // 'REFT' + private const byte BlockTypeRef = 0x72; // 'r' + private const byte BlockTypeObj = 0x6f; // 'o' + private const byte BlockTypeLog = 0x67; // 'g' + private const byte BlockTypeIndex = 0x69; // 'i' + + private const int HeaderSize = 24; + private const int FooterSize = 68; + + public static ImmutableDictionary ReadReftableReferences(string reftableDirectory) + { + // https://git-scm.com/docs/gitrepository-layout#Documentation/gitrepository-layout.txt-reftable + + if (!Directory.Exists(reftableDirectory)) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(); + + try + { + // Read all .ref files in the reftable directory + var refFiles = Directory.GetFiles(reftableDirectory, "*.ref"); + + // Sort to ensure we process them in order (newer files override older ones) + Array.Sort(refFiles, StringComparer.Ordinal); + + foreach (var refFile in refFiles) + { + try + { + ReadReftableFile(refFile, builder); + } + catch + { + // If we can't read a file, skip it and try others + continue; + } + } + } + catch + { + // If we can't read the reftable directory, return empty + return ImmutableDictionary.Empty; + } + + return builder.ToImmutable(); + } + + private static void ReadReftableFile(string path, ImmutableDictionary.Builder builder) + { + using var stream = File.OpenRead(path); + using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true); + + // Read and validate header + var header = ReadHeader(reader); + if (header == null) + { + return; + } + + // Seek to the first block (after header) + stream.Seek(HeaderSize, SeekOrigin.Begin); + + // Read blocks until we reach the footer + long footerStart = stream.Length - FooterSize; + while (stream.Position < footerStart) + { + long blockStart = stream.Position; + + // Read block header + if (stream.Length - stream.Position < 4) + { + break; + } + + byte[] blockHeader = reader.ReadBytes(4); + if (blockHeader.Length < 4) + { + break; + } + + byte blockType = blockHeader[0]; + + // Block size is stored in bytes 1-3 (24-bit big-endian) + int blockSize = (blockHeader[1] << 16) | (blockHeader[2] << 8) | blockHeader[3]; + + if (blockSize == 0 || blockSize > stream.Length - blockStart) + { + break; + } + + // Only process ref blocks + if (blockType == BlockTypeRef) + { + ReadRefBlock(reader, blockStart, blockSize, builder); + } + + // Move to next block + stream.Seek(blockStart + blockSize, SeekOrigin.Begin); + } + } + + private static ReftableHeader? ReadHeader(BinaryReader reader) + { + try + { + uint magic = ReadUInt32BE(reader); + if (magic != ReftableMagic) + { + return null; + } + + uint version = ReadUInt32BE(reader); + if (version != 1) + { + return null; // Only version 1 is supported + } + + // Read the rest of the header + ulong minUpdateIndex = ReadUInt64BE(reader); + ulong maxUpdateIndex = ReadUInt64BE(reader); + + return new ReftableHeader + { + Version = version, + MinUpdateIndex = minUpdateIndex, + MaxUpdateIndex = maxUpdateIndex + }; + } + catch + { + return null; + } + } + + private static void ReadRefBlock(BinaryReader reader, long blockStart, int blockSize, ImmutableDictionary.Builder builder) + { + try + { + long blockEnd = blockStart + blockSize; + + // Skip the 4-byte block header we already read + long dataStart = blockStart + 4; + reader.BaseStream.Seek(dataStart, SeekOrigin.Begin); + + // Track last ref name for prefix compression within this block + string lastRefName = ""; + + // Read restart points count (last 2 bytes before block end, excluding padding) + // For simplicity, we'll do a sequential scan instead of using restart points + + while (reader.BaseStream.Position < blockEnd - 2) + { + var (refName, objectId) = ReadRefRecord(reader, ref lastRefName); + + if (refName == null || objectId == null) + { + break; + } + + // Store the reference (later entries override earlier ones) + builder[refName] = objectId; + } + } + catch + { + // Skip this block if we can't read it + } + } + + private static (string? RefName, string? ObjectId) ReadRefRecord(BinaryReader reader, ref string lastRefName) + { + try + { + // Read varint for prefix length + int prefixLength = ReadVarint(reader); + if (prefixLength < 0) + { + return (null, null); + } + + // Read varint for suffix length + int suffixLength = ReadVarint(reader); + if (suffixLength < 0) + { + return (null, null); + } + + // Read suffix + byte[] suffixBytes = reader.ReadBytes(suffixLength); + if (suffixBytes.Length != suffixLength) + { + return (null, null); + } + + string suffix = Encoding.UTF8.GetString(suffixBytes); + + // Build full reference name using prefix compression + string refName; + if (prefixLength == 0) + { + refName = suffix; + } + else if (prefixLength <= lastRefName.Length) + { + refName = lastRefName.Substring(0, prefixLength) + suffix; + } + else + { + // Invalid prefix length + return (null, null); + } + + lastRefName = refName; + + // Read value type + byte valueType = reader.ReadByte(); + + string? objectId = null; + + // Value type bits: + // 0x1 = has value1 (object ID) + // 0x2 = has value2 (peeled) + // 0x4 = has symref + if ((valueType & 0x1) != 0) + { + // Read object ID (20 bytes for SHA-1) + byte[] oid = reader.ReadBytes(20); + if (oid.Length == 20) + { + objectId = BitConverter.ToString(oid).Replace("-", "").ToLowerInvariant(); + } + } + + if ((valueType & 0x2) != 0) + { + // Skip peeled object ID + reader.ReadBytes(20); + } + + if ((valueType & 0x4) != 0) + { + // Skip symref (varint length + string) + int symrefLength = ReadVarint(reader); + if (symrefLength > 0) + { + reader.ReadBytes(symrefLength); + } + } + + return (refName, objectId); + } + catch + { + return (null, null); + } + } + + private static int ReadVarint(BinaryReader reader) + { + try + { + int result = 0; + int shift = 0; + + while (true) + { + byte b = reader.ReadByte(); + result |= (b & 0x7F) << shift; + + if ((b & 0x80) == 0) + { + break; + } + + shift += 7; + + if (shift > 28) + { + return -1; // Overflow + } + } + + return result; + } + catch + { + return -1; + } + } + + private static uint ReadUInt32BE(BinaryReader reader) + { + byte[] bytes = reader.ReadBytes(4); + if (bytes.Length != 4) + { + throw new EndOfStreamException(); + } + + return ((uint)bytes[0] << 24) | ((uint)bytes[1] << 16) | ((uint)bytes[2] << 8) | bytes[3]; + } + + private static ulong ReadUInt64BE(BinaryReader reader) + { + byte[] bytes = reader.ReadBytes(8); + if (bytes.Length != 8) + { + throw new EndOfStreamException(); + } + + return ((ulong)bytes[0] << 56) | ((ulong)bytes[1] << 48) | ((ulong)bytes[2] << 40) | ((ulong)bytes[3] << 32) | + ((ulong)bytes[4] << 24) | ((ulong)bytes[5] << 16) | ((ulong)bytes[6] << 8) | bytes[7]; + } + + private class ReftableHeader + { + public uint Version { get; set; } + public ulong MinUpdateIndex { get; set; } + public ulong MaxUpdateIndex { get; set; } + } + } +} diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs index fc7446ea..db07d498 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs @@ -25,7 +25,7 @@ internal sealed class GitRepository private const string GitModulesFileName = ".gitmodules"; private static readonly ImmutableArray s_knownExtensions = - ImmutableArray.Create("noop", "preciousObjects", "partialclone", "worktreeConfig"); + ImmutableArray.Create("noop", "preciousObjects", "partialclone", "worktreeConfig", "refstorage"); public GitConfig Config { get; }