✨ Read and write DS extended header including validate and regenerate HMAC

README.md

@@ -13,11 +13,13 @@ The library supports .NET 6.0 and above on Linux, Window and MacOS.
 
 ## Supported formats
 
-_Encryption, decryption or signature validation not supported yet._
+_Encryption and decryption not supported yet._
 
 - DS cartridge filesystem: read and write
-  - Header: read and write
-  - Banner and icon: read and write
+  - Header: read and write, including extended header
+  - Banner and icon: read and write, including animated icons from DSi.
+  - HMAC validation and re-generation when keys are provided.
+  - Signature validation when keys are provided.
 
 ## Documentation
 

src/Directory.Packages.props

@@ -4,6 +4,7 @@
         <PackageVersion Include="Yarhl" Version="4.0.0-preview.153" />
         <PackageVersion Include="Texim" Version="0.1.0-preview.129" />
         <PackageVersion Include="System.Data.HashFunction.CRC" Version="2.0.0" />
+        <PackageVersion Include="Portable.BouncyCastle" Version="1.9.0" />
 
         <PackageVersion Include="YamlDotNet" Version="11.2.1" />
         <PackageVersion Include="FluentAssertions" Version="6.5.1" />

src/Ekona.Tests/Containers/Rom/Binary2BannerTests.cs

@@ -24,6 +24,7 @@
 using FluentAssertions;
 using NUnit.Framework;
 using SceneGate.Ekona.Containers.Rom;
+using SceneGate.Ekona.Security;
 using Texim.Formats;
 using Texim.Images;
 using Texim.Palettes;
@@ -75,19 +76,19 @@ public void DeserializeBannerMatchInfo(string bannerPath)
 
             var actual = node.Children["info"].GetFormatAs<Banner>();
             actual.Version.Should().Be(expected.Version);
-            actual.ChecksumBase.IsValid.Should().BeTrue();
+            actual.ChecksumBase.Status.Should().Be(HashStatus.Valid);
 
             if (actual.Version.Minor > 1) {
-                actual.ChecksumChinese.IsValid.Should().BeTrue();
+                actual.ChecksumChinese.Status.Should().Be(HashStatus.Valid);
             }
 
             if (actual.Version.Minor > 2) {
-                actual.ChecksumKorean.IsValid.Should().BeTrue();
+                actual.ChecksumKorean.Status.Should().Be(HashStatus.Valid);
             }
 
             if (actual.Version is { Major: > 1 } or { Major: 1, Minor: >= 3 }) {
                 actual.SupportAnimatedIcon.Should().BeTrue();
-                actual.ChecksumAnimatedIcon.IsValid.Should().BeTrue();
+                actual.ChecksumAnimatedIcon.Status.Should().Be(HashStatus.Valid);
             } else {
                 actual.SupportAnimatedIcon.Should().BeFalse();
             }

src/Ekona.Tests/Containers/Rom/Binary2NitroRomTests.cs

@@ -1,4 +1,4 @@
-// Copyright(c) 2021 SceneGate
+// Copyright(c) 2021 SceneGate
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -23,6 +23,7 @@
 using FluentAssertions;
 using NUnit.Framework;
 using SceneGate.Ekona.Containers.Rom;
+using SceneGate.Ekona.Security;
 using YamlDotNet.Serialization;
 using YamlDotNet.Serialization.NamingConventions;
 using Yarhl.FileFormat;
@@ -63,6 +64,37 @@ public void DeserializeRomMatchInfo(string infoPath, string romPath)
             node.Should().MatchInfo(expected);
         }
 
+        [TestCaseSource(nameof(GetFiles))]
+        public void DeserializeRomWithKeysHasValidSignatures(string infoPath, string romPath)
+        {
+            TestDataBase.IgnoreIfFileDoesNotExist(romPath);
+            DsiKeyStore keys = TestDataBase.GetDsiKeyStore();
+
+            using Node node = NodeFactory.FromFile(romPath, FileOpenMode.Read);
+            node.Invoking(n => n.TransformWith<Binary2NitroRom, DsiKeyStore>(keys)).Should().NotThrow();
+
+            NitroRom rom = node.GetFormatAs<NitroRom>();
+            ProgramInfo programInfo = rom.Information;
+            bool isDsi = programInfo.UnitCode != DeviceUnitKind.DS;
+
+            if (isDsi || programInfo.ProgramFeatures.HasFlag(DsiRomFeatures.BannerSigned)) {
+                programInfo.BannerMac.Status.Should().Be(HashStatus.Valid);
+            }
+
+            if (programInfo.ProgramFeatures.HasFlag(DsiRomFeatures.ProgramSigned)) {
+                // TODO: Verify header (0x160 bytes) + armX (secure area encrypted) HMAC
+                // programInfo.ProgramMac.Status.Should().Be(HashStatus.Valid)
+                programInfo.OverlaysMac.Status.Should().Be(HashStatus.Valid);
+                programInfo.Signature.Status.Should().Be(HashStatus.Valid);
+            }
+
+            if (isDsi) {
+                programInfo.OverlaysMac.IsNull.Should().BeTrue();
+                programInfo.ProgramMac.IsNull.Should().BeTrue();
+                programInfo.Signature.Status.Should().Be(HashStatus.Valid);
+            }
+        }
+
         [TestCaseSource(nameof(GetFiles))]
         public void TwoWaysIdenticalRomStream(string infoPath, string romPath)
         {
@@ -75,7 +107,7 @@ public void TwoWaysIdenticalRomStream(string infoPath, string romPath)
 
             generatedStream.Stream.Length.Should().Be(node.Stream!.Length);
 
-            // TODO: After identical header and banner
+            // TODO: After implementing ARM9 tail and DSi fields
             // generatedStream.Stream!.Compare(node.Stream).Should().BeTrue()
         }
 
@@ -93,10 +125,38 @@ public void ReadWriteThreeWaysRomMatchInfo(string infoPath, string romPath)
 
             using Node node = NodeFactory.FromFile(romPath, FileOpenMode.Read);
             node.Invoking(n => n.TransformWith<Binary2NitroRom>()).Should().NotThrow();
+            ProgramInfo originalInfo = node.GetFormatAs<NitroRom>().Information;
+
             node.Invoking(n => n.TransformWith<NitroRom2Binary>()).Should().NotThrow();
             node.Invoking(n => n.TransformWith<Binary2NitroRom>()).Should().NotThrow();
+            ProgramInfo newInfo = node.GetFormatAs<NitroRom>().Information;
 
             node.Should().MatchInfo(expected);
+
+            // Keep old hashes
+            newInfo.OverlaysMac.Hash.Should().BeEquivalentTo(originalInfo.OverlaysMac.Hash);
+            newInfo.BannerMac.Hash.Should().BeEquivalentTo(originalInfo.BannerMac.Hash);
+        }
+
+        [TestCaseSource(nameof(GetFiles))]
+        public void ReadWriteThreeWaysRomWithKeyGeneratesSameHashes(string infoPath, string romPath)
+        {
+            TestDataBase.IgnoreIfFileDoesNotExist(romPath);
+            DsiKeyStore keys = TestDataBase.GetDsiKeyStore();
+
+            using Node node = NodeFactory.FromFile(romPath, FileOpenMode.Read);
+
+            node.Invoking(n => n.TransformWith<Binary2NitroRom>()).Should().NotThrow();
+            ProgramInfo originalInfo = node.GetFormatAs<NitroRom>().Information;
+
+            var nitroParameters = new NitroRom2BinaryParams { KeyStore = keys };
+            node.Invoking(n => n.TransformWith<NitroRom2Binary, NitroRom2BinaryParams>(nitroParameters)).Should().NotThrow();
+
+            node.Invoking(n => n.TransformWith<Binary2NitroRom>()).Should().NotThrow();
+            ProgramInfo newInfo = node.GetFormatAs<NitroRom>().Information;
+
+            newInfo.OverlaysMac.Hash.Should().BeEquivalentTo(originalInfo.OverlaysMac.Hash);
+            newInfo.BannerMac.Hash.Should().BeEquivalentTo(originalInfo.BannerMac.Hash);
         }
     }
 }

src/Ekona.Tests/Containers/Rom/Binary2RomHeaderTests.cs

@@ -1,4 +1,4 @@
-// Copyright(c) 2021 SceneGate
+// Copyright(c) 2021 SceneGate
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -23,6 +23,7 @@
 using FluentAssertions;
 using NUnit.Framework;
 using SceneGate.Ekona.Containers.Rom;
+using SceneGate.Ekona.Security;
 using YamlDotNet.Serialization;
 using YamlDotNet.Serialization.NamingConventions;
 using Yarhl.FileFormat;
@@ -63,7 +64,43 @@ public void DeserializeRomHeaderMatchInfo(string infoPath, string headerPath)
 
             node.GetFormatAs<RomHeader>().Should().BeEquivalentTo(
                 expected,
-                opts => opts.Excluding(p => p.CopyrightLogo));
+                opts => opts
+                    .Excluding(p => p.CopyrightLogo)
+                    .Excluding((FluentAssertions.Equivalency.IMemberInfo info) => info.Type == typeof(HashInfo)));
+        }
+
+        [TestCaseSource(nameof(GetFiles))]
+        public void DeserializeRomHeaderWithoutKeysHasHashes(string infoPath, string headerPath)
+        {
+            TestDataBase.IgnoreIfFileDoesNotExist(infoPath);
+            TestDataBase.IgnoreIfFileDoesNotExist(headerPath);
+
+            using Node node = NodeFactory.FromFile(headerPath, FileOpenMode.Read);
+            node.Invoking(n => n.TransformWith<Binary2RomHeader>()).Should().NotThrow();
+
+            ProgramInfo programInfo = node.GetFormatAs<RomHeader>().ProgramInfo;
+            programInfo.ChecksumSecureArea.Status.Should().Be(HashStatus.NotValidated);
+            programInfo.ChecksumLogo.Status.Should().Be(HashStatus.Valid);
+            programInfo.ChecksumHeader.Status.Should().Be(HashStatus.Valid);
+
+            bool isDsi = programInfo.UnitCode != DeviceUnitKind.DS;
+            if (isDsi || programInfo.ProgramFeatures.HasFlag(DsiRomFeatures.BannerSigned)) {
+                programInfo.BannerMac.Should().NotBeNull();
+                programInfo.BannerMac.Status.Should().Be(HashStatus.NotValidated);
+            }
+
+            if (programInfo.ProgramFeatures.HasFlag(DsiRomFeatures.ProgramSigned)) {
+                programInfo.OverlaysMac.Should().NotBeNull();
+                programInfo.OverlaysMac.Status.Should().Be(HashStatus.NotValidated);
+
+                programInfo.ProgramMac.Should().NotBeNull();
+                programInfo.ProgramMac.Status.Should().Be(HashStatus.NotValidated);
+            }
+
+            if (isDsi || programInfo.ProgramFeatures.HasFlag(DsiRomFeatures.ProgramSigned)) {
+                programInfo.Signature.Should().NotBeNull();
+                programInfo.Signature.Status.Should().Be(HashStatus.NotValidated);
+            }
         }
 
         [TestCaseSource(nameof(GetFiles))]
@@ -79,8 +116,10 @@ public void TwoWaysIdenticalRomHeaderStream(string infoPath, string headerPath)
             var originalStream = new DataStream(node.Stream!, 0, header.SectionInfo.HeaderSize);
             generatedStream.Stream.Length.Should().Be(originalStream.Length);
 
-            // TODO: After DSi support
-            // generatedStream.Stream.Compare(originalStream).Should().BeTrue()
+            // TODO: Enable after adding the DSi flags
+            if (header.ProgramInfo.UnitCode == DeviceUnitKind.DS) {
+                generatedStream.Stream.Compare(originalStream).Should().BeTrue();
+            }
         }
 
         [TestCaseSource(nameof(GetFiles))]
@@ -91,13 +130,10 @@ public void ThreeWaysIdenticalRomHeaderObjects(string infoPath, string headerPat
             using Node node = NodeFactory.FromFile(headerPath, FileOpenMode.Read);
 
             var originalHeader = (RomHeader)ConvertFormat.With<Binary2RomHeader>(node.Format!);
-            var generatedStream = (BinaryFormat)ConvertFormat.With<RomHeader2Binary>(originalHeader);
+            using var generatedStream = (BinaryFormat)ConvertFormat.With<RomHeader2Binary>(originalHeader);
             var generatedHeader = (RomHeader)ConvertFormat.With<Binary2RomHeader>(generatedStream);
 
-            // Ignore ChecksumHeader as we are not generating identical headers due to DSi flags yet (#9).
-            generatedHeader.Should().BeEquivalentTo(
-                originalHeader,
-                opts => opts.Excluding(p => p.ProgramInfo.ChecksumHeader));
+            generatedHeader.Should().BeEquivalentTo(originalHeader);
         }
     }
 }

src/Ekona.Tests/TestDataBase.cs

@@ -1,4 +1,4 @@
-// Copyright(c) 2021 SceneGate
+// Copyright(c) 2021 SceneGate
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -22,6 +22,8 @@
 using System.IO;
 using System.Linq;
 using NUnit.Framework;
+using SceneGate.Ekona.Security;
+using YamlDotNet.Serialization;
 
 namespace SceneGate.Ekona.Tests
 {
@@ -40,6 +42,17 @@ public static string RootFromOutputPath {
             }
         }
 
+        public static DsiKeyStore GetDsiKeyStore()
+        {
+            string keysPath = Path.Combine(RootFromOutputPath, "dsi_keys.yml");
+            TestDataBase.IgnoreIfFileDoesNotExist(keysPath);
+
+            string keysYaml = File.ReadAllText(keysPath);
+            return new DeserializerBuilder()
+                .Build()
+                .Deserialize<DsiKeyStore>(keysYaml);
+        }
+
         public static void IgnoreIfFileDoesNotExist(string file)
         {
             if (!File.Exists(file)) {

src/Ekona/Containers/Rom/ArmEncodeFieldFinder.cs

@@ -46,7 +46,7 @@ public static class ArmEncodeFieldFinder
         /// <param name="arm">The ARM file to analyze.</param>
         /// <param name="info">The information of the program.</param>
         /// <returns>The encoded size address. 0 if not found.</returns>
-        public static int SearchEncodedSizeAddress(IBinary arm, RomInfo info)
+        public static int SearchEncodedSizeAddress(IBinary arm, ProgramInfo info)
         {
             // Steps to find the ARM9 size address that we need to change
             // in order to fix the BLZ decoded error.

src/Ekona/Containers/Rom/Banner.cs

@@ -18,6 +18,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 using System;
+using SceneGate.Ekona.Security;
 using Yarhl.FileFormat;
 
 namespace SceneGate.Ekona.Containers.Rom
@@ -65,27 +66,27 @@ public Version Version {
         /// Gets or sets the CRC-16 checksum for the banner binary data of version 0.1.
         /// </summary>
         /// <remarks>This field may be null if the model was not deserialized from binary data.</remarks>
-        public ChecksumInfo<ushort> ChecksumBase { get; set; }
+        public HashInfo ChecksumBase { get; set; }
 
         /// <summary>
         /// Gets or sets the CRC-16 checksum for the banner binary data of version 0.2
         /// that includes Chinese title.
         /// </summary>
         /// <remarks>This field may be null if the model was not deserialized from binary data.</remarks>
-        public ChecksumInfo<ushort> ChecksumChinese { get; set; }
+        public HashInfo ChecksumChinese { get; set; }
 
         /// <summary>
         /// Gets or sets the CRC-16 checksum for the banner binary data of version 0.3
         /// that includes Chinese and Korean titles.
         /// </summary>
         /// <remarks>This field may be null if the model was not deserialized from binary data.</remarks>
-        public ChecksumInfo<ushort> ChecksumKorean { get; set; }
+        public HashInfo ChecksumKorean { get; set; }
 
         /// <summary>
         /// Gets or sets the CRC-16 checksum for the animated DSi icon.
         /// </summary>
         /// <remarks>This field may be null if the model was not deserialized from binary data.</remarks>
-        public ChecksumInfo<ushort> ChecksumAnimatedIcon { get; set; }
+        public HashInfo ChecksumAnimatedIcon { get; set; }
 
         /// <summary>
         /// Gets or sets the Japenese title.