Fix serialisation of isBeta and isExternal properties

The isBeta and isExternal properties weren't being serialised as part of `NavigatorItem`.

These properties are used to initialise to `RenderIndex.Node` during the conversion to `index.json` [1] and must be preserved when navigator indexes are written to disk [2] so that when they are read [3], we don't drop beta and external information.

This serialisation happens during the `finalize` step [4] and the deserialisation can be invoked via `NavigatorIndex.readNavigatorIndex` [5].

Otherwise, the values can be lost on serialisation roundtrip.

[1]: https://github.com/swiftlang/swift-docc/blob/65aaf926ec079ddbd40f29540d4180a70af99e5e/Sources/SwiftDocC/Indexing/RenderIndexJSON/RenderIndex.swift#L329
[2]: https://github.com/swiftlang/swift-docc/blob/65aaf926ec079ddbd40f29540d4180a70af99e5e/Sources/SwiftDocC/Indexing/Navigator/NavigatorTree.swift#L195
[3]: https://github.com/swiftlang/swift-docc/blob/65aaf926ec079ddbd40f29540d4180a70af99e5e/Sources/SwiftDocC/Indexing/Navigator/NavigatorTree.swift#L266
[4]: https://github.com/swiftlang/swift-docc/blob/65aaf926ec079ddbd40f29540d4180a70af99e5e/Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex.swift#L1193
[5]: https://github.com/swiftlang/swift-docc/blob/65aaf926ec079ddbd40f29540d4180a70af99e5e/Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex.swift#L157

Author: anferbui

Sources/SwiftDocC/Indexing/Navigator/NavigatorItem.swift

@@ -146,8 +146,27 @@ public final class NavigatorItem: Serializable, Codable, Equatable, CustomString
 
         let pathData = data[cursor..<cursor + Int(pathLength)]
         self.path = String(data: pathData, encoding: .utf8)!
+        cursor += Int(pathLength)
 
-        assert(cursor+Int(pathLength) == data.count)
+        // isBeta and isExternal should be encoded because they are relevant when creating a RenderIndex node.
+        // Without proper serialization, these indicators would be lost when navigator indexes are loaded from disk.
+        
+        length = MemoryLayout<UInt8>.stride
+        // To ensure backwards compatibility, handle both when `isBeta` has been encoded and when it hasn't
+        if cursor < data.count {
+            let betaValue: UInt8 = unpackedValueFromData(data[cursor..<cursor + length])
+            cursor += length
+            self.isBeta = betaValue != 0
+        }
+
+        // To ensure backwards compatibility, handle both when `isExternal` has been encoded and when it hasn't
+        if cursor < data.count {
+            let externalValue: UInt8 = unpackedValueFromData(data[cursor..<cursor + length])
+            cursor += length
+            self.isExternal = externalValue != 0
+        }
+
+        assert(cursor == data.count)
     }
 
     /// Returns the `Data` representation of the current `NavigatorItem` instance.
@@ -164,6 +183,9 @@ public final class NavigatorItem: Serializable, Codable, Equatable, CustomString
         data.append(Data(title.utf8))
         data.append(Data(path.utf8))
 
+        data.append(packedDataFromValue(isBeta ? UInt8(1) : UInt8(0)))
+        data.append(packedDataFromValue(isExternal ? UInt8(1) : UInt8(0)))
+        
         return data
     }
 
@@ -176,7 +198,9 @@ public final class NavigatorItem: Serializable, Codable, Equatable, CustomString
             languageID: \(languageID),
             title: \(title),
             platformMask: \(platformMask),
-            availabilityID: \(availabilityID)
+            availabilityID: \(availabilityID),
+            isBeta: \(isBeta),
+            isExternal: \(isExternal)
         }
         """
     }

Tests/SwiftDocCTests/Indexing/NavigatorIndexTests.swift

@@ -138,6 +138,66 @@ Root
         XCTAssertEqual(item, fromData)
     }
 
+    func testNavigatorEquality() {
+        // Test for equal
+        var item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024, isExternal: true, isBeta: true)
+        var item2 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024, isExternal: true, isBeta: true)
+        XCTAssertEqual(item1, item2)
+
+        // Tests for not equal
+        item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024, isBeta: true)
+        item2 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024, isBeta: false)
+        XCTAssertNotEqual(item1, item2)
+
+        item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024, isExternal: true)
+        item2 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024, isExternal: false)
+        XCTAssertNotEqual(item1, item2)
+        
+        item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024)
+        item2 = NavigatorItem(pageType: 2, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024)
+        XCTAssertNotEqual(item1, item2)
+
+        item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024)
+        item2 = NavigatorItem(pageType: 1, languageID: 5, title: "My Title", platformMask: 256, availabilityID: 1024)
+        XCTAssertNotEqual(item1, item2)
+        
+        item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024)
+        item2 = NavigatorItem(pageType: 1, languageID: 4, title: "My Other Title", platformMask: 256, availabilityID: 1024)
+        XCTAssertNotEqual(item1, item2)
+        
+        item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024)
+        item2 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 257, availabilityID: 1024)
+        XCTAssertNotEqual(item1, item2)
+
+        item1 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024)
+        item2 = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1025)
+        XCTAssertNotEqual(item1, item2)
+    }
+    
+    func testNavigatorItemRawDumpWithExtraProperties() {
+        let item = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024, isExternal: true, isBeta: true)
+        let data = item.rawValue
+        let fromData = NavigatorItem(rawValue: data)
+        XCTAssertEqual(item, fromData)
+    }
+    
+    func testNavigatorItemRawDumpBackwardCompatibility() {
+        let item = NavigatorItem(pageType: 1, languageID: 4, title: "My Title", platformMask: 256, availabilityID: 1024)
+        var data = Data()
+        data.append(packedDataFromValue(item.pageType))
+        data.append(packedDataFromValue(item.languageID))
+        data.append(packedDataFromValue(item.platformMask))
+        data.append(packedDataFromValue(item.availabilityID))
+        data.append(packedDataFromValue(UInt64(item.title.utf8.count)))
+        data.append(packedDataFromValue(UInt64(item.path.utf8.count)))
+        data.append(Data(item.title.utf8))
+        data.append(Data(item.path.utf8))
+        // Note: NOT adding isBeta and isExternal flags to simulate when they were not supported
+
+        let fromData = NavigatorItem(rawValue: data)
+        XCTAssertEqual(item, fromData)
+    }
+    
     func testObjCLanguage() {
         let root = generateLargeTree()
         var objcFiltered: Node?