Xref table expose

examples/AdvancedMerge.cs

@@ -0,0 +1,205 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using UglyToad.PdfPig;
+using UglyToad.PdfPig.Core;
+using UglyToad.PdfPig.Tokens;
+using UglyToad.PdfPig.Writer;
+
+namespace UglyToad.Examples;
+
+public class AdvancedMerge
+{
+    public static void Run(Stream input1, Stream output)
+    {
+        using var pdf = PdfDocument.Open(input1);
+        var pdfObjects = pdf.Structure
+            .CrossReferenceTable
+            .ObjectOffsets
+            .ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
+
+        if (!pdf.Structure.Catalog.CatalogDictionary.TryGet<IndirectReferenceToken>(NameToken.Pages, out var pages))
+            throw new ArgumentException("No pages reference were found");
+
+        if (ResolveIndirect(pdf, pages) is not DictionaryToken pagesObj)
+            throw new ArgumentException("No pages object were found");
+
+        // Assume, we have only 1 page in here
+        if (!pagesObj.TryGet(NameToken.Kids, out ArrayToken kids) || kids.Length != 1)
+            throw new ArgumentException("Invalid catalog dictionary");
+
+        var kidReference = kids.Data[0] as IndirectReferenceToken;
+        if (ResolveIndirect(pdf, kidReference) is not DictionaryToken pageObj)
+            throw new ArgumentException("Invalid catalog dictionary");
+
+
+        // Skip all pdf meta structure objects 
+        var skippedRefs = new HashSet<IndirectReference>
+        {
+            pages.Data,  // Pages
+            kidReference!.Data,  // Page
+            pdf.Structure.Trailer.Root,  // Catalog
+        };
+
+        // Skip all refs from "skippedRefs" and order it by object number
+        var oldRefs = pdf.Structure.CrossReferenceTable.ObjectOffsets.Keys
+            .Where(k => !skippedRefs.Contains(k))
+            .OrderBy(k => k.ObjectNumber)
+            .ToList();
+
+        // Building refs map, to rebind old objects to their new values
+        var refMap = new Dictionary<IndirectReference, IndirectReference>();
+        var currentObjectNumber = pdf.Structure.Trailer.Size;
+        foreach (var oldRef in oldRefs)
+        {
+            var newRef = new IndirectReference(currentObjectNumber++, 0);
+            refMap[oldRef] = newRef;
+        }
+
+        // Here you can extract page content and bind it in output page document
+        IndirectReference? root = null;
+        if (pageObj.TryGet<IndirectReferenceToken>(NameToken.Contents, out var contentObj))
+        {
+            using var outputPdf = PdfDocument.Open(output);
+            var lastPageRef = FindLastPage(outputPdf);
+
+            if (ResolveIndirect(outputPdf, lastPageRef) is not DictionaryToken outputPage)
+                throw new ArgumentException("Invalid catalog dictionary");
+            root = outputPdf.Structure.Trailer.Root;
+
+            IReadOnlyList<IToken> newContents = [new IndirectReferenceToken(refMap[contentObj.Data])];
+
+            if (outputPage.TryGet<ArrayToken>(NameToken.Contents, out var oldContents))
+                newContents = newContents.Concat(oldContents.Data).ToList();
+
+            output.Seek(0, SeekOrigin.End);
+
+            var xrefLocation = XrefLocation.File(output.Position);
+            var newPageObject = outputPage.With(NameToken.Contents, new ArrayToken(newContents));
+            TokenWriter.Instance.WriteToken(new ObjectToken(xrefLocation, lastPageRef.Data, newPageObject), output);
+
+            pdfObjects[lastPageRef.Data] = xrefLocation;
+        }
+
+        foreach (var oldRef in oldRefs)
+        {
+            var newObjRef = refMap[oldRef];
+            var newXref = XrefLocation.File(output.Position);
+            var token = ResolveIndirect(pdf, oldRef);
+            var updatedToken = ReplaceReferences(token, refMap);
+
+            pdfObjects[newObjRef] = newXref;
+            output.Seek(0, SeekOrigin.End);
+            TokenWriter.Instance.WriteToken(new ObjectToken(newXref, newObjRef, updatedToken), output);
+        }
+
+        // Writer new xref table
+        output.Seek(0, SeekOrigin.End);
+        TokenWriter.Instance.WriteCrossReferenceTable(
+            pdfObjects.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value1),
+            root ?? pdf.Structure.Trailer.Root,
+            output,
+            null,
+            pdf.Structure.Trailer.PreviousCrossReferenceOffset);
+    }
+
+    /// <summary>
+    /// Recursively replaces IndirectReferenceToken in the token tree according to the map.
+    /// </summary>
+    private static IToken ReplaceReferences(IToken token, Dictionary<IndirectReference, IndirectReference> mapping)
+    {
+        return token switch
+        {
+            IndirectReferenceToken irt => mapping.TryGetValue(irt.Data, out var newRef) ? new IndirectReferenceToken(newRef) : token,
+            DictionaryToken dict => ReplaceDictionary(dict, mapping),
+            ArrayToken arr => ReplaceArray(arr, mapping),
+            StreamToken stream => ReplaceStream(stream, mapping),
+            _ => token
+        };
+    }
+
+    private static DictionaryToken ReplaceDictionary(DictionaryToken original, Dictionary<IndirectReference, IndirectReference> mapping)
+    {
+        var newDict = new Dictionary<NameToken, IToken>(original.Data.Count);
+        foreach (var kvp in original.Data)
+        {
+            newDict[NameToken.Create(kvp.Key)] = ReplaceReferences(kvp.Value, mapping);
+        }
+        return new DictionaryToken(newDict);
+    }
+
+    private static ArrayToken ReplaceArray(ArrayToken original, Dictionary<IndirectReference, IndirectReference> mapping)
+    {
+        var newData = new IToken[original.Length];
+        for (int i = 0; i < original.Length; i++)
+        {
+            newData[i] = ReplaceReferences(original.Data[i], mapping);
+        }
+        return new ArrayToken(newData);
+    }
+
+    private static StreamToken ReplaceStream(StreamToken original, Dictionary<IndirectReference, IndirectReference> mapping)
+    {
+        var updatedDict = ReplaceDictionary(original.StreamDictionary, mapping);
+        // We create a new StreamToken with the replaced dictionary, preserving the original byte stream.
+        return new StreamToken(updatedDict, original.Data);
+    }
+
+    private const int MaxIndirectResolutionDepth = 32;
+
+    private static IndirectReferenceToken FindLastPage(PdfDocument pdf)
+    {
+        if (!pdf.Structure.Catalog.CatalogDictionary.TryGet<IndirectReferenceToken>(NameToken.Pages, out var pagesRef))
+            throw new ArgumentException("No pages were found in the input document.");
+
+        if (ResolveIndirect(pdf, pagesRef) is not DictionaryToken pages)
+            throw new ArgumentException("No pages were found in the input file.");
+
+        if (!pages.TryGet<ArrayToken>(NameToken.Kids, out var kids))
+            throw new ArgumentException("No pages were found in the input document.");
+
+        return FindLastPage(pdf, kids);
+    }
+
+    private static IndirectReferenceToken FindLastPage(PdfDocument pdf, ArrayToken pageTree)
+    {
+        while (true)
+        {
+            if (pageTree.Length == 0) 
+                throw new ArgumentException("No leaf in page tree");
+
+            var root = pageTree.Data.Last()!;
+            if (ResolveIndirect(pdf, root) is not DictionaryToken newRoot) 
+                throw new ArgumentException("Indirect page tree");
+
+            if (newRoot.Data[NameToken.Type] is not NameToken type) 
+                throw new ArgumentException("Indirect page tree");
+
+            if (type.Data == NameToken.Page) 
+                return (root as IndirectReferenceToken)!;
+            pageTree = (newRoot.Data[NameToken.Kids] as ArrayToken)!;
+        }
+    }
+
+    private static IToken ResolveIndirect(PdfDocument doc, IndirectReference reference)
+    {
+        return ResolveIndirect(doc, new IndirectReferenceToken(reference));
+    }
+
+    private static IToken ResolveIndirect(PdfDocument doc, IToken token)
+    {
+        var depth = 0;
+        while (token is IndirectReferenceToken ir)
+        {
+            if (++depth > MaxIndirectResolutionDepth)
+                throw new ArgumentException(
+                    "Cyclic or excessively deep indirect reference in PDF signature dictionary.");
+
+            var obj = doc.Structure.GetObject(ir.Data);
+            token = obj.Data ?? throw new ArgumentException("Failed to parse PDF digital signature.");
+        }
+
+        return token;
+    }
+}

examples/Program.cs

@@ -47,12 +47,24 @@ public static void Main()
                     ("Advance text extraction using layout analysis algorithms",
                     () => AdvancedTextExtraction.Run(Path.Combine(filesDirectory, "ICML03-081.pdf")))                
                 },
-                {
-                8,
+                {8,
                     ("Extract Words with newline detection (example with algorithm). Issue 512",
                     () => OpenDocumentAndExtractWords.Run(Path.Combine(filesDirectory, "OPEN.RABBIT.ENGLISH.LOP.pdf")))
-                } 
-        };
+                } ,
+                {9,
+                    ("Advanced pdf merge, using low level pdf tools, like trailer dictionary and xref table",
+                    () =>
+                    {
+                        using var input2 = File.Open(Path.Combine(filesDirectory, "EmptyPdf.pdf"), FileMode.Open);
+                        using var output = new FileStream(Path.Combine(filesDirectory, "AdvancedMergeResult.pdf"), FileMode.Create);
+                        using var input = File.Open(Path.Combine(filesDirectory, "Single Page Simple - from google drive.pdf"), FileMode.Open);
+
+                        input2.CopyToAsync(output);
+                        output.Seek(0, SeekOrigin.Begin);
+                        AdvancedMerge.Run(input, output);
+                    })
+                }
+            };
 
             var choices = string.Join(Environment.NewLine, examples.Select(x => $"{x.Key}: {x.Value.name}"));
 

src/UglyToad.PdfPig.Tests/Integration/IntegrationDocumentTests.cs

@@ -92,13 +92,13 @@ public void CanTokenizeAllAccessibleObjects(string documentName)
             {
                 Assert.NotNull(document.Structure.Catalog);
 
-                //Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count > 0, "Cross reference table was empty.");
-                //foreach (var objectOffset in document.Structure.CrossReferenceTable.ObjectOffsets)
-                //{
-                //    var token = document.Structure.GetObject(objectOffset.Key);
+                Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count > 0, "Cross reference table was empty.");
+                foreach (var objectOffset in document.Structure.CrossReferenceTable.ObjectOffsets)
+                {
+                    var token = document.Structure.GetObject(objectOffset.Key);
 
-                //    Assert.NotNull(token);
-                //}
+                    Assert.NotNull(token);
+                }
             }
         }
 

src/UglyToad.PdfPig.Tests/PublicApiScannerTests.cs

@@ -96,6 +96,7 @@ public void OnlyExposedApiIsPublic()
                 "UglyToad.PdfPig.Content.TextOrientation",
                 "UglyToad.PdfPig.Content.XmpMetadata",
                 "UglyToad.PdfPig.CrossReference.CrossReferenceTable",
+                "UglyToad.PdfPig.CrossReference.CrossReferenceTablePart",
                 "UglyToad.PdfPig.CrossReference.CrossReferenceType",
                 "UglyToad.PdfPig.CrossReference.TrailerDictionary",
                 "UglyToad.PdfPig.Exceptions.PdfDocumentEncryptedException",

src/UglyToad.PdfPig.Tests/Writer/PdfDocumentBuilderTests.cs

@@ -174,12 +174,12 @@ public void CanReadSingleBlankPage()
 
                 Assert.NotNull(document.Structure.Catalog);
 
-                //foreach (var offset in document.Structure.CrossReferenceTable.ObjectOffsets)
-                //{
-                //    var obj = document.Structure.GetObject(offset.Key);
-
-                //    Assert.NotNull(obj);
-                //}
+                foreach (var offset in document.Structure.CrossReferenceTable.ObjectOffsets)
+                {
+                    var obj = document.Structure.GetObject(offset.Key);
+                
+                    Assert.NotNull(obj);
+                }
             }
         }
 
@@ -988,8 +988,8 @@ public void CanDedupObjectsFromSameDoc_Builder()
                 using (var document = PdfDocument.Open(result, ParsingOptions.LenientParsingOff))
                 {
                     Assert.Equal(2, document.NumberOfPages);
-                    // Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 29,
-                    //  "Expected object count to be lower than 30"); // 45 objects with duplicates, 29 with correct re-use
+                    Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 29,
+                     "Expected object count to be lower than 30"); // 45 objects with duplicates, 29 with correct re-use
                 }
             }
         }
@@ -1010,8 +1010,8 @@ public void CanDedupObjectsFromDifferentDoc_HashBuilder()
                 using (var document = PdfDocument.Open(result, ParsingOptions.LenientParsingOff))
                 {
                     Assert.Equal(2, document.NumberOfPages);
-                    // Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 29,
-                       // "Expected object count to be lower than 30"); // 45 objects with duplicates, 29 with correct re-use
+                    Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 29,
+                       "Expected object count to be lower than 30"); // 45 objects with duplicates, 29 with correct re-use
                 }
             }
         }
@@ -1404,10 +1404,12 @@ public void WriteObject(long objectNumber, int generation, byte[] data, Stream o
             Objects++;
         }
 
-        public void WriteCrossReferenceTable(IReadOnlyDictionary<IndirectReference, long> objectOffsets,
+        public void WriteCrossReferenceTable(
+            IReadOnlyDictionary<IndirectReference, long> objectOffsets,
             IndirectReference catalogToken,
             Stream outputStream,
-            IndirectReference? documentInformationReference)
+            IndirectReference? documentInformationReference,
+            long? prevTableLocation)
         {
             WroteCrossReferenceTable = true;
         }

src/UglyToad.PdfPig.Tests/Writer/PdfMergerTests.cs

@@ -96,8 +96,8 @@ public void ObjectCountLower()
             using (var document = PdfDocument.Open(result, ParsingOptions.LenientParsingOff))
             {
                 Assert.Equal(2, document.NumberOfPages);
-                // Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 24,
-                 //   "Expected object count to be lower than 24");
+                Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 24,
+                   "Expected object count to be lower than 24");
             }
         }
 
@@ -111,8 +111,8 @@ public void DedupsObjectsFromSameDoc()
             using (var document = PdfDocument.Open(result, ParsingOptions.LenientParsingOff))
             {
                 Assert.Equal(2, document.NumberOfPages);
-                // Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 29,
-                //    "Expected object count to be lower than 30"); // 45 objects with duplicates, 29 with correct re-use
+                Assert.True(document.Structure.CrossReferenceTable.ObjectOffsets.Count <= 29,
+                   "Expected object count to be lower than 30"); // 45 objects with duplicates, 29 with correct re-use
             }
         }