fix tracked offset with preserve spaces at last non-blank position

Author: vsch

Diff for: .idea/codeStyles/Project.xml

@@ -7,6 +7,7 @@
 - [Release 0.60.0](#release-0600)
   - [API Refactoring](#api-refactoring)
 - [Next 0.61.xx](#next-061xx)
+- [0.61.26](#06126)
 - [0.61.24](#06124)
 - [0.61.22](#06122)
 - [0.61.20](#06120)
@@ -224,6 +225,16 @@ Please give feedback on the upcoming changes if you have concerns about breaking
 * [ ] Fix: Html converter to not add spaces between end of inline marker and next punctuation:
       `.,:;`
 
+## 0.61.26
+
+* Fix: `SequenceBuilder.toString()` not to add space between sequence parts.
+* Break: `TextCollectingVisitor` needs `TextContainer.F_ADD_SPACES_BETWEEN_NODES` to ensure
+  there is at least one space between texts of different nodes in collected text. Previously
+  this was added automatically by sequence builder. Use one of the `getAndCollect` functions
+  taking options flags.
+* Fix: `MarkdownParagraph` wrapping with preserving tracked offsets with spaces to preserve
+  spaces right before last non-blank character.
+
 ## 0.61.24
 
 * Fix: link refs in link text should be collapsed.

Diff for: VERSION-TODO.md

@@ -1593,3 +1593,50 @@ BasedSegmentBuilder{[0, 104), s=0:0, u=1:1, t=1:1, l=105, sz=3, na=2: [0, 104),
 ````````````````````````````````
 
 
+```````````````````````````````` example(Wrap - Restore Spaces: 54) options(margin[76], insert-space, restore-tracked-spaces, show-ranges, running-tests)
+[B ⦙] 
+.
+[B ⦙]
+---- Tracked Offsets ---------------------------------------------------
+[0]: {3 1|0 si -> 3}
+
+---- Ranges ------------------------------------------------------------
+⟦[B ]⟧
+⟦⟧
+---- Segments ----------------------------------------------------------
+BasedSegmentBuilder{[0, 4), s=0:0, u=1:1, t=1:1, l=5, sz=3, na=2: [0, 4), a:'\n', [4) }
+````````````````````````````````
+
+
+```````````````````````````````` example(Wrap - Restore Spaces: 55) options(margin[76], show-ranges)
+[B ] 
+.
+[B]
+---- Ranges ------------------------------------------------------------
+⟦[B⟧⟦]⟧
+⟦⟧
+---- Segments ----------------------------------------------------------
+BasedSegmentBuilder{[0, 4), s=0:0, u=1:1, t=1:1, l=4, sz=4, na=3: [0, 2), [3, 4), a:'\n', [4) }
+.
+Document[0, 5]
+  Paragraph[0, 5]
+    LinkRef[0, 4] referenceOpen:[0, 1, "["] reference:[1, 2, "B"] referenceClose:[3, 4, "]"]
+      Text[1, 2] chars:[1, 2, "B"]
+````````````````````````````````
+
+
+```````````````````````````````` example(Wrap - Restore Spaces: 56) options(margin[76], restore-tracked-spaces, show-ranges)
+[B ⦙] 
+.
+[B ⦙]
+---- Tracked Offsets ---------------------------------------------------
+[0]: {3 1|0 -> 3}
+
+---- Ranges ------------------------------------------------------------
+⟦[B ]⟧
+⟦⟧
+---- Segments ----------------------------------------------------------
+BasedSegmentBuilder{[0, 4), s=0:0, u=1:1, t=1:1, l=5, sz=3, na=2: [0, 4), a:'\n', [4) }
+````````````````````````````````
+
+

Diff for: flexmark-core-test/src/test/resources/core_wrapping_spec.md

@@ -32,7 +32,7 @@ public void test_basic() {
                 "";
         Node document = parser.parse(markdown);
         TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
-        final String text = collectingVisitor.collectAndGetText(document);
+        final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_ADD_SPACES_BETWEEN_NODES);
         //System.out.println(text);
 
         //final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
@@ -61,7 +61,7 @@ public void test_linkURL() {
                 "";
         Node document = parser.parse(markdown);
         TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
-        final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL);
+        final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL | TextContainer.F_ADD_SPACES_BETWEEN_NODES);
         //System.out.println(text);
 
         //final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
@@ -86,7 +86,7 @@ public void test_linkNodeText() {
                 "";
         Node document = parser.parse(markdown);
         TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
-        final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_NODE_TEXT);
+        final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_NODE_TEXT | TextContainer.F_ADD_SPACES_BETWEEN_NODES);
         //System.out.println(text);
 
         //final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
@@ -111,14 +111,14 @@ public void test_linkUrlNodeText() {
                 "";
         Node document = parser.parse(markdown);
         TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
-        final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL | TextContainer.F_NODE_TEXT);
+        final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL | TextContainer.F_ADD_SPACES_BETWEEN_NODES);
         //System.out.println(text);
 
         //final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
         //System.out.println(astText);
         assertEquals("" +
                 "First Header Second Header\n" +
-                "**Content Cell** ![](image%20spaces.png)\n" +
+                "Content Cell image spaces.png\n" +
                 "", text);
     }
 }

Diff for: flexmark-ext-tables/src/test/java/com/vladsch/flexmark/ext/tables/TableTextCollectingVisitorTest.java

@@ -1,5 +1,6 @@
 package com.vladsch.flexmark.util.ast;
 
+import com.vladsch.flexmark.util.misc.BitFieldSet;
 import com.vladsch.flexmark.util.misc.CharPredicate;
 import com.vladsch.flexmark.util.sequence.BasedSequence;
 import com.vladsch.flexmark.util.sequence.Range;
@@ -13,36 +14,38 @@
 public class SpaceInsertingSequenceBuilder implements ISequenceBuilder<SpaceInsertingSequenceBuilder, BasedSequence> {
     @NotNull
     public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base) {
-        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base));
+        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base), false);
     }
 
     @NotNull
     public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base, @NotNull SegmentOptimizer optimizer) {
-        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, optimizer));
+        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, optimizer), false);
     }
 
     @NotNull
     public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base, int options) {
-        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options));
+        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options), BitFieldSet.any(options, TextContainer.F_ADD_SPACES_BETWEEN_NODES));
     }
 
     @NotNull
     public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base, int options, @NotNull SegmentOptimizer optimizer) {
-        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options, optimizer));
+        return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options, optimizer), BitFieldSet.any(options, TextContainer.F_ADD_SPACES_BETWEEN_NODES));
     }
 
     @NotNull
     public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull SequenceBuilder builder) {
-        return new SpaceInsertingSequenceBuilder(builder);
+        return new SpaceInsertingSequenceBuilder(builder, false);
     }
 
     final SequenceBuilder out;
     Node lastNode;
-    boolean addSpaceOnNonBlank;
     boolean needEol;
+    final boolean addSpacesBetweenNodes;
+    boolean addSpaces;
 
-    private SpaceInsertingSequenceBuilder(SequenceBuilder out) {
+    private SpaceInsertingSequenceBuilder(SequenceBuilder out, boolean addSpacesBetweenNodes) {
         this.out = out;
+        this.addSpacesBetweenNodes = addSpacesBetweenNodes;
     }
 
     public SequenceBuilder getOut() {
@@ -54,14 +57,6 @@ public char charAt(int index) {
         return out.charAt(index);
     }
 
-    public boolean isAddSpaceOnNonBlank() {
-        return addSpaceOnNonBlank;
-    }
-
-    public void setAddSpaceOnNonBlank(boolean addSpaceOnNonBlank) {
-        this.addSpaceOnNonBlank = addSpaceOnNonBlank;
-    }
-
     public boolean isNeedEol() {
         return needEol;
     }
@@ -79,10 +74,10 @@ public void setLastNode(Node lastNode) {
 
         if (this.lastNode != null && this.lastNode.getEndOffset() < lastNode.getStartOffset()) {
             BasedSequence sequence = getBaseSequence().subSequence(this.lastNode.getEndOffset(), lastNode.getStartOffset());
-            this.addSpaceOnNonBlank = sequence.indexOfAny(CharPredicate.SPACE_TAB_EOL) != -1;
             this.needEol = sequence.trim(CharPredicate.SPACE_TAB).length() > 0 && sequence.trim(CharPredicate.WHITESPACE).isEmpty();
         }
 
+        addSpaces = addSpacesBetweenNodes;
         this.lastNode = lastNode;
     }
 
@@ -155,14 +150,14 @@ public boolean needEol() {
 
     @Override
     @NotNull
-    public SpaceInsertingSequenceBuilder getBuilder() {return new SpaceInsertingSequenceBuilder(out.getBuilder());}
+    public SpaceInsertingSequenceBuilder getBuilder() {return new SpaceInsertingSequenceBuilder(out.getBuilder(), addSpacesBetweenNodes);}
 
     @Override
     @NotNull
     public SpaceInsertingSequenceBuilder append(@Nullable CharSequence chars, int startIndex, int endIndex) {
-        if (addSpaceOnNonBlank && chars != null && startIndex < endIndex && !CharPredicate.WHITESPACE.test(chars.charAt(startIndex)) && needSpace()) {
+        if (addSpaces && chars != null && startIndex < endIndex && !CharPredicate.WHITESPACE.test(chars.charAt(startIndex)) && needSpace()) {
             out.append(' ');
-            addSpaceOnNonBlank = false;
+            addSpaces = false;
         }
         out.append(chars, startIndex, endIndex);
         return this;
@@ -171,9 +166,9 @@ public SpaceInsertingSequenceBuilder append(@Nullable CharSequence chars, int st
     @Override
     @NotNull
     public SpaceInsertingSequenceBuilder append(char c) {
-        if (addSpaceOnNonBlank && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
+        if (addSpaces && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
             out.append(' ');
-            addSpaceOnNonBlank = false;
+            addSpaces = false;
         }
         out.append(c);
         return this;
@@ -182,19 +177,19 @@ public SpaceInsertingSequenceBuilder append(char c) {
     @Override
     @NotNull
     public SpaceInsertingSequenceBuilder append(char c, int count) {
-        if (addSpaceOnNonBlank && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
+        if (addSpaces && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
             out.append(' ');
-            addSpaceOnNonBlank = false;
+            addSpaces = false;
         }
         out.append(c, count);
         return this;
     }
 
     @NotNull
     public SpaceInsertingSequenceBuilder append(int startOffset, int endOffset) {
-        if (addSpaceOnNonBlank && startOffset < endOffset && !CharPredicate.WHITESPACE.test(out.getBaseSequence().charAt(startOffset)) && needSpace()) {
+        if (addSpaces && startOffset < endOffset && !CharPredicate.WHITESPACE.test(out.getBaseSequence().charAt(startOffset)) && needSpace()) {
             out.append(' ');
-            addSpaceOnNonBlank = false;
+            addSpaces = false;
         }
         out.append(startOffset, endOffset);
         return this;

Diff for: flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/SpaceInsertingSequenceBuilder.java

@@ -62,7 +62,7 @@ public BasedSequence collectAndGetSequence(Node node) {
     }
 
     public void collect(Node node, int flags) {
-        out = SpaceInsertingSequenceBuilder.emptyBuilder(node.getChars());
+        out = SpaceInsertingSequenceBuilder.emptyBuilder(node.getChars(), flags);
         this.flags = flags;
         myVisitor.visit(node);
     }

Diff for: flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/TextCollectingVisitor.java

@@ -12,6 +12,7 @@ enum Flags implements BitField {
         FOR_HEADING_ID,  // text for heading ID
         NO_TRIM_REF_TEXT_START,  // don't trim ref text start
         NO_TRIM_REF_TEXT_END,  // don't trim ref text end
+        ADD_SPACES_BETWEEN_NODES,  // when appending text from different nodes, ensure there is at least one space
         ;
 
         final int bits;
@@ -41,6 +42,7 @@ public int getBits() {
     int F_FOR_HEADING_ID = BitFieldSet.intMask(Flags.FOR_HEADING_ID);
     int F_NO_TRIM_REF_TEXT_START = BitFieldSet.intMask(Flags.NO_TRIM_REF_TEXT_START);
     int F_NO_TRIM_REF_TEXT_END = BitFieldSet.intMask(Flags.NO_TRIM_REF_TEXT_END);
+    int F_ADD_SPACES_BETWEEN_NODES = BitFieldSet.intMask(Flags.ADD_SPACES_BETWEEN_NODES);
 
     /**
      * Append node's text

Diff for: flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/TextContainer.java

@@ -371,7 +371,7 @@ BasedSequence resolveTrackedOffsetsEdit(BasedSequence baseSpliced, BasedSequence
 
             if (addSpacesBefore + addSpacesAfter > 0) {
                 int lastNonBlank = wrapped.lastIndexOfAnyNot(WHITESPACE_NBSP);
-                if (wrappedIndex < lastNonBlank) {
+                if (wrappedIndex <= lastNonBlank) {
                     // insert in middle
                     wrapped = wrapped.insert(wrappedIndex, RepeatedSequence.ofSpaces(addSpacesBefore + addSpacesAfter));
 

Diff for: flexmark-util-format/src/main/java/com/vladsch/flexmark/util/format/MarkdownParagraph.java

@@ -337,14 +337,6 @@ public String toString() {
                 BasedSequence s = baseSeq.subSequence(((Range) part).getStart(), ((Range) part).getEnd());
 
                 if (s.isNotEmpty()) {
-                    if (last != null && last.isNotEmpty() && last.getEndOffset() < s.getStartOffset()
-                            && (BasedSequence.WHITESPACE.indexOf(last.charAt(last.length() - 1)) == -1)
-                            && BasedSequence.WHITESPACE.indexOf(s.charAt(0)) == -1
-                            && s.baseSubSequence(last.getEndOffset(), s.getStartOffset()).endsWith(" ")
-                    ) {
-                        sb.append(' ');
-                    }
-
                     s.appendTo(sb);
                 }
 

Diff for: flexmark-util-sequence/src/main/java/com/vladsch/flexmark/util/sequence/builder/SequenceBuilder.java

@@ -292,7 +292,9 @@ public void addSegments(@NotNull IBasedSegmentBuilder<?> builder, int startIndex
             builder.appendAnchor(startOffset);
         }
 
+        int currentEnd = startOffset;
         BasedSequence baseSequence = builder.getBaseSequence();
+
         for (int i = startPos; i < endPos; i++) {
             Segment segment = getSegment(i, baseSequence);
 
@@ -318,12 +320,14 @@ public void addSegments(@NotNull IBasedSegmentBuilder<?> builder, int startIndex
                 }
             } else {
                 assert charSequence instanceof BasedSequence;
-                builder.append(((BasedSequence) charSequence).getStartOffset(), ((BasedSequence) charSequence).getEndOffset());
+                BasedSequence basedSequence = (BasedSequence) charSequence;
+                currentEnd = Math.max(currentEnd, basedSequence.getEndOffset());
+                builder.append(basedSequence.getStartOffset(), basedSequence.getEndOffset());
             }
         }
 
         if (endOffset != -1) {
-            builder.appendAnchor(endOffset);
+            builder.appendAnchor(Math.max(currentEnd, endOffset));
         }
     }
 
@@ -335,7 +339,6 @@ public void addSegments(@NotNull IBasedSegmentBuilder<?> builder, int startIndex
      * @param endIndex   end index of sub-sequence of segment tree
      * @param startPos   start pos of sub-sequence segments  in tree
      * @param endPos     end  pos of sub-sequence segments  in tree
-     *
      * @return subsequence of segment corresponding to part of it which is in the sub-sequence of the tree
      */
     @NotNull
@@ -461,7 +464,7 @@ public static SegmentTreePos findSegmentPos(int index, int[] treeData, int start
             }
 
             assert lastStart != startPos || lastEnd != endPos : "Range and position did not change after iteration: pos=" + pos + ", startPos=" + startPos + ", endPos=" + endPos
-                    + "\n" + Arrays.toString(treeData) 
+                    + "\n" + Arrays.toString(treeData)
                     ;
         }
         return null;
@@ -531,7 +534,6 @@ public static SegmentTree build(@NotNull BasedSegmentBuilder builder) {
      * @param segments       segments of the tree
      * @param allText        all out of base text
      * @param buildIndexData true to build index search data, false to build base offset tree data
-     *
      * @return segment tree instance with the data
      */
     @NotNull
@@ -619,7 +621,6 @@ public static SegmentTreeData buildTreeData(@NotNull Iterable<Seg> segments, @No
      * Efficiently reuses segmentBytes and only computes offset treeData for BASE and ANCHOR segments
      *
      * @param baseSeq base sequence for the sequence for this segment tree
-     *
      * @return SegmentOffsetTree for this segment tree
      */
     @NotNull

Diff for: flexmark-util-sequence/src/main/java/com/vladsch/flexmark/util/sequence/builder/tree/SegmentTree.java

@@ -41,7 +41,7 @@ public boolean collectText(ISequenceBuilder<? extends ISequenceBuilder<?, BasedS
         } else {
             ReplacedTextMapper textMapper = new ReplacedTextMapper(getChars());
             BasedSequence unescaped = Escaping.unescape(getChars(), textMapper);
-            out.append(unescaped);
+            if (!unescaped.isEmpty()) out.append(unescaped);
         }
         return false;
     }