Asciidoctor: Work around "cramped" include statements (#529)

We do a fair bit of:

```
include::resources/1.adoc[]
include::resources/2.adoc[]
```

In our asciidoc files. Asciidoctor would prefer that we do:

```
include::resources/1.adoc[]

include::resources/2.adoc[]
```

but asciidoc doesn't care. To be compatible with asciidoc this works
around asciidoctor's preference, preventing the parsing error that come
about because of using these "cramped" include statements by adding a
trailing new line every time we include an asciidoc file. This is a
little crude but it works like a charm.

With this, I can now build the Elasticsearch Java REST Client reference
without `--lenient` and the document looks great! This is how long the
build takes with asciidoctor:

```
real  0m7.652s
user  0m7.080s
sys   0m0.538s
```

And this is how long it takes with asciidoc:

```
real  0m44.987s
user  0m38.715s
sys   0m6.256s
```

Author: nik9000

resources/asciidoctor/lib/cramped_include/extension.rb

@@ -0,0 +1,23 @@
+require 'asciidoctor/extensions'
+
+include Asciidoctor
+
+# Preprocessor to support more "cramped" include statements. Usually something
+# like
+#   include::resources/1.adoc[]
+#   include::resources/2.adoc[]
+# will result in syntax errors if 1.adoc ends in only a single new line. Things
+# like callout lists require that they be followed by an empty line or else
+# the thing below them will get sucked into the callout list. This isn't a
+# problem with asciidoc, and to be better compatible with it try and work around
+# this problem by adding an extra new line after every sequence of lines we
+# include. In theory this *shouldn't* bother us because we don't include things
+# that are sensitive to the extra line.
+class CrampedInclude < Extensions::Preprocessor
+  def process document, reader
+    def reader.prepare_lines data, opts = {}
+      super << ''
+    end
+    reader
+  end
+end

resources/asciidoctor/lib/extensions.rb

@@ -1,4 +1,5 @@
 require_relative 'added/extension'
+require_relative 'cramped_include/extension'
 require_relative 'edit_me/extension'
 require_relative 'elastic_compat_tree_processor/extension'
 require_relative 'elastic_compat_preprocessor/extension'
@@ -8,6 +9,7 @@
   # Enable storing the source locations so we can look at them. This is required
   # for EditMe to get a nice location.
   document.sourcemap = true
+  preprocessor CrampedInclude
   preprocessor ElasticCompatPreprocessor
   treeprocessor EditMe
   treeprocessor ElasticCompatTreeProcessor

resources/asciidoctor/spec/cramped_include_spec.rb

@@ -0,0 +1,72 @@
+require 'cramped_include/extension'
+require 'shared_examples/does_not_break_line_numbers'
+
+RSpec.describe CrampedInclude do
+  before(:each) do
+    Extensions.register do
+      preprocessor CrampedInclude
+    end
+  end
+
+  after(:each) do
+    Extensions.unregister_all
+  end
+
+  include_examples "doesn't break line numbers"
+
+  it "allows cramped includes of callout lists" do
+    actual = convert <<~ASCIIDOC
+      = Test
+
+      == Test
+
+      include::resources/cramped_include/colist1.adoc[]
+      include::resources/cramped_include/colist2.adoc[]
+    ASCIIDOC
+    expected = <<~DOCBOOK
+      <chapter id="_test">
+      <title>Test</title>
+      <section id="P1">
+      <title>P1</title>
+      <section id="P1_1">
+      <title>P1.1</title>
+      <programlisting language="java" linenumbering="unnumbered">words <1> <2></programlisting>
+      <calloutlist>
+      <callout arearefs="CO1-1">
+      <para>foo</para>
+      </callout>
+      </calloutlist>
+      </section>
+      </section>
+      <section id="P2">
+      <title>P2</title>
+      <section id="P2_1">
+      <title>P2.1</title>
+      <programlisting language="java" linenumbering="unnumbered">words <1> <2></programlisting>
+      <calloutlist>
+      <callout arearefs="CO2-1">
+      <para>foo</para>
+      </callout>
+      </calloutlist>
+      </section>
+      </section>
+      </chapter>
+    DOCBOOK
+    expect(actual).to eq(expected.strip)
+  end
+
+  it "doesn't break includes of non-asciidoc files" do
+    actual = convert <<~ASCIIDOC
+      ----
+      include::resources/cramped_include/Example.java[]
+      ----
+    ASCIIDOC
+    expected = <<~DOCBOOK
+      <preface>
+      <title></title>
+      <screen>public class Example {}</screen>
+      </preface>
+    DOCBOOK
+    expect(actual).to eq(expected.strip)
+  end
+end

resources/asciidoctor/spec/elastic_compat_preprocessor_spec.rb

@@ -1,6 +1,7 @@
 require 'added/extension'
 require 'elastic_compat_preprocessor/extension'
 require 'elastic_include_tagged/extension'
+require 'shared_examples/does_not_break_line_numbers'
 
 RSpec.describe ElasticCompatPreprocessor do
   before(:each) do
@@ -15,6 +16,8 @@
     Extensions.unregister_all
   end
 
+  include_examples "doesn't break line numbers"
+
   it "invokes added[version]" do
     actual = convert <<~ASCIIDOC
       == Example
@@ -51,24 +54,6 @@
     expect(actual).to eq(expected.strip)
   end
 
-  it "doesn't break line numbers" do
-    input = <<~ASCIIDOC
-      ---
-      ---
-      <1> callout
-    ASCIIDOC
-    expect { convert(input) }.to raise_error(
-        ConvertError, /<stdin>: line 3: no callout found for <1>/)
-  end
-
-  it "doesn't break line numbers in included files" do
-    input = <<~ASCIIDOC
-      include::resources/elastic_compat_preprocessor/missing_callout.adoc[]
-    ASCIIDOC
-    expect { convert(input) }.to raise_error(
-        ConvertError, /missing_callout.adoc: line 3: no callout found for <1>/)
-  end
-
   it "un-blocks blocks containing only attributes" do
     actual = convert <<~ASCIIDOC
       :inheader: foo

resources/asciidoctor/spec/resources/cramped_include/Example.java

@@ -0,0 +1 @@
+public class Example {}

resources/asciidoctor/spec/resources/cramped_include/colist1.adoc

@@ -0,0 +1,11 @@
+[[P1]]
+=== P1
+
+[[P1_1]]
+==== P1.1
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+words <1> <2>
+--------------------------------------------------
+<1> foo

resources/asciidoctor/spec/resources/cramped_include/colist2.adoc

@@ -0,0 +1,11 @@
+[[P2]]
+=== P2
+
+[[P2_1]]
+==== P2.1
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+words <1> <2>
+--------------------------------------------------
+<1> foo

resources/asciidoctor/spec/resources/elastic_compat_preprocessor/missing_callout.adoc renamed to resources/asciidoctor/spec/resources/does_not_break_line_numbers/missing_callout.adoc

@@ -0,0 +1,19 @@
+RSpec.shared_examples "doesn't break line numbers" do 
+  it "doesn't break line numbers" do
+    input = <<~ASCIIDOC
+      ---
+      ---
+      <1> callout
+    ASCIIDOC
+    expect { convert(input) }.to raise_error(
+        ConvertError, /<stdin>: line 3: no callout found for <1>/)
+  end
+
+  it "doesn't break line numbers in included files" do
+    input = <<~ASCIIDOC
+      include::resources/does_not_break_line_numbers/missing_callout.adoc[]
+    ASCIIDOC
+    expect { convert(input) }.to raise_error(
+        ConvertError, /missing_callout.adoc: line 3: no callout found for <1>/)
+  end
+end