LUCENE-9450: Use BinaryDocValue fields in the taxonomy index based on the existing index version

lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java

@@ -62,6 +62,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
 
 /**
  * {@link TaxonomyWriter} which uses a {@link Directory} to store the taxonomy information on disk,
@@ -475,8 +476,20 @@ private int addCategoryDocument(FacetLabel categoryPath, int parent) throws IOEx
 
     String fieldPath = FacetsConfig.pathToString(categoryPath.components, categoryPath.length);
     fullPathField.setStringValue(fieldPath);
+
+    boolean commitExists = indexWriter.getLiveCommitData().iterator().hasNext();
+    /* no commits so this is a fresh index, or the old index was built using a Lucene 9 or greater version */
+    if ((commitExists == false)
+        || (SegmentInfos.readLatestCommit(dir)
+            .getMinSegmentLuceneVersion()
+            .onOrAfter(Version.LUCENE_9_0_0))) {
+      /* Lucene 9 introduces BinaryDocValuesField for storing taxonomy categories */
+      d.add(new BinaryDocValuesField(Consts.FULL, new BytesRef(fieldPath)));
+    } else {
+      fullPathField = new StringField(Consts.FULL, fieldPath, Field.Store.YES);
+    }
+
     d.add(fullPathField);
-    d.add(new BinaryDocValuesField(Consts.FULL, new BytesRef(fieldPath)));
 
     // Note that we do no pass an Analyzer here because the fields that are
     // added to the Document are untokenized or contains their own TokenStream.

lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestBackwardsCompatibility.java

@@ -49,13 +49,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
   //
   // Then move the zip file to your trunk checkout and use it in your test cases
 
-  public static final String oldTaxonomyIndexName = "taxonomy.8.6.3-cfs";
+  public static final String oldTaxonomyIndexName = "taxonomy.8.10.0-cfs";
 
-  // LUCENE-9334 requires consistency of field data structures between documents.
-  // Old taxonomy index had $full_path$ field indexed only with postings,
-  // It is not allowed to add the same field $full_path$ indexed with BinaryDocValues
-  // for a new segment, that this test is trying to do.
-  @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-9334")
   public void testCreateNewTaxonomy() throws IOException {
     createNewTaxonomyIndex(oldTaxonomyIndexName);
   }
@@ -68,8 +63,8 @@ private void createNewTaxonomyIndex(String dirName) throws IOException {
 
     DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
 
-    FacetLabel cp_b = new FacetLabel("b");
-    writer.addCategory(cp_b);
+    FacetLabel cp_c = new FacetLabel("c");
+    writer.addCategory(cp_c);
     writer.getInternalIndexWriter().forceMerge(1);
     writer.commit();
 
@@ -80,10 +75,15 @@ private void createNewTaxonomyIndex(String dirName) throws IOException {
     // Just asserting ord1 != TaxonomyReader.INVALID_ORDINAL is not enough to check compatibility
     assertNotNull(reader.getPath(ord1));
 
-    int ord2 = reader.getOrdinal(cp_b);
+    int ord2 = reader.getOrdinal(new FacetLabel("b"));
     assert ord2 != TaxonomyReader.INVALID_ORDINAL;
+    // Just asserting ord2 != TaxonomyReader.INVALID_ORDINAL is not enough to check compatibility
     assertNotNull(reader.getPath(ord2));
 
+    int ord3 = reader.getOrdinal(cp_c);
+    assert ord3 != TaxonomyReader.INVALID_ORDINAL;
+    assertNotNull(reader.getPath(ord3));
+
     reader.close();
     writer.close();
     dir.close();
@@ -103,6 +103,7 @@ private void createOldTaxonomyIndex(String dirName) throws IOException {
     TaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
 
     writer.addCategory(new FacetLabel("a"));
+    writer.addCategory(new FacetLabel("b"));
     writer.commit();
     writer.close();
     dir.close();