AVRO-1329: Extended enums

doc/src/content/xdocs/idl.xml

@@ -355,7 +355,7 @@ void fireAndForget(string message) oneway;
         <code>//</code> on a line is ignored, as is any text between <code>/*</code> and
         <code>*/</code>, possibly spanning multiple lines.</p>
         <p>Comments that begin with <code>/**</code> are used as the
-        documentation string for the type or field definition that
+        documentation string for the type, field definition or enum symbol that
         follows the comment.</p>
       </section>
       <section id="minutiae_escaping">

doc/src/content/xdocs/spec.xml

@@ -183,13 +183,27 @@
 	      the <code>symbols</code> array.
 	      See documentation on schema resolution for how this gets
 	      used.</li>
+      <li><code>symbol-props</code>: a JSON array of objects
+        that provide additional information for the individual symbols.
+        (optional)
+        If provided, there must be a props object for each defined
+        symbol, each specifying a the symbol <code>name</code>, in the same order as
+        provided by the <code>symbols</code> array. Notably, the props
+        field <code>doc</code> may be used to provide per-symbol documentation.
+      </li>
 	  </ul>
 	  <p>For example, playing card suits might be defined with:</p>
 	  <source>
 {
   "type": "enum",
   "name": "Suit",
-  "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"]
+  "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"],
+  "symbol-props" : [
+      { "name": "SPADES", "doc": "Spades or Pikes" },
+      { "name": "HEARTS" },
+      { "name": "DIAMONDS", "doc": "Diamonds or Tiles" },
+      { "name": "CLUBS", "doc": "Clubs or Clovers" }
+  ]
 }
 	  </source>
 	</section>

lang/java/avro/src/main/java/org/apache/avro/Schema.java

@@ -127,7 +127,7 @@ private Type() {
     public String getName() {
       return name;
     }
-  };
+  }
 
   private final Type type;
   private LogicalType logicalType = null;
@@ -165,8 +165,10 @@ public static Schema create(Type type) {
       Arrays.asList("doc", "fields", "items", "name", "namespace", "size", "symbols", "values", "type", "aliases"));
 
   private static final Set<String> ENUM_RESERVED = new HashSet<>(SCHEMA_RESERVED);
+
   static {
     ENUM_RESERVED.add("default");
+    ENUM_RESERVED.add("symbol-props");
   }
 
   int hashCode = NO_HASHCODE;
@@ -227,6 +229,18 @@ public static Schema createEnum(String name, String doc, String namespace, List<
     return new EnumSchema(new Name(name, namespace), doc, new LockableArrayList<>(values), enumDefault);
   }
 
+  /** Create an extended enum schema with symbol-specific doc and properties. */
+  public static Schema createEnumWithDefinitions(String name, String doc, String namespace,
+      List<SymbolDefinition> symbols) {
+    return createEnumWithDefinitions(name, doc, namespace, symbols, null);
+  }
+
+  /** Create an extended enum schema with symbol-specific doc and properties. */
+  public static Schema createEnumWithDefinitions(String name, String doc, String namespace,
+      List<SymbolDefinition> symbols, String enumDefault) {
+    return new EnumSchema(new Name(name, namespace), doc, symbols, enumDefault);
+  }
+
   /** Create an array schema. */
   public static Schema createArray(Schema elementType) {
     return new ArraySchema(elementType);
@@ -287,6 +301,11 @@ public List<String> getEnumSymbols() {
     throw new AvroRuntimeException("Not an enum: " + this);
   }
 
+  /** If this is an enum, return its extended symbol definitions */
+  public List<SymbolDefinition> getEnumSymbolDefinitions() {
+    throw new AvroRuntimeException("Not an enum: " + this);
+  }
+
   /** If this is an enum, return its default value. */
   public String getEnumDefault() {
     throw new AvroRuntimeException("Not an enum: " + this);
@@ -520,7 +539,7 @@ public enum Order {
       private Order() {
         this.name = this.name().toLowerCase(Locale.ENGLISH);
       }
-    };
+    }
 
     /**
      * For Schema unions with a "null" type as the first entry, this can be used to
@@ -594,7 +613,7 @@ public Field(String name, Schema schema, String doc, Object defaultValue, Order
 
     public String name() {
       return name;
-    };
+    }
 
     /** The position of this field within the record. */
     public int pos() {
@@ -1017,6 +1036,7 @@ void fieldsToJson(Names names, JsonGenerator gen) throws IOException {
 
   private static class EnumSchema extends NamedSchema {
     private final List<String> symbols;
+    private final List<SymbolDefinition> symbolDefinitions;
     private final Map<String, Integer> ordinals;
     private final String enumDefault;
 
@@ -1026,11 +1046,35 @@ public EnumSchema(Name name, String doc, LockableArrayList<String> symbols, Stri
       this.ordinals = new HashMap<>(Math.multiplyExact(2, symbols.size()));
       this.enumDefault = enumDefault;
       int i = 0;
+      this.symbolDefinitions = new ArrayList<>(symbols.size());
       for (String symbol : symbols) {
         if (ordinals.put(validateName(symbol), i++) != null) {
           throw new SchemaParseException("Duplicate enum symbol: " + symbol);
         }
+        symbolDefinitions.add(new SymbolDefinition(symbol, null));
+      }
+      if (enumDefault != null && !symbols.contains(enumDefault)) {
+        throw new SchemaParseException(
+            "The Enum Default: " + enumDefault + " is not in the enum symbol set: " + symbols);
+      }
+    }
+
+    public EnumSchema(Name name, String doc, List<SymbolDefinition> symbolDefinitions, String enumDefault) {
+      super(Type.ENUM, name, doc);
+
+      LockableArrayList<String> symbols = new LockableArrayList<>(symbolDefinitions.size());
+      this.ordinals = new HashMap<>(Math.multiplyExact(2, symbols.size()));
+      this.enumDefault = enumDefault;
+      int i = 0;
+      this.symbolDefinitions = symbolDefinitions;
+      for (SymbolDefinition thisProperties : symbolDefinitions) {
+        String symbolName = thisProperties.getName();
+        if (ordinals.put(validateName(symbolName), i++) != null) {
+          throw new SchemaParseException("Duplicate enum symbol: " + symbolName);
+        }
+        symbols.add(symbolName);
       }
+      this.symbols = symbols.lock();
       if (enumDefault != null && !symbols.contains(enumDefault)) {
         throw new SchemaParseException(
             "The Enum Default: " + enumDefault + " is not in the enum symbol set: " + symbols);
@@ -1042,6 +1086,11 @@ public List<String> getEnumSymbols() {
       return symbols;
     }
 
+    @Override
+    public List<SymbolDefinition> getEnumSymbolDefinitions() {
+      return symbolDefinitions;
+    }
+
     @Override
     public boolean hasEnumSymbol(String symbol) {
       return ordinals.containsKey(symbol);
@@ -1059,7 +1108,8 @@ public boolean equals(Object o) {
       if (!(o instanceof EnumSchema))
         return false;
       EnumSchema that = (EnumSchema) o;
-      return equalCachedHash(that) && equalNames(that) && symbols.equals(that.symbols) && propsEqual(that);
+      return equalCachedHash(that) && equalNames(that) && propsEqual(that)
+          && symbolDefinitions.equals(that.symbolDefinitions) && Objects.equals(enumDefault, that.enumDefault);
     }
 
     @Override
@@ -1069,7 +1119,20 @@ public String getEnumDefault() {
 
     @Override
     int computeHash() {
-      return super.computeHash() + symbols.hashCode();
+      return Objects.hash(super.computeHash(), symbolDefinitions, enumDefault);
+    }
+
+    /**
+     * return true if any symbol of this enum has information stored besides the
+     * name
+     */
+    boolean hasExtendedSymbols() {
+      for (SymbolDefinition thisSymbol : symbolDefinitions) {
+        if (thisSymbol.isExtended()) {
+          return true;
+        }
+      }
+      return false;
     }
 
     @Override
@@ -1085,6 +1148,13 @@ void toJson(Names names, JsonGenerator gen) throws IOException {
       for (String symbol : symbols)
         gen.writeString(symbol);
       gen.writeEndArray();
+      if (hasExtendedSymbols()) {
+        gen.writeArrayFieldStart("symbol-props");
+        for (SymbolDefinition thisSymbol : symbolDefinitions) {
+          thisSymbol.toJson(gen);
+        }
+        gen.writeEndArray();
+      }
       if (getEnumDefault() != null)
         gen.writeStringField("default", getEnumDefault());
       writeProps(gen);
@@ -1093,6 +1163,102 @@ void toJson(Names names, JsonGenerator gen) throws IOException {
     }
   }
 
+  public static class SymbolDefinition extends JsonProperties {
+
+    private static final HashSet<String> RESERVED = new HashSet<>(Arrays.asList("name", "doc", "alias"));
+
+    private final String name;
+    private final String doc;
+
+    public SymbolDefinition(String name, String doc) {
+      super(RESERVED);
+      this.name = name;
+      this.doc = doc;
+    }
+
+    public SymbolDefinition withProp(String name, Object obj) {
+      addProp(name, obj);
+      return this;
+    }
+
+    /**
+     * return true if symbol properties contain any information besides the symbol
+     * name.
+     */
+    boolean isExtended() {
+      return hasProps() || doc != null;
+    }
+
+    public String getName() {
+      return this.name;
+    }
+
+    public String getDoc() {
+      return this.doc;
+    }
+
+    void toJson(JsonGenerator gen) throws IOException {
+      gen.writeStartObject();
+      gen.writeStringField("name", name);
+      if (doc != null) {
+        gen.writeStringField("doc", doc);
+      }
+      writeProps(gen);
+      gen.writeEndObject();
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(propsHashCode(), name, doc);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o == this)
+        return true;
+      if (!(o instanceof SymbolDefinition))
+        return false;
+      SymbolDefinition that = (SymbolDefinition) o;
+      return (hashCode() == that.hashCode()) && Objects.equals(this.name, that.name)
+          && Objects.equals(this.doc, that.doc) && propsEqual(that);
+    }
+
+    static SymbolDefinition fromJsonNode(JsonNode node) throws SchemaParseException {
+      if (!node.isObject()) {
+        throw new SchemaParseException("Symbol properties value must be an object.");
+      }
+      String name = getRequiredText(node, "name", "No field name");
+      String doc = getOptionalText(node, "doc");
+
+      SymbolDefinition properties = new SymbolDefinition(name, doc);
+      Iterator<String> i = node.fieldNames();
+      while (i.hasNext()) { // add field props
+        String prop = i.next();
+        if (!RESERVED.contains(prop))
+          properties.addProp(prop, node.get(prop));
+      }
+      return properties;
+    }
+
+    /**
+     * return true if symbols and symbol properties align, i.e. for each symbol,
+     * there exists exactly one properties entry in the same order.
+     */
+    static boolean symbolDefinitionsAreAligned(List<String> names, List<SymbolDefinition> properties) {
+      if (names.size() != properties.size()) {
+        return false;
+      }
+
+      for (int i = 0; i < names.size(); i++) {
+        if (!names.get(i).equals(properties.get(i).getName())) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+  }
+
   private static class ArraySchema extends Schema {
     private final Schema elementType;
 
@@ -1485,6 +1651,7 @@ public static Schema parse(String jsonSchema, boolean validate) {
   }
 
   static final Map<String, Type> PRIMITIVES = new HashMap<>();
+
   static {
     PRIMITIVES.put("string", Type.STRING);
     PRIMITIVES.put("bytes", Type.BYTES);
@@ -1697,9 +1864,24 @@ static Schema parse(JsonNode schema, Names names) {
         String defaultSymbol = null;
         if (enumDefault != null)
           defaultSymbol = enumDefault.textValue();
-        result = new EnumSchema(name, doc, symbols, defaultSymbol);
-        if (name != null)
-          names.add(result);
+        JsonNode symbolPropsNode = schema.get("symbol-props");
+        List<SymbolDefinition> symbolDefinitions;
+        if (symbolPropsNode != null) {
+          if (!symbolPropsNode.isArray()) {
+            throw new SchemaParseException("Symbol props of enum are no array: " + schema);
+          }
+          symbolDefinitions = new ArrayList<>(symbolPropsNode.size());
+          for (JsonNode thisPropsNode : symbolPropsNode) {
+            symbolDefinitions.add(SymbolDefinition.fromJsonNode(thisPropsNode));
+          }
+          if (!SymbolDefinition.symbolDefinitionsAreAligned(symbols, symbolDefinitions)) {
+            throw new SchemaParseException("Symbol props are mismatched to symbols: " + schema);
+          }
+          result = new EnumSchema(name, doc, symbolDefinitions, defaultSymbol);
+        } else {
+          result = new EnumSchema(name, doc, symbols, defaultSymbol);
+        }
+        names.add(result);
       } else if (type.equals("array")) { // array
         JsonNode itemsNode = schema.get("items");
         if (itemsNode == null)

lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java

@@ -789,6 +789,13 @@ public R symbols(String... symbols) {
       return context().complete(schema);
     }
 
+    public R symbols(Schema.SymbolDefinition... symbolProperties) {
+      Schema schema = Schema.createEnumWithDefinitions(name(), doc(), space(), Arrays.asList(symbolProperties),
+          this.enumDefault);
+      completeSchema(schema);
+      return context().complete(schema);
+    }
+
     /** Set the default value of the enum. */
     public EnumBuilder<R> defaultSymbol(String enumDefault) {
       this.enumDefault = enumDefault;

lang/java/avro/src/test/java/org/apache/avro/TestSchema.java

@@ -26,6 +26,7 @@
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
@@ -355,4 +356,24 @@ public void testDoubleAsFloatDefaultValue() {
     assertEquals(1.0f, field.defaultVal());
     assertEquals(1.0f, GenericData.get().getDefaultValue(field));
   }
+
+  @Test
+  /** test that extended enum schemas serialize and deserialize well */
+  public void testExtendedEnumSchemaSerializeAndDeserialize() {
+    Schema schema = Schema.createEnumWithDefinitions("e", null, "ns", Arrays.asList(
+        new Schema.SymbolDefinition("s1", "doc1").withProp("prop", 1), new Schema.SymbolDefinition("s2", "d2")));
+
+    String schemaString = schema.toString(true);
+    Schema readSchema = new Schema.Parser().parse(schemaString);
+    assertEquals(schema, readSchema);
+  }
+
+  @Test
+  /** test that trivial enum schemas don't end up in the JSON schema */
+  public void testSimpleEnumSchemaSerialize() {
+    Schema schema = Schema.createEnumWithDefinitions("e", null, "ns",
+        Arrays.asList(new Schema.SymbolDefinition("s1", null), new Schema.SymbolDefinition("s2", null)));
+    String schemaString = schema.toString();
+    assertFalse(schemaString.contains("schema-properties"));
+  }
 }