diff --git a/doc/src/content/xdocs/idl.xml b/doc/src/content/xdocs/idl.xml index 52a607503b3..eea477699be 100644 --- a/doc/src/content/xdocs/idl.xml +++ b/doc/src/content/xdocs/idl.xml @@ -355,7 +355,7 @@ void fireAndForget(string message) oneway; // on a line is ignored, as is any text between /* and */, possibly spanning multiple lines.

Comments that begin with /** are used as the - documentation string for the type or field definition that + documentation string for the type, field definition or enum symbol that follows the comment.

diff --git a/doc/src/content/xdocs/spec.xml b/doc/src/content/xdocs/spec.xml index e65cf3b89cb..e7568def554 100644 --- a/doc/src/content/xdocs/spec.xml +++ b/doc/src/content/xdocs/spec.xml @@ -183,13 +183,27 @@ the symbols array. See documentation on schema resolution for how this gets used. +
  • symbol-props: a JSON array of objects + that provide additional information for the individual symbols. + (optional) + If provided, there must be a props object for each defined + symbol, each specifying a the symbol name, in the same order as + provided by the symbols array. Notably, the props + field doc may be used to provide per-symbol documentation. +
  • For example, playing card suits might be defined with:

    { "type": "enum", "name": "Suit", - "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] + "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"], + "symbol-props" : [ + { "name": "SPADES", "doc": "Spades or Pikes" }, + { "name": "HEARTS" }, + { "name": "DIAMONDS", "doc": "Diamonds or Tiles" }, + { "name": "CLUBS", "doc": "Clubs or Clovers" } + ] }
    diff --git a/lang/java/avro/src/main/java/org/apache/avro/Schema.java b/lang/java/avro/src/main/java/org/apache/avro/Schema.java index ff603cd439e..4fda3239adf 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/Schema.java +++ b/lang/java/avro/src/main/java/org/apache/avro/Schema.java @@ -127,7 +127,7 @@ private Type() { public String getName() { return name; } - }; + } private final Type type; private LogicalType logicalType = null; @@ -165,8 +165,10 @@ public static Schema create(Type type) { Arrays.asList("doc", "fields", "items", "name", "namespace", "size", "symbols", "values", "type", "aliases")); private static final Set ENUM_RESERVED = new HashSet<>(SCHEMA_RESERVED); + static { ENUM_RESERVED.add("default"); + ENUM_RESERVED.add("symbol-props"); } int hashCode = NO_HASHCODE; @@ -227,6 +229,18 @@ public static Schema createEnum(String name, String doc, String namespace, List< return new EnumSchema(new Name(name, namespace), doc, new LockableArrayList<>(values), enumDefault); } + /** Create an extended enum schema with symbol-specific doc and properties. */ + public static Schema createEnumWithDefinitions(String name, String doc, String namespace, + List symbols) { + return createEnumWithDefinitions(name, doc, namespace, symbols, null); + } + + /** Create an extended enum schema with symbol-specific doc and properties. */ + public static Schema createEnumWithDefinitions(String name, String doc, String namespace, + List symbols, String enumDefault) { + return new EnumSchema(new Name(name, namespace), doc, symbols, enumDefault); + } + /** Create an array schema. */ public static Schema createArray(Schema elementType) { return new ArraySchema(elementType); @@ -287,6 +301,11 @@ public List getEnumSymbols() { throw new AvroRuntimeException("Not an enum: " + this); } + /** If this is an enum, return its extended symbol definitions */ + public List getEnumSymbolDefinitions() { + throw new AvroRuntimeException("Not an enum: " + this); + } + /** If this is an enum, return its default value. */ public String getEnumDefault() { throw new AvroRuntimeException("Not an enum: " + this); @@ -520,7 +539,7 @@ public enum Order { private Order() { this.name = this.name().toLowerCase(Locale.ENGLISH); } - }; + } /** * For Schema unions with a "null" type as the first entry, this can be used to @@ -594,7 +613,7 @@ public Field(String name, Schema schema, String doc, Object defaultValue, Order public String name() { return name; - }; + } /** The position of this field within the record. */ public int pos() { @@ -1017,6 +1036,7 @@ void fieldsToJson(Names names, JsonGenerator gen) throws IOException { private static class EnumSchema extends NamedSchema { private final List symbols; + private final List symbolDefinitions; private final Map ordinals; private final String enumDefault; @@ -1026,11 +1046,35 @@ public EnumSchema(Name name, String doc, LockableArrayList symbols, Stri this.ordinals = new HashMap<>(Math.multiplyExact(2, symbols.size())); this.enumDefault = enumDefault; int i = 0; + this.symbolDefinitions = new ArrayList<>(symbols.size()); for (String symbol : symbols) { if (ordinals.put(validateName(symbol), i++) != null) { throw new SchemaParseException("Duplicate enum symbol: " + symbol); } + symbolDefinitions.add(new SymbolDefinition(symbol, null)); + } + if (enumDefault != null && !symbols.contains(enumDefault)) { + throw new SchemaParseException( + "The Enum Default: " + enumDefault + " is not in the enum symbol set: " + symbols); + } + } + + public EnumSchema(Name name, String doc, List symbolDefinitions, String enumDefault) { + super(Type.ENUM, name, doc); + + LockableArrayList symbols = new LockableArrayList<>(symbolDefinitions.size()); + this.ordinals = new HashMap<>(Math.multiplyExact(2, symbols.size())); + this.enumDefault = enumDefault; + int i = 0; + this.symbolDefinitions = symbolDefinitions; + for (SymbolDefinition thisProperties : symbolDefinitions) { + String symbolName = thisProperties.getName(); + if (ordinals.put(validateName(symbolName), i++) != null) { + throw new SchemaParseException("Duplicate enum symbol: " + symbolName); + } + symbols.add(symbolName); } + this.symbols = symbols.lock(); if (enumDefault != null && !symbols.contains(enumDefault)) { throw new SchemaParseException( "The Enum Default: " + enumDefault + " is not in the enum symbol set: " + symbols); @@ -1042,6 +1086,11 @@ public List getEnumSymbols() { return symbols; } + @Override + public List getEnumSymbolDefinitions() { + return symbolDefinitions; + } + @Override public boolean hasEnumSymbol(String symbol) { return ordinals.containsKey(symbol); @@ -1059,7 +1108,8 @@ public boolean equals(Object o) { if (!(o instanceof EnumSchema)) return false; EnumSchema that = (EnumSchema) o; - return equalCachedHash(that) && equalNames(that) && symbols.equals(that.symbols) && propsEqual(that); + return equalCachedHash(that) && equalNames(that) && propsEqual(that) + && symbolDefinitions.equals(that.symbolDefinitions) && Objects.equals(enumDefault, that.enumDefault); } @Override @@ -1069,7 +1119,20 @@ public String getEnumDefault() { @Override int computeHash() { - return super.computeHash() + symbols.hashCode(); + return Objects.hash(super.computeHash(), symbolDefinitions, enumDefault); + } + + /** + * return true if any symbol of this enum has information stored besides the + * name + */ + boolean hasExtendedSymbols() { + for (SymbolDefinition thisSymbol : symbolDefinitions) { + if (thisSymbol.isExtended()) { + return true; + } + } + return false; } @Override @@ -1085,6 +1148,13 @@ void toJson(Names names, JsonGenerator gen) throws IOException { for (String symbol : symbols) gen.writeString(symbol); gen.writeEndArray(); + if (hasExtendedSymbols()) { + gen.writeArrayFieldStart("symbol-props"); + for (SymbolDefinition thisSymbol : symbolDefinitions) { + thisSymbol.toJson(gen); + } + gen.writeEndArray(); + } if (getEnumDefault() != null) gen.writeStringField("default", getEnumDefault()); writeProps(gen); @@ -1093,6 +1163,102 @@ void toJson(Names names, JsonGenerator gen) throws IOException { } } + public static class SymbolDefinition extends JsonProperties { + + private static final HashSet RESERVED = new HashSet<>(Arrays.asList("name", "doc", "alias")); + + private final String name; + private final String doc; + + public SymbolDefinition(String name, String doc) { + super(RESERVED); + this.name = name; + this.doc = doc; + } + + public SymbolDefinition withProp(String name, Object obj) { + addProp(name, obj); + return this; + } + + /** + * return true if symbol properties contain any information besides the symbol + * name. + */ + boolean isExtended() { + return hasProps() || doc != null; + } + + public String getName() { + return this.name; + } + + public String getDoc() { + return this.doc; + } + + void toJson(JsonGenerator gen) throws IOException { + gen.writeStartObject(); + gen.writeStringField("name", name); + if (doc != null) { + gen.writeStringField("doc", doc); + } + writeProps(gen); + gen.writeEndObject(); + } + + @Override + public int hashCode() { + return Objects.hash(propsHashCode(), name, doc); + } + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + if (!(o instanceof SymbolDefinition)) + return false; + SymbolDefinition that = (SymbolDefinition) o; + return (hashCode() == that.hashCode()) && Objects.equals(this.name, that.name) + && Objects.equals(this.doc, that.doc) && propsEqual(that); + } + + static SymbolDefinition fromJsonNode(JsonNode node) throws SchemaParseException { + if (!node.isObject()) { + throw new SchemaParseException("Symbol properties value must be an object."); + } + String name = getRequiredText(node, "name", "No field name"); + String doc = getOptionalText(node, "doc"); + + SymbolDefinition properties = new SymbolDefinition(name, doc); + Iterator i = node.fieldNames(); + while (i.hasNext()) { // add field props + String prop = i.next(); + if (!RESERVED.contains(prop)) + properties.addProp(prop, node.get(prop)); + } + return properties; + } + + /** + * return true if symbols and symbol properties align, i.e. for each symbol, + * there exists exactly one properties entry in the same order. + */ + static boolean symbolDefinitionsAreAligned(List names, List properties) { + if (names.size() != properties.size()) { + return false; + } + + for (int i = 0; i < names.size(); i++) { + if (!names.get(i).equals(properties.get(i).getName())) { + return false; + } + } + + return true; + } + } + private static class ArraySchema extends Schema { private final Schema elementType; @@ -1485,6 +1651,7 @@ public static Schema parse(String jsonSchema, boolean validate) { } static final Map PRIMITIVES = new HashMap<>(); + static { PRIMITIVES.put("string", Type.STRING); PRIMITIVES.put("bytes", Type.BYTES); @@ -1697,9 +1864,24 @@ static Schema parse(JsonNode schema, Names names) { String defaultSymbol = null; if (enumDefault != null) defaultSymbol = enumDefault.textValue(); - result = new EnumSchema(name, doc, symbols, defaultSymbol); - if (name != null) - names.add(result); + JsonNode symbolPropsNode = schema.get("symbol-props"); + List symbolDefinitions; + if (symbolPropsNode != null) { + if (!symbolPropsNode.isArray()) { + throw new SchemaParseException("Symbol props of enum are no array: " + schema); + } + symbolDefinitions = new ArrayList<>(symbolPropsNode.size()); + for (JsonNode thisPropsNode : symbolPropsNode) { + symbolDefinitions.add(SymbolDefinition.fromJsonNode(thisPropsNode)); + } + if (!SymbolDefinition.symbolDefinitionsAreAligned(symbols, symbolDefinitions)) { + throw new SchemaParseException("Symbol props are mismatched to symbols: " + schema); + } + result = new EnumSchema(name, doc, symbolDefinitions, defaultSymbol); + } else { + result = new EnumSchema(name, doc, symbols, defaultSymbol); + } + names.add(result); } else if (type.equals("array")) { // array JsonNode itemsNode = schema.get("items"); if (itemsNode == null) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java index ffb198f6a0a..88793948157 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java @@ -789,6 +789,13 @@ public R symbols(String... symbols) { return context().complete(schema); } + public R symbols(Schema.SymbolDefinition... symbolProperties) { + Schema schema = Schema.createEnumWithDefinitions(name(), doc(), space(), Arrays.asList(symbolProperties), + this.enumDefault); + completeSchema(schema); + return context().complete(schema); + } + /** Set the default value of the enum. */ public EnumBuilder defaultSymbol(String enumDefault) { this.enumDefault = enumDefault; diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java index b7e0dbea092..6e165f40b57 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java @@ -26,6 +26,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -355,4 +356,24 @@ public void testDoubleAsFloatDefaultValue() { assertEquals(1.0f, field.defaultVal()); assertEquals(1.0f, GenericData.get().getDefaultValue(field)); } + + @Test + /** test that extended enum schemas serialize and deserialize well */ + public void testExtendedEnumSchemaSerializeAndDeserialize() { + Schema schema = Schema.createEnumWithDefinitions("e", null, "ns", Arrays.asList( + new Schema.SymbolDefinition("s1", "doc1").withProp("prop", 1), new Schema.SymbolDefinition("s2", "d2"))); + + String schemaString = schema.toString(true); + Schema readSchema = new Schema.Parser().parse(schemaString); + assertEquals(schema, readSchema); + } + + @Test + /** test that trivial enum schemas don't end up in the JSON schema */ + public void testSimpleEnumSchemaSerialize() { + Schema schema = Schema.createEnumWithDefinitions("e", null, "ns", + Arrays.asList(new Schema.SymbolDefinition("s1", null), new Schema.SymbolDefinition("s2", null))); + String schemaString = schema.toString(); + assertFalse(schemaString.contains("schema-properties")); + } } diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java index c00252ea7ca..bda54548314 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java @@ -73,8 +73,8 @@ public SchemaVisitorAction visitTerminal(final Schema terminal) { newSchema = Schema.create(type); break; case ENUM: - newSchema = Schema.createEnum(terminal.getName(), terminal.getDoc(), terminal.getNamespace(), - terminal.getEnumSymbols(), terminal.getEnumDefault()); + newSchema = Schema.createEnumWithDefinitions(terminal.getName(), terminal.getDoc(), terminal.getNamespace(), + terminal.getEnumSymbolDefinitions(), terminal.getEnumDefault()); break; case FIXED: newSchema = Schema.createFixed(terminal.getName(), terminal.getDoc(), terminal.getNamespace(), diff --git a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj index 7787b458df7..f1b71ebd06e 100644 --- a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj +++ b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj @@ -1101,7 +1101,7 @@ Protocol ProtocolDeclaration(): Schema EnumDeclaration(): { String name; - List symbols; + List symbols; String defaultSymbol = null; } { @@ -1110,16 +1110,16 @@ Schema EnumDeclaration(): symbols = EnumBody() [ defaultSymbol=Identifier() ] { - Schema s = Schema.createEnum(name, doc, this.namespace, symbols, + Schema s = Schema.createEnumWithDefinitions(name, doc, this.namespace, symbols, defaultSymbol); names.put(s.getFullName(), s); return s; } } -List EnumBody(): +List EnumBody(): { - List symbols = new ArrayList(); + List symbols = new ArrayList(); } { "{" @@ -1130,12 +1130,12 @@ List EnumBody(): } } -void EnumConstant(List symbols): +void EnumConstant(List symbols): { String sym; } { - sym = Identifier() { symbols.add(sym); } + sym = Identifier() { symbols.add(new Schema.SymbolDefinition(sym,getDoc())); } } void ProtocolBody(Protocol p): diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm index c3feab9436e..6de0275d5ce 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm @@ -26,8 +26,15 @@ package $this.mangle($schema.getNamespace()); #end @org.apache.avro.specific.AvroGenerated public enum ${this.mangle($schema.getName())} implements org.apache.avro.generic.GenericEnumSymbol<${this.mangle($schema.getName())}> { - #foreach ($symbol in ${schema.getEnumSymbols()})${this.mangle($symbol)}#if ($foreach.hasNext), #end#end - ; + +#foreach ($symbol in ${schema.getEnumSymbolDefinitions()}) +#if ($symbol.getDoc()) + /** ${symbol.getDoc()} */ +#end + ${this.mangle($symbol.getName())}#if ($foreach.hasNext),#else;#end + +#end + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } diff --git a/lang/java/compiler/src/test/idl/input/enum_with_symbol_doc.avdl b/lang/java/compiler/src/test/idl/input/enum_with_symbol_doc.avdl new file mode 100644 index 00000000000..13787ec1cc7 --- /dev/null +++ b/lang/java/compiler/src/test/idl/input/enum_with_symbol_doc.avdl @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.apache.avro.test") +protocol EnumWithSymbolDocs { + enum ExampleEnum { + /** one */ + ONE, + + /** two */ + TWO + } +} diff --git a/lang/java/compiler/src/test/idl/output/enum_with_symbol_doc.avpr b/lang/java/compiler/src/test/idl/output/enum_with_symbol_doc.avpr new file mode 100644 index 00000000000..a3bc41faa28 --- /dev/null +++ b/lang/java/compiler/src/test/idl/output/enum_with_symbol_doc.avpr @@ -0,0 +1 @@ +{"protocol":"EnumWithSymbolDocs","namespace":"org.apache.avro.test","types":[{"type":"enum","name":"ExampleEnum","symbols":["ONE","TWO"],"symbol-props":[{"name":"ONE","doc":"one"},{"name":"TWO","doc":"two"}]}],"messages":{}} \ No newline at end of file diff --git a/lang/java/tools/src/test/compiler/output-string/Position.java b/lang/java/tools/src/test/compiler/output-string/Position.java deleted file mode 100644 index a4504bbc4c9..00000000000 --- a/lang/java/tools/src/test/compiler/output-string/Position.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Autogenerated by Avro - * - * DO NOT EDIT DIRECTLY - */ -package avro.examples.baseball; -@org.apache.avro.specific.AvroGenerated -public enum Position implements org.apache.avro.generic.GenericEnumSymbol { - P, C, B1, B2, B3, SS, LF, CF, RF, DH ; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"Position\",\"namespace\":\"avro.examples.baseball\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}"); - public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } - public org.apache.avro.Schema getSchema() { return SCHEMA$; } -} diff --git a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Position.java b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Position.java index a4504bbc4c9..a1bfcfb9923 100644 --- a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Position.java +++ b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Position.java @@ -6,7 +6,18 @@ package avro.examples.baseball; @org.apache.avro.specific.AvroGenerated public enum Position implements org.apache.avro.generic.GenericEnumSymbol { - P, C, B1, B2, B3, SS, LF, CF, RF, DH ; + + P, + C, + B1, + B2, + B3, + SS, + LF, + CF, + RF, + DH; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"Position\",\"namespace\":\"avro.examples.baseball\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } diff --git a/lang/java/tools/src/test/compiler/output/Position.java b/lang/java/tools/src/test/compiler/output/Position.java index a4504bbc4c9..a1bfcfb9923 100644 --- a/lang/java/tools/src/test/compiler/output/Position.java +++ b/lang/java/tools/src/test/compiler/output/Position.java @@ -6,7 +6,18 @@ package avro.examples.baseball; @org.apache.avro.specific.AvroGenerated public enum Position implements org.apache.avro.generic.GenericEnumSymbol { - P, C, B1, B2, B3, SS, LF, CF, RF, DH ; + + P, + C, + B1, + B2, + B3, + SS, + LF, + CF, + RF, + DH; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"Position\",\"namespace\":\"avro.examples.baseball\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; }