Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ISPN-346 Fix JSON for maps #293

Merged
merged 1 commit into from
Jul 3, 2024
Merged

ISPN-346 Fix JSON for maps #293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion core/src/main/java/org/infinispan/protostream/ProtobufParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import org.infinispan.protostream.descriptors.Descriptor;
import org.infinispan.protostream.descriptors.FieldDescriptor;
import org.infinispan.protostream.descriptors.MapDescriptor;
import org.infinispan.protostream.descriptors.Type;
import org.infinispan.protostream.descriptors.WireType;
import org.infinispan.protostream.impl.TagReaderImpl;
Expand Down Expand Up @@ -64,12 +65,19 @@ private void parseMessage(TagHandler tagHandler, Descriptor messageDescriptor, T
final int fieldNumber = WireType.getTagFieldNumber(tag);
final WireType wireType = WireType.fromTag(tag);
final FieldDescriptor fd = messageDescriptor != null ? messageDescriptor.findFieldByNumber(fieldNumber) : null;

switch (wireType) {
case LENGTH_DELIMITED: {
if (fd == null) {
byte[] value = in.readByteArray();
tagHandler.onTag(fieldNumber, null, value);
} else if (fd instanceof MapDescriptor md) {
int length = in.readUInt32();
int oldLimit = in.pushLimit(length);
tagHandler.onStartNested(fieldNumber, fd);
parseMessage(tagHandler, md.asDescriptor(), in);
tagHandler.onEndNested(fieldNumber, fd);
in.checkLastTagWas(0);
in.popLimit(oldLimit);
} else if (fd.getType() == Type.STRING) {
String value = in.readString();
tagHandler.onTag(fieldNumber, fd, value);
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/infinispan/protostream/descriptors/FieldDescriptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ public boolean isRepeated() {
return label == Label.REPEATED;
}

public boolean isMap() {
return false;
}

public JavaType getJavaType() {
return getType().getJavaType();
}
Expand Down
22 changes: 22 additions & 0 deletions core/src/main/java/org/infinispan/protostream/descriptors/MapDescriptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,34 @@
public class MapDescriptor extends FieldDescriptor {
private final String keyTypeName;
private final Type keyType;
private final Descriptor descriptor;

private MapDescriptor(Builder builder) {
super(builder);
keyTypeName = builder.keyTypeName;
keyType = Type.primitiveFromString(keyTypeName);
Descriptor.Builder b = new Descriptor.Builder().withName(name).withFullName(fullName);
FieldDescriptor.Builder kb = new FieldDescriptor.Builder().withNumber(1).withName("key").withTypeName(keyTypeName);
b.addField(kb);
FieldDescriptor.Builder vb = new FieldDescriptor.Builder().withNumber(2).withName("value").withTypeName(typeName);
b.addField(vb);
descriptor = b.build();
}

public Descriptor asDescriptor() {
return descriptor;
}

@Override
public boolean isRepeated() {
return true;
}

@Override
public boolean isMap() {
return true;
}

public Type getKeyType() {
return keyType;
}
Expand Down Expand Up @@ -44,6 +60,12 @@ public Label getLabel() {
return Label.OPTIONAL;
}

@Override
void setMessageType(Descriptor descriptor) {
super.setMessageType(descriptor);
this.descriptor.getFields().get(1).setMessageType(descriptor);
}

@Override
public String toString() {
return "MapDescriptor{" +
Expand Down
126 changes: 103 additions & 23 deletions core/src/main/java/org/infinispan/protostream/impl/JsonUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.infinispan.protostream.descriptors.FieldDescriptor;
import org.infinispan.protostream.descriptors.GenericDescriptor;
import org.infinispan.protostream.descriptors.Label;
import org.infinispan.protostream.descriptors.MapDescriptor;
import org.infinispan.protostream.descriptors.Type;

import com.fasterxml.jackson.core.JsonFactory;
Expand Down Expand Up @@ -230,11 +231,15 @@ private static void processObject(ImmutableSerializationContext ctx, JsonParser
break;
case START_OBJECT: {
FieldDescriptor fd = messageDescriptor.findFieldByName(currentField);
Descriptor messageType = fd.getMessageType();
if (messageType == null) {
throw new IllegalStateException("Field '" + currentField + "' is not an object");
if (fd.isMap()) {
processMap(ctx, (MapDescriptor) fd, parser, nestedWriter);
} else {
Descriptor messageType = fd.getMessageType();
if (messageType == null) {
throw new IllegalStateException("Field '" + currentField + "' is not an object");
}
processObject(ctx, parser, nestedWriter, messageType, fd.getNumber(), false);
}
processObject(ctx, parser, nestedWriter, messageType, fd.getNumber(), false);
break;
}
case FIELD_NAME:
Expand Down Expand Up @@ -280,6 +285,76 @@ private static void processObject(ImmutableSerializationContext ctx, JsonParser
writer.flush();
}

private static void processMap(ImmutableSerializationContext ctx, MapDescriptor md, JsonParser parser, TagWriter writer) throws IOException {
while (true) {
JsonToken token = parser.nextToken();
if (token == JsonToken.END_OBJECT) {
break;
}
if (token != JsonToken.FIELD_NAME) {
throw new IllegalStateException("Unexpected token");
}
ByteArrayOutputStream baos = new ByteArrayOutputStream(ProtobufUtil.DEFAULT_ARRAY_BUFFER_SIZE);
TagWriter nestedWriter = TagWriterImpl.newInstanceNoBuffer(ctx, baos);
String key = parser.getCurrentName();
switch (md.getKeyType()) {
case STRING -> nestedWriter.writeString(1, key);
case INT32 -> nestedWriter.writeInt32(1, Integer.parseInt(key));
case INT64 -> nestedWriter.writeInt64(1, Long.parseLong(key));
case FIXED32 -> nestedWriter.writeFixed32(1, Integer.parseInt(key));
case FIXED64 -> nestedWriter.writeFixed64(1, Long.parseLong(key));
case SINT32 -> nestedWriter.writeSInt32(1, Integer.parseInt(key));
case SINT64 -> nestedWriter.writeSInt64(1, Long.parseLong(key));
case SFIXED32 -> nestedWriter.writeSFixed32(1, Integer.parseInt(key));
case SFIXED64 -> nestedWriter.writeSFixed64(1, Long.parseLong(key));
case UINT32 -> nestedWriter.writeUInt32(1, Integer.parseInt(key));
case UINT64 -> nestedWriter.writeUInt64(1, Long.parseLong(key));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

isn't double/float missing?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Those are not valid protobuf map keys. Only scalars are supported

}
processMapValue(ctx, parser, nestedWriter, md);
writer.writeBytes(md.getNumber(), baos.toByteArray());
writer.flush();
}

}

private static void processMapValue(ImmutableSerializationContext ctx, JsonParser parser, TagWriter writer, MapDescriptor md) throws IOException {
JsonToken token = parser.nextToken();
if (token == null) {
return;
}
switch (token) {
case START_OBJECT: {
processObject(ctx, parser, writer, md.getMessageType(),2, false);
break;
}
case VALUE_STRING:
writer.writeString(2, parser.getValueAsString());
break;
case VALUE_NUMBER_INT:
switch (md.getType()) {
case INT32 -> writer.writeInt32(2, parser.getIntValue());
case INT64 -> writer.writeInt64(2, parser.getLongValue());
case FIXED32 -> writer.writeFixed32(2, parser.getIntValue());
case FIXED64 -> writer.writeFixed64(2, parser.getLongValue());
case SINT32 -> writer.writeSInt32(2, parser.getIntValue());
case SINT64 -> writer.writeSInt64(2, parser.getLongValue());
case SFIXED32 -> writer.writeSFixed32(2, parser.getIntValue());
case SFIXED64 -> writer.writeSFixed64(2, parser.getLongValue());
case UINT32 -> writer.writeUInt32(2, parser.getIntValue());
case UINT64 -> writer.writeUInt64(2, parser.getLongValue());
}
break;
case VALUE_NUMBER_FLOAT:
switch (md.getType()) {
case FLOAT -> writer.writeFloat(2, parser.getFloatValue());
case DOUBLE -> writer.writeDouble(2, parser.getDoubleValue());
}
break;
}
writer.flush();
}


private static void processPrimitive(JsonParser parser, TagWriter writer, Type fieldType) throws IOException {
while (true) {
JsonToken token = parser.nextToken();
Expand Down Expand Up @@ -546,15 +621,14 @@ public void onStartNested(int fieldNumber, FieldDescriptor fieldDescriptor) {
return;
}
startSlot(fieldDescriptor);

nestingLevel = new JsonNestingLevel(nestingLevel);

if (prettyPrint) {
indent();
nestingLevel.indent++;
}

jsonOut.append('{');
if (!fieldDescriptor.isMap()) {
jsonOut.append('{');
}
}

@Override
Expand All @@ -566,7 +640,9 @@ public void onEndNested(int fieldNumber, FieldDescriptor fieldDescriptor) {
nestingLevel.indent--;
indent();
}
jsonOut.append('}');
if (!fieldDescriptor.isMap()) {
jsonOut.append('}');
}
nestingLevel = nestingLevel.previous;
}

Expand All @@ -590,37 +666,40 @@ private void startSlot(FieldDescriptor fieldDescriptor) {
if (nestingLevel.repeatedFieldDescriptor != null && nestingLevel.repeatedFieldDescriptor != fieldDescriptor) {
endArraySlot();
}

boolean map = nestingLevel.previous != null && nestingLevel.previous.repeatedFieldDescriptor != null && nestingLevel.previous.repeatedFieldDescriptor.isMap();
if (nestingLevel.isFirstField) {
nestingLevel.isFirstField = false;
} else {
jsonOut.append(',');
jsonOut.append(map ? ':' : ',');
}
if (!fieldDescriptor.isRepeated() || nestingLevel.repeatedFieldDescriptor == null) {
if (prettyPrint) {
indent();
if (!map) {
if (!fieldDescriptor.isRepeated() || nestingLevel.repeatedFieldDescriptor == null) {
if (prettyPrint) {
indent();
}
if (fieldDescriptor.getLabel() == Label.ONE_OF) {
jsonOut.append('"').append(JSON_VALUE_FIELD).append("\":");
} else {
jsonOut.append('"').append(fieldDescriptor.getName()).append("\":");
}
}
if (fieldDescriptor.getLabel() == Label.ONE_OF) {
jsonOut.append('"').append(JSON_VALUE_FIELD).append("\":");
} else {
jsonOut.append('"').append(fieldDescriptor.getName()).append("\":");
if (prettyPrint) {
jsonOut.append(' ');
}
}
if (prettyPrint) {
jsonOut.append(' ');
}
if (fieldDescriptor.isRepeated() && nestingLevel.repeatedFieldDescriptor == null) {
nestingLevel.repeatedFieldDescriptor = fieldDescriptor;
jsonOut.append('[');
jsonOut.append(fieldDescriptor.isMap() ? '{' : '[');
}
}

private void endArraySlot() {
boolean map = nestingLevel.repeatedFieldDescriptor.isMap();
if (prettyPrint && nestingLevel.repeatedFieldDescriptor.getType() == Type.MESSAGE) {
indent();
}
nestingLevel.repeatedFieldDescriptor = null;
jsonOut.append(']');
jsonOut.append(map ? '}' : ']');
}
};

Expand Down Expand Up @@ -773,6 +852,7 @@ private static void expectField(String expectedFieldName, String actualFieldName
}

// todo [anistor] do we really need html escaping? so far I'm keeping it so we behave like previous implementation

/**
* Escapes a string literal in order to have a valid JSON representation. Optionally it can also escape some html chars.
*/
Expand Down
6 changes: 6 additions & 0 deletions integrationtests/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.testing.compile</groupId>
<artifactId>compile-testing</artifactId>
Expand Down
Loading
Loading