Skip to content

Commit

Permalink
Add FlatDeserializationContext.
Browse files Browse the repository at this point in the history 
FlatDeserializationContext only accepts deserializeFully. It's clear that
interned types actually only work with deserializeFully because partially
deserialized values would not be correctly interned.

* Make InterningObjectCodec use FlatDeserializationContext instead of
  AsyncDeserializationContext.
* Splits out FLAT_OBJECT_ARRAY_PROCESSOR from OBJECT_ARRAY_PROCESSOR that uses
  the FlatDeserializationContext.

PiperOrigin-RevId: 596016993
Change-Id: Ic799000d8caa446e2291858737669c16ab7bafa7
  • Loading branch information
@aoeui @copybara-github
aoeui authored and copybara-github committed Jan 5, 2024
1 parent c6ba1c3 commit 23d6d70
Show file tree
Hide file tree
Showing 9 changed files with 259 additions and 135 deletions.
8 changes: 4 additions & 4 deletions src/main/java/com/google/devtools/build/lib/cmdline/LabelCodec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import static com.google.devtools.build.lib.unsafe.UnsafeProvider.getFieldOffset;
import static com.google.devtools.build.lib.unsafe.UnsafeProvider.unsafe;

import com.google.devtools.build.lib.skyframe.serialization.AsyncDeserializationContext;
import com.google.devtools.build.lib.skyframe.serialization.FlatDeserializationContext;
import com.google.devtools.build.lib.skyframe.serialization.InterningObjectCodec;
import com.google.devtools.build.lib.skyframe.serialization.SerializationContext;
import com.google.devtools.build.lib.skyframe.serialization.SerializationException;
Expand Down Expand Up @@ -53,16 +53,16 @@ public void serialize(SerializationContext context, Label obj, CodedOutputStream
}

@Override
public Label deserializeInterned(AsyncDeserializationContext context, CodedInputStream codedIn)
public Label deserializeInterned(FlatDeserializationContext context, CodedInputStream codedIn)
throws SerializationException, IOException {
Label label;
try {
label = (Label) unsafe().allocateInstance(Label.class);
} catch (ReflectiveOperationException e) {
throw new SerializationException("Could not instantiate Label", e);
}
context.deserialize(codedIn, label, PACKAGE_IDENTIFIER_OFFSET);
context.deserialize(codedIn, label, LabelCodec::setName);
context.deserializeFully(codedIn, label, PACKAGE_IDENTIFIER_OFFSET);
context.deserializeFully(codedIn, label, LabelCodec::setName);
return label;
}

Expand Down
207 changes: 132 additions & 75 deletions src/main/java/com/google/devtools/build/lib/skyframe/serialization/ArrayProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@
*
* <p>Clients should obtain instances using {@link #forType}.
*/
public abstract class ArrayProcessor {
public interface ArrayProcessor {
/**
* Serializes an array.
*
* @param type the type of the array. Can be a multidimensional array type.
* @param arr the array instance to serialize.
*/
public abstract void serialize(
void serialize(
SerializationContext context, CodedOutputStream codedOut, Class<?> type, Object arr)
throws IOException, SerializationException;

Expand All @@ -53,7 +53,7 @@ public abstract void serialize(
* @param obj the object to contain the array, that could be an array itself.
* @param offset offset within obj to write the deserialized value.
*/
public abstract void deserialize(
void deserialize(
AsyncDeserializationContext context,
CodedInputStream codedIn,
Class<?> type,
Expand Down Expand Up @@ -92,7 +92,8 @@ static ArrayProcessor forType(Class<?> type) {
return processor;
}

private static Class<?> resolveBaseArrayType(Class<?> arrayType) {
// This method should be marked private, but that's not supported in Java 8.
static Class<?> resolveBaseArrayType(Class<?> arrayType) {
Class<?> componentType = arrayType.getComponentType();
if (componentType.isArray()) {
return resolveBaseArrayType(componentType);
Expand All @@ -106,7 +107,7 @@ private static Class<?> resolveBaseArrayType(Class<?> arrayType) {
* <p>Subclasses handle the base arrays by implementing {@link #serializeArrayData} and {@link
* #deserializeArrayData}.
*/
public abstract static class PrimitiveArrayProcessor extends ArrayProcessor {
abstract static class PrimitiveArrayProcessor implements ArrayProcessor {
@Override
public final void serialize(
SerializationContext context, CodedOutputStream codedOut, Class<?> type, Object arr)
Expand Down Expand Up @@ -150,6 +151,11 @@ public final void deserialize(
Object obj,
long offset)
throws IOException {
deserialize(codedIn, type, obj, offset);
}

public final void deserialize(CodedInputStream codedIn, Class<?> type, Object obj, long offset)
throws IOException {
int length = codedIn.readInt32();
if (length == 0) {
return; // It was null.
Expand All @@ -162,7 +168,6 @@ public final void deserialize(
unsafe().putObject(obj, offset, arr);
for (int i = 0; i < length; ++i) {
deserialize(
context,
codedIn,
componentType,
arr,
Expand All @@ -178,7 +183,7 @@ public abstract Object deserializeArrayData(CodedInputStream codedIn, int length
throws IOException;
}

public static final PrimitiveArrayProcessor BOOLEAN_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor BOOLEAN_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -201,7 +206,7 @@ public boolean[] deserializeArrayData(CodedInputStream codedIn, int length)
}
};

public static final PrimitiveArrayProcessor BYTE_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor BYTE_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -221,7 +226,7 @@ public byte[] deserializeArrayData(CodedInputStream codedIn, int length)
}
};

public static final PrimitiveArrayProcessor SHORT_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor SHORT_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -244,7 +249,7 @@ public short[] deserializeArrayData(CodedInputStream codedIn, int length)
}
};

public static final PrimitiveArrayProcessor CHAR_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor CHAR_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -267,7 +272,7 @@ public char[] deserializeArrayData(CodedInputStream codedIn, int length)
}
};

public static final PrimitiveArrayProcessor INT_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor INT_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -289,7 +294,7 @@ public int[] deserializeArrayData(CodedInputStream codedIn, int length) throws I
}
};

public static final PrimitiveArrayProcessor LONG_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor LONG_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -312,7 +317,7 @@ public long[] deserializeArrayData(CodedInputStream codedIn, int length)
}
};

public static final PrimitiveArrayProcessor FLOAT_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor FLOAT_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -335,7 +340,7 @@ public float[] deserializeArrayData(CodedInputStream codedIn, int length)
}
};

public static final PrimitiveArrayProcessor DOUBLE_ARRAY_PROCESSOR =
static final PrimitiveArrayProcessor DOUBLE_ARRAY_PROCESSOR =
new PrimitiveArrayProcessor() {
@Override
public void serializeArrayData(CodedOutputStream codedOut, Object untypedArr)
Expand All @@ -358,6 +363,8 @@ public double[] deserializeArrayData(CodedInputStream codedIn, int length)
}
};

static final AsyncObjectArrayProcessor OBJECT_ARRAY_PROCESSOR = new AsyncObjectArrayProcessor();

/**
* Handles possibly nested arrays of {@code Object} or any type derived from {@code Object}.
*
Expand All @@ -366,77 +373,117 @@ public double[] deserializeArrayData(CodedInputStream codedIn, int length)
* nested arrays of appropriate type and nesting level.
*
* <p>Finally, at the leaf level, it uses the {@code (Ser|Deser)ializationContext} to apply codecs
* to the base array components.
* to the base array components. The {@link DeserializationContext} can be either a {@link
* FlatDeserializationContext} or an {@link AsyncDeserializationContext} in the different
* subclasses.
*/
public static final ArrayProcessor OBJECT_ARRAY_PROCESSOR =
new ArrayProcessor() {
// Special case uses `Array.newInstance` to also create leaf-level arrays. Additionally,
// unlike the `PrimitiveArrayProcessor`, the unnested arrays rely on serialization contexts.
abstract static class ObjectArrayProcessor<T extends FlatDeserializationContext> {
// Special case uses `Array.newInstance` to also create leaf-level arrays. Unlike the
// `PrimitiveArrayProcessor`, the unnested arrays depend on serialization contexts.

@Override
public void serialize(
SerializationContext context, CodedOutputStream codedOut, Class<?> type, Object arr)
throws IOException, SerializationException {
// Tagging works exactly the same as PrimitiveArrayProcessor.serialize: 0 for null;
// 1 + length otherwise. See comment there for more details.
if (arr == null) {
codedOut.writeInt32NoTag(0);
return;
}
/** Overrides {@link ArrayProcessor#serialize}. */
public void serialize(
SerializationContext context, CodedOutputStream codedOut, Class<?> type, Object arr)
throws IOException, SerializationException {
// Tagging works exactly the same as PrimitiveArrayProcessor.serialize: 0 for null;
// 1 + length otherwise. See comment there for more details.
if (arr == null) {
codedOut.writeInt32NoTag(0);
return;
}

Class<?> componentType = type.getComponentType();
if (componentType.isArray()) {
Object[] subarrays = (Object[]) arr;
codedOut.writeInt32NoTag(subarrays.length + 1);
for (Object subarray : subarrays) {
serialize(context, codedOut, componentType, subarray);
}
return;
}
Class<?> componentType = type.getComponentType();
if (componentType.isArray()) {
Object[] subarrays = (Object[]) arr;
codedOut.writeInt32NoTag(subarrays.length + 1);
for (Object subarray : subarrays) {
serialize(context, codedOut, componentType, subarray);
}
return;
}

serializeObjectArray(context, codedOut, arr);
}

/**
* Overrides {@link ArrayProcessor#deserialize} if {@code T} is {@link
* AsyncDeserializationContext}.
*/
public void deserialize(
T context, CodedInputStream codedIn, Class<?> type, Object obj, long offset)
throws IOException, SerializationException {
int length = codedIn.readInt32();
if (length == 0) {
return; // It was null.
}
length--; // Shifts the length back. It was shifted to allow 0 to be used for null.

Class<?> componentType = type.getComponentType();
Object arr = Array.newInstance(componentType, length);
unsafe().putObject(obj, offset, arr);

if (length == 0) {
return; // Empty array.
}

serializeObjectArray(context, codedOut, arr);
// It's a non-empty array if this is reached.
if (componentType.isArray()) {
for (int i = 0; i < length; ++i) {
deserialize(
context,
codedIn,
componentType,
arr,
ARRAY_OBJECT_BASE_OFFSET + ARRAY_OBJECT_INDEX_SCALE * i);
}
return;
}

@Override
public void deserialize(
AsyncDeserializationContext context,
CodedInputStream codedIn,
Class<?> type,
Object obj,
long offset)
throws IOException, SerializationException {
int length = codedIn.readInt32();
if (length == 0) {
return; // It was null.
}
length--; // Shifts the length back. It was shifted to allow 0 to be used for null.
deserializeObjectArray(context, codedIn, arr, length);
}

Class<?> componentType = type.getComponentType();
Object arr = Array.newInstance(componentType, length);
unsafe().putObject(obj, offset, arr);
/** Deserializes {@code length} objects from {@code codedIn} into {@code untypedArr}. */
abstract void deserializeObjectArray(
T context, CodedInputStream codedIn, Object untypedArr, int length)
throws IOException, SerializationException;
}

if (length == 0) {
return; // Empty array.
}
/** {@link ArrayProcessor} for {@code Object} arrays. */
static class AsyncObjectArrayProcessor extends ObjectArrayProcessor<AsyncDeserializationContext>
implements ArrayProcessor {
@Override
void deserializeObjectArray(
AsyncDeserializationContext context,
CodedInputStream codedIn,
Object untypedArr,
int length)
throws IOException, SerializationException {
ArrayProcessor.deserializeObjectArray(context, codedIn, untypedArr, length);
}

// It's a non-empty array if this is reached.
if (componentType.isArray()) {
for (int i = 0; i < length; ++i) {
deserialize(
context,
codedIn,
componentType,
arr,
ARRAY_OBJECT_BASE_OFFSET + ARRAY_OBJECT_INDEX_SCALE * i);
}
return;
}
private AsyncObjectArrayProcessor() {}
}

deserializeObjectArray(context, codedIn, arr, length);
/**
* Like {@link AsyncObjectArrayProcessor} but requires full deserializization of array values.
*
* <p>Used by {@code AutoCodec} implementations.
*/
static final ObjectArrayProcessor<FlatDeserializationContext> FLAT_OBJECT_ARRAY_PROCESSOR =
new ObjectArrayProcessor<FlatDeserializationContext>() {
@Override
void deserializeObjectArray(
FlatDeserializationContext context,
CodedInputStream codedIn,
Object untypedArr,
int length)
throws IOException, SerializationException {
deserializeObjectArrayFully(context, codedIn, untypedArr, length);
}
};

public static void serializeObjectArray(
/** Serializes an object array using the given {@code context} to the {@code codedOut} stream. */
static void serializeObjectArray(
SerializationContext context, CodedOutputStream codedOut, Object untypedArr)
throws IOException, SerializationException {
Object[] values = (Object[]) untypedArr;
Expand All @@ -446,16 +493,26 @@ public static void serializeObjectArray(
}
}

public static void deserializeObjectArray(
/**
* Deserializes {@code length} objects into the untyped {@code Object[]} in {@code arr}.
*
* <p>Partially deserialized values may be visible to the caller.
*/
static void deserializeObjectArray(
AsyncDeserializationContext context, CodedInputStream codedIn, Object arr, int length)
throws IOException, SerializationException {
for (int i = 0; i < length; ++i) {
context.deserialize(codedIn, arr, ARRAY_OBJECT_BASE_OFFSET + ARRAY_OBJECT_INDEX_SCALE * i);
}
}

public static void deserializeObjectArrayFully(
AsyncDeserializationContext context, CodedInputStream codedIn, Object arr, int length)
/**
* Deserializes {@code length} objects into the untyped {@code Object[]} in {@code arr}.
*
* <p>Only fully deserialized values will be visible to the caller.
*/
static void deserializeObjectArrayFully(
FlatDeserializationContext context, CodedInputStream codedIn, Object arr, int length)
throws IOException, SerializationException {
for (int i = 0; i < length; ++i) {
context.deserializeFully(
Expand Down
Loading

0 comments on commit 23d6d70

Please sign in to comment.