Skip to content

Commit 0092abb

Browse files
sryzaJoshRosen
authored andcommitted
Some minor cleanup after SPARK-4550.
JoshRosen this PR addresses the comments you left on #4450 after it got merged. Author: Sandy Ryza <[email protected]> Closes #5916 from sryza/sandy-spark-4550-cleanup and squashes the following commits: dee3d85 [Sandy Ryza] Some minor cleanup after SPARK-4550.
1 parent c688e3c commit 0092abb

File tree

2 files changed

+4
-9
lines changed

2 files changed

+4
-9
lines changed

core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ private[spark] abstract class BlockObjectWriter(val blockId: BlockId) extends Ou
5959
def write(key: Any, value: Any)
6060

6161
/**
62-
* Notify the writer that a record worth of bytes has been written with writeBytes.
62+
* Notify the writer that a record worth of bytes has been written with OutputStream#write.
6363
*/
6464
def recordWritten()
6565

@@ -215,12 +215,7 @@ private[spark] class DiskBlockObjectWriter(
215215

216216
objOut.writeKey(key)
217217
objOut.writeValue(value)
218-
numRecordsWritten += 1
219-
writeMetrics.incShuffleRecordsWritten(1)
220-
221-
if (numRecordsWritten % 32 == 0) {
222-
updateBytesWritten()
223-
}
218+
recordWritten()
224219
}
225220

226221
override def write(b: Int): Unit = throw new UnsupportedOperationException()

core/src/main/scala/org/apache/spark/util/collection/PartitionedSerializedPairBuffer.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,10 @@ private[spark] class PartitionedSerializedPairBuffer[K, V](
7171
if (keyStart < 0) {
7272
throw new Exception(s"Can't grow buffer beyond ${1 << 31} bytes")
7373
}
74-
kvSerializationStream.writeObject[Any](key)
74+
kvSerializationStream.writeKey[Any](key)
7575
kvSerializationStream.flush()
7676
val valueStart = kvBuffer.size
77-
kvSerializationStream.writeObject[Any](value)
77+
kvSerializationStream.writeValue[Any](value)
7878
kvSerializationStream.flush()
7979
val valueEnd = kvBuffer.size
8080

0 commit comments

Comments
 (0)