Skip to content

Drop old frame splitting in NumPy serialization #3868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
quasiben merged 3 commits into from
Jun 9, 2020
Merged

Drop old frame splitting in NumPy serialization #3868

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5110ae3
Drop old frame splitting in NumPy serialization
jakirkham Jun 9, 2020
8914042
Drop `merge_frames` in NumPy deserialization
jakirkham Jun 9, 2020
3fc13a5
Unpack `frames` in NumPy deserialization
jakirkham Jun 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 4 additions & 9 deletions distributed/protocol/numpy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import math
import numpy as np

from .utils import frame_split_size, merge_frames
from .serialize import dask_serialize, dask_deserialize
from . import pickle

Expand Down Expand Up @@ -87,10 +86,7 @@ def serialize_numpy_ndarray(x):
if broadcast_to is not None:
header["broadcast_to"] = broadcast_to

if x.nbytes > 1e5:
frames = frame_split_size(data)
else:
frames = [data]
frames = [data]

header["lengths"] = [x.nbytes]

Expand All @@ -100,11 +96,10 @@ def serialize_numpy_ndarray(x):
@dask_deserialize.register(np.ndarray)
def deserialize_numpy_ndarray(header, frames):
with log_errors():
if len(frames) > 1:
frames = merge_frames(header, frames)
(frame,) = frames

if header.get("pickle"):
return pickle.loads(frames[0])
return pickle.loads(frame)

is_custom, dt = header["dtype"]
if is_custom:
Expand All @@ -117,7 +112,7 @@ def deserialize_numpy_ndarray(header, frames):
else:
shape = header["shape"]

x = np.ndarray(shape, dtype=dt, buffer=frames[0], strides=header["strides"])
x = np.ndarray(shape, dtype=dt, buffer=frame, strides=header["strides"])

return x

Expand Down