Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Byteskip minus1 fixing #74

Merged
merged 9 commits into from
Nov 12, 2018
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 16 additions & 15 deletions nrrd/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,22 +339,24 @@ def read_data(header, fh=None, filename=None):
# Get the total number of data points by multiplying the size of each dimension together
total_data_points = header['sizes'].prod()

# If encoding is raw and byte skip is -1, then seek backwards to the data
# Otherwise skip the number of lines requested
if header['encoding'] == 'raw' and byte_skip == -1:
fh.seek(-dtype.itemsize * total_data_points, 2)
else:
for _ in range(line_skip):
fh.readline()

# If a compression encoding is used, then byte skip AFTER decompressing
if header['encoding'] == 'raw':
# Skip the requested number of bytes and then parse the data using NumPy
# Skip the number of lines requested
for _ in range(line_skip):
addisonElliott marked this conversation as resolved.
Show resolved Hide resolved
fh.readline()

# Skip the requested number of bytes or seek backward, and then parse the data using NumPy
if byte_skip < -1:
raise Exception('Invalid byteskip, allowed values are greater than or equal to -1')
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Switch to NRRDError rather than Exception

Suggested change
raise Exception('Invalid byteskip, allowed values are greater than or equal to -1')
raise NRRDError('Invalid byteskip, allowed values are greater than or equal to -1')

Copy link
Contributor Author

@tashrifbillah tashrifbillah Nov 10, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking about that, but then I thought shouldn't it be just Exception? But yeah, we can do that.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm open to hearing your reasoning, but my thought process is to use NRRDError for any errors in pynrrd. I'm treating NRRDError as the exceptions for the module.

elif byte_skip >= 0:
fh.seek(byte_skip, os.SEEK_CUR)
elif byte_skip == -1 and header['encoding'] not in ['gzip', 'gz', 'bzip2', 'bz2']:
fh.seek(-dtype.itemsize * total_data_points, os.SEEK_END)
else: # The only case left should be: byte_skip == -1 and header['encoding'] == 'gzip'
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a nitpicky thing, but just to stick with the syntax of the repository (and my personal preference), I would move this line comment to the next line to be on it's own line.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, doing it.

byte_skip = -dtype.itemsize*total_data_points
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
byte_skip = -dtype.itemsize*total_data_points
byte_skip = -dtype.itemsize * total_data_points

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure


# If a compression encoding is used, then byte skip AFTER decompressing
if header['encoding'] == 'raw':
data = np.fromfile(fh, dtype)
elif header['encoding'] in ['ASCII', 'ascii', 'text', 'txt']:
# Skip the requested number of bytes and then parse the data using NumPy
fh.seek(byte_skip, os.SEEK_CUR)
data = np.fromfile(fh, dtype, sep=' ')
else:
# Handle compressed data now
Expand All @@ -380,7 +382,7 @@ def read_data(header, fh=None, filename=None):

# Byte skip is applied AFTER the decompression. Skip first x bytes of the decompressed data and parse it using
# NumPy
data = np.fromstring(decompressed_data[byte_skip:], dtype)
data = np.fromstring(decompressed_data[byte_skip:], dtype)

# Close the file
# Even if opened using with keyword, closing it does not hurt
Expand Down Expand Up @@ -423,7 +425,6 @@ def read(filename, custom_field_map=None):
"""

"""Read a NRRD file and return a tuple (data, header)."""

with open(filename, 'rb') as fh:
header = read_header(fh, custom_field_map)
data = read_data(header, fh, filename)
Expand Down
14 changes: 14 additions & 0 deletions nrrd/tests/data/BallBinary30x30x30_byteskip_minus_one.nhdr
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
NRRD0004
# Complete NRRD file format specification at:
# http://teem.sourceforge.net/nrrd/format.html
type: short
dimension: 3
space: left-posterior-superior
sizes: 30 30 30
byte skip: -1
space directions: (1,0,0) (0,1,0) (0,0,1)
kinds: domain domain domain
endian: little
encoding: raw
space origin: (0,0,0)
data file: BallBinary30x30x30.raw
Binary file not shown.
29 changes: 28 additions & 1 deletion nrrd/tests/test_reading.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,19 @@ def test_read_detached_header_and_data(self):

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_detached_header_and_data_with_byteskip_minus1(self):
expected_header = self.expected_header
expected_header[u'data file'] = os.path.basename(RAW_DATA_FILE_PATH)
expected_header[u'byte skip'] = -1

data, header = nrrd.read(RAW_BYTESKIP_NHDR_FILE_PATH)

np.testing.assert_equal(self.expected_header, header)
np.testing.assert_equal(data, self.expected_data)

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_header_and_gz_compressed_data(self):
expected_header = self.expected_header
Expand All @@ -80,6 +93,20 @@ def test_read_header_and_gz_compressed_data(self):

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_header_and_gz_compressed_data_with_byteskip_minus1(self):
expected_header = self.expected_header
expected_header[u'encoding'] = 'gzip'
expected_header[u'type'] = 'int16'
expected_header[u'byte skip'] = -1

data, header = nrrd.read(GZ_BYTESKIP_NRRD_FILE_PATH)

np.testing.assert_equal(self.expected_header, header)
np.testing.assert_equal(data, self.expected_data)

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_header_and_bz2_compressed_data(self):
expected_header = self.expected_header
Expand Down Expand Up @@ -130,7 +157,7 @@ def test_read_dup_field_error_and_warn(self):
self.assertTrue("Duplicate header field: 'type'" in str(w[0].message))

self.assertEqual(expected_header, header)
nrrd.reader._NRRD_ALLOW_DUPLICATE_FIELD = False
nrrd.reader.ALLOW_DUPLICATE_FIELD = False

addisonElliott marked this conversation as resolved.
Show resolved Hide resolved
def test_read_header_and_ascii_1d_data(self):
expected_header = {u'dimension': 1,
Expand Down
2 changes: 2 additions & 0 deletions nrrd/tests/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
DATA_DIR_PATH = os.path.join(os.path.dirname(__file__), 'data')
RAW_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30.nrrd')
RAW_NHDR_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30.nhdr')
RAW_BYTESKIP_NHDR_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_byteskip_minus_one.nhdr')
RAW_DATA_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30.raw')
GZ_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_gz.nrrd')
BZ2_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_bz2.nrrd')
GZ_LINESKIP_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_gz_lineskip.nrrd')
GZ_BYTESKIP_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_gz_byteskip_minus_one.nrrd')
RAW_4D_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'test_simple4d_raw.nrrd')

ASCII_1D_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'test1d_ascii.nrrd')
Expand Down