Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Byteskip minus1 fixing #74

Merged
merged 9 commits into from
Nov 12, 2018
Merged

Byteskip minus1 fixing #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7339cff
byteskip -1 fixed
tashrifbillah Nov 10, 2018
a54994b
adding test data for byteskip -1
tashrifbillah Nov 10, 2018
eef5b62
code path optimized, missing lines added
tashrifbillah Nov 10, 2018
f0fa781
np.fromstring()
tashrifbillah Nov 10, 2018
2bbe06a
minor correction
tashrifbillah Nov 10, 2018
7cc511b
making statements consistent with repo
tashrifbillah Nov 10, 2018
dae4782
unnecessary w/space removal
tashrifbillah Nov 10, 2018
4a144ef
byte skip -1 tests added
tashrifbillah Nov 12, 2018
f82bfcb
byte skip -1 tests added
tashrifbillah Nov 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions nrrd/reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,22 +339,30 @@ def read_data(header, fh=None, filename=None):
# Get the total number of data points by multiplying the size of each dimension together
total_data_points = header['sizes'].prod()

# If encoding is raw and byte skip is -1, then seek backwards to the data
# Otherwise skip the number of lines requested
if header['encoding'] == 'raw' and byte_skip == -1:
fh.seek(-dtype.itemsize * total_data_points, 2)
else:
# Skip the number of lines requested when line_skip >= 0
# Irrespective of the NRRD file having attached/detached header
# Lines are skipped before getting to the beginning of the data
if line_skip >= 0:
for _ in range(line_skip):
fh.readline()

# If a compression encoding is used, then byte skip AFTER decompressing
if header['encoding'] == 'raw':
# Skip the requested number of bytes and then parse the data using NumPy
else:
raise NRRDError('Invalid lineskip, allowed values are greater than or equal to 0')

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also added a sanity check for line skip as noted here

# Skip the requested number of bytes or seek backward, and then parse the data using NumPy
if byte_skip < -1:
raise NRRDError('Invalid byteskip, allowed values are greater than or equal to -1')
elif byte_skip >= 0:
fh.seek(byte_skip, os.SEEK_CUR)
elif byte_skip == -1 and header['encoding'] not in ['gzip', 'gz', 'bzip2', 'bz2']:
fh.seek(-dtype.itemsize * total_data_points, os.SEEK_END)
else:
# The only case left should be: byte_skip == -1 and header['encoding'] == 'gzip'
byte_skip = -dtype.itemsize * total_data_points

# If a compression encoding is used, then byte skip AFTER decompressing
if header['encoding'] == 'raw':
data = np.fromfile(fh, dtype)
elif header['encoding'] in ['ASCII', 'ascii', 'text', 'txt']:
# Skip the requested number of bytes and then parse the data using NumPy
fh.seek(byte_skip, os.SEEK_CUR)
data = np.fromfile(fh, dtype, sep=' ')
else:
# Handle compressed data now
Expand Down Expand Up @@ -423,7 +431,6 @@ def read(filename, custom_field_map=None):
"""

"""Read a NRRD file and return a tuple (data, header)."""

with open(filename, 'rb') as fh:
header = read_header(fh, custom_field_map)
data = read_data(header, fh, filename)
Expand Down
Binary file added nrrd/tests/data/BallBinary30x30x30.nii.gz
View file
Open in desktop
Binary file not shown.
14 changes: 14 additions & 0 deletions nrrd/tests/data/BallBinary30x30x30_byteskip_minus_five.nhdr
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
NRRD0004
# Complete NRRD file format specification at:
# http://teem.sourceforge.net/nrrd/format.html
type: short
dimension: 3
space: left-posterior-superior
sizes: 30 30 30
byte skip: -5
space directions: (1,0,0) (0,1,0) (0,0,1)
kinds: domain domain domain
endian: little
encoding: raw
space origin: (0,0,0)
data file: BallBinary30x30x30.raw
14 changes: 14 additions & 0 deletions nrrd/tests/data/BallBinary30x30x30_byteskip_minus_one.nhdr
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
NRRD0004
# Complete NRRD file format specification at:
# http://teem.sourceforge.net/nrrd/format.html
type: short
dimension: 3
space: left-posterior-superior
sizes: 30 30 30
byte skip: -1
space directions: (1,0,0) (0,1,0) (0,0,1)
kinds: domain domain domain
endian: little
encoding: raw
space origin: (0,0,0)
data file: BallBinary30x30x30.raw
14 changes: 14 additions & 0 deletions nrrd/tests/data/BallBinary30x30x30_byteskip_minus_one_nifti.nhdr
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
NRRD0004
# Complete NRRD file format specification at:
# http://teem.sourceforge.net/nrrd/format.html
type: short
dimension: 3
space: left-posterior-superior
sizes: 30 30 30
byte skip: -1
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The nifti file has the following header at the beginning before accessing image data:

sizeof_hdr      : 348
data_type       : b''
db_name         : b''
extents         : 0
session_error   : 0
regular         : b'r'
dim_info        : 0
dim             : [ 3 30 30 30  1  1  1  1]
intent_p1       : 0.0
intent_p2       : 0.0
intent_p3       : 0.0
intent_code     : none
datatype        : int16
bitpix          : 16
slice_start     : 0
pixdim          : [1. 1. 1. 1. 1. 0. 0. 0.]
vox_offset      : 0.0
scl_slope       : nan
scl_inter       : nan
slice_end       : 0
slice_code      : unknown
xyzt_units      : 10
cal_max         : 0.0
cal_min         : 0.0
slice_duration  : 0.0
toffset         : 0.0
glmax           : 0
glmin           : 0
descrip         : b'BYTESKIP -1 TEST'
aux_file        : b''
qform_code      : scanner
sform_code      : scanner
quatern_b       : 0.0
quatern_c       : 0.0
quatern_d       : 1.0
qoffset_x       : 0.0
qoffset_y       : 0.0
qoffset_z       : 0.0
srow_x          : [-1.  0.  0.  0.]
srow_y          : [ 0. -1.  0.  0.]
srow_z          : [0. 0. 1. 0.]
intent_name     : b''
magic           : b'n+1'

When we add byteskip: -1, the nifti data file as accessed from the end, only the required number of bytes. This way, we are introducing a great way of nifti to nrrd conversion. We should not need to use DWIConvert anymore. Just make a detached header, and point to existing nifti file as the data file and we are done. At the PNL, BWH we are developing our current nifti pipeline dealing with nrrd files this way.

space directions: (1,0,0) (0,1,0) (0,0,1)
kinds: domain domain domain
endian: little
encoding: gzip
space origin: (0,0,0)
data file: BallBinary30x30x30.nii.gz
Binary file added nrrd/tests/data/BallBinary30x30x30_gz_byteskip_minus_one.nrrd
View file
Open in desktop
Binary file not shown.
14 changes: 14 additions & 0 deletions nrrd/tests/data/BallBinary30x30x30_nifti.nhdr
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
NRRD0004
# Complete NRRD file format specification at:
# http://teem.sourceforge.net/nrrd/format.html
type: short
dimension: 3
space: left-posterior-superior
sizes: 30 30 30
# byte skip: -1
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The byte skip has been intentionally commented out here to point out the error when nifti file is attempted to read from the beginning.

space directions: (1,0,0) (0,1,0) (0,0,1)
kinds: domain domain domain
endian: little
encoding: gzip
space origin: (0,0,0)
data file: BallBinary30x30x30.nii.gz
56 changes: 55 additions & 1 deletion nrrd/tests/test_reading.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,46 @@ def test_read_detached_header_and_data(self):

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_detached_header_and_data_with_byteskip_minus1(self):
expected_header = self.expected_header
expected_header[u'data file'] = os.path.basename(RAW_DATA_FILE_PATH)
expected_header[u'byte skip'] = -1

data, header = nrrd.read(RAW_BYTESKIP_NHDR_FILE_PATH)

np.testing.assert_equal(self.expected_header, header)
np.testing.assert_equal(data, self.expected_data)

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_detached_header_and_nifti_data_with_byteskip_minus1(self):
expected_header = self.expected_header
expected_header[u'data file'] = os.path.basename(RAW_DATA_FILE_PATH)
expected_header[u'byte skip'] = -1
expected_header[u'encoding'] = 'gzip'
expected_header[u'data file'] = 'BallBinary30x30x30.nii.gz'

data, header = nrrd.read(GZ_BYTESKIP_NIFTI_NHDR_FILE_PATH)

np.testing.assert_equal(self.expected_header, header)
np.testing.assert_equal(data, self.expected_data)

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_detached_header_and_nifti_data(self):

with self.assertRaisesRegex(nrrd.NRRDError, 'Size of the data does not equal '
+ 'the product of all the dimensions: 27000-27176=-176'):
nrrd.read(GZ_NIFTI_NHDR_FILE_PATH)

def test_read_detached_header_and_data_with_byteskip_minus5(self):

with self.assertRaisesRegex(nrrd.NRRDError, 'Invalid byteskip, allowed values '
+'are greater than or equal to -1'):
nrrd.read(RAW_INVALID_BYTESKIP_NHDR_FILE_PATH)

def test_read_header_and_gz_compressed_data(self):
expected_header = self.expected_header
Expand All @@ -80,6 +120,20 @@ def test_read_header_and_gz_compressed_data(self):

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_header_and_gz_compressed_data_with_byteskip_minus1(self):
expected_header = self.expected_header
expected_header[u'encoding'] = 'gzip'
expected_header[u'type'] = 'int16'
expected_header[u'byte skip'] = -1

data, header = nrrd.read(GZ_BYTESKIP_NRRD_FILE_PATH)

np.testing.assert_equal(self.expected_header, header)
np.testing.assert_equal(data, self.expected_data)

# Test that the data read is able to be edited
self.assertTrue(data.flags['WRITEABLE'])

def test_read_header_and_bz2_compressed_data(self):
expected_header = self.expected_header
Expand Down Expand Up @@ -130,7 +184,7 @@ def test_read_dup_field_error_and_warn(self):
self.assertTrue("Duplicate header field: 'type'" in str(w[0].message))

self.assertEqual(expected_header, header)
nrrd.reader._NRRD_ALLOW_DUPLICATE_FIELD = False
nrrd.reader.ALLOW_DUPLICATE_FIELD = False

addisonElliott marked this conversation as resolved.
Show resolved Hide resolved
def test_read_header_and_ascii_1d_data(self):
expected_header = {u'dimension': 1,
Expand Down
5 changes: 5 additions & 0 deletions nrrd/tests/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,15 @@
DATA_DIR_PATH = os.path.join(os.path.dirname(__file__), 'data')
RAW_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30.nrrd')
RAW_NHDR_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30.nhdr')
RAW_BYTESKIP_NHDR_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_byteskip_minus_one.nhdr')
GZ_BYTESKIP_NIFTI_NHDR_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_byteskip_minus_one_nifti.nhdr')
GZ_NIFTI_NHDR_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_nifti.nhdr')
RAW_INVALID_BYTESKIP_NHDR_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_byteskip_minus_five.nhdr')
RAW_DATA_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30.raw')
GZ_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_gz.nrrd')
BZ2_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_bz2.nrrd')
GZ_LINESKIP_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_gz_lineskip.nrrd')
GZ_BYTESKIP_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'BallBinary30x30x30_gz_byteskip_minus_one.nrrd')
RAW_4D_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'test_simple4d_raw.nrrd')

ASCII_1D_NRRD_FILE_PATH = os.path.join(DATA_DIR_PATH, 'test1d_ascii.nrrd')
Expand Down