Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support boolean values #467

Merged
merged 8 commits into from
Oct 20, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion +io/getBaseType.m
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
elseif strcmp(type, 'single')
id = 'H5T_IEEE_F32LE';
elseif strcmp(type, 'logical')
id = 'H5T_STD_I32LE';
id = H5T.enum_create('H5T_STD_I8LE');
H5T.enum_insert(id, 'FALSE', 0);
H5T.enum_insert(id, 'TRUE', 1);
elseif startsWith(type, {'int' 'uint'})
prefix = 'H5T_STD_';
pattern = 'int%d';
Expand Down
7 changes: 5 additions & 2 deletions +io/getMatType.m
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
function type = getMatType(tid)
%GETMATTYPE Given HDF5 type ID, returns string indicating MATLAB type.
% only works for numeric values.
%GETMATTYPE Given HDF5 type ID, returns string indicating probable MATLAB type.
if H5T.equal(tid, 'H5T_IEEE_F64LE')
type = 'double';
elseif H5T.equal(tid, 'H5T_IEEE_F32LE')
Expand All @@ -27,6 +26,10 @@
type = 'types.untyped.ObjectView';
elseif H5T.equal(tid, 'H5T_STD_REF_DSETREG')
type = 'types.untyped.RegionView';
elseif io.isBool(tid)
type = 'logical';
elseif H5ML.get_constant_value('H5T_COMPOUND') == H5T.get_class(tid)
type = 'table';
else
if isa(tid, 'H5ML.id')
identifier = tid.identifier;
Expand Down
2 changes: 0 additions & 2 deletions +io/getMatTypeSize.m
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
function typeSize = getMatTypeSize(type)
%GETMATSIZE Summary of this function goes here
% Detailed explanation goes here
switch (type)
case {'uint8', 'int8'}
typeSize = 1;
Expand Down
43 changes: 43 additions & 0 deletions +io/isBool.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
function tf = isBool(source)
%ISBOOL Checks a h5read Type struct if the defined type is a h5py boolean.
if isstruct(source)
tf = isBoolFromTypeStruct(source);
elseif isa(source, 'H5ML.id')
tf = isBoolFromTypeId(source);
else
error('NWB:IO:IsBool:InvalidArgument',...
['isBool(source) must provide either a `h5info` Type struct or a ' ...
'type id from low-level HDF5.']);
end
end

function tf = isBoolFromTypeStruct(Type)
tf = all([...
strcmp('H5T_STD_I8LE', Type.Type),...
2 == length(Type.Member),...
all(strcmp({'FALSE', 'TRUE'}, sort({Type.Member.Name}))),...
all([0,1] == sort([Type.Member.Value]))...
]);
end

function tf = isBoolFromTypeId(tid)
% we are more loose with the type id implementation.
% any enum with any backing type can be defined so long as member values
% 'FALSE' exists and is equal to 0, and 'TRUE' exists and is equal to 1.
if H5ML.get_constant_value('H5T_ENUM') ~= H5T.get_class(tid)
tf = false;
return;
end

try
hasFalse = 0 == H5T.enum_valueof(tid, 'FALSE');
hasTrue = 1 == H5T.enum_valueof(tid, 'TRUE');
tf = hasFalse && hasTrue;
catch ME
if ~contains(ME.message, 'string doesn''t exist in the enumeration type')
% unknown error.
rethrow(ME);
end
tf = false;
end
end
4 changes: 2 additions & 2 deletions +io/mapData2H5.m
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@
%will throw errors if refdata DNE. Caught at NWBData level.
data = io.getRefData(fid, data);
case 'logical'
%In HDF5, HBOOL is mapped to INT32LE
data = int32(data);
% encode as int8 values.
data = int8(data);
case 'char'
data = mat2cell(data, size(data, 1));
case {'cell', 'datetime'}
Expand Down
33 changes: 23 additions & 10 deletions +io/parseAttributes.m
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,29 @@
attributes(deleteMask) = [];
for i=1:length(attributes)
attr = attributes(i);
if strcmp(attr.Datatype.Class, 'H5T_REFERENCE')
fid = H5F.open(filename, 'H5F_ACC_RDONLY', 'H5P_DEFAULT');
aid = H5A.open_by_name(fid, context, attr.Name);
tid = H5A.get_type(aid);
args(attr.Name) = io.parseReference(aid, tid, attr.Value);
H5T.close(tid);
H5A.close(aid);
H5F.close(fid);
else
args(attr.Name) = attr.Value;
switch attr.Datatype.Class
case 'H5T_REFERENCE'
fid = H5F.open(filename, 'H5F_ACC_RDONLY', 'H5P_DEFAULT');
aid = H5A.open_by_name(fid, context, attr.Name);
tid = H5A.get_type(aid);
attributeValue = io.parseReference(aid, tid, attr.Value);
H5T.close(tid);
H5A.close(aid);
H5F.close(fid);
case 'H5T_ENUM'
if io.isBool(attr.Datatype.Type)
% attr.Value should be cell array of strings here since
% MATLAB can't have arbitrary enum values.
attributeValue = strcmp('TRUE', attr.Value);
else
warning('MatNWB:Attribute:UnknownEnum', ...
['Encountered unknown enum under field `%s` with %d members. ' ...
'Will be saved as cell array of characters.'], ...
attr.Name, length(attr.Datatype.Type.Member));
end
otherwise
attributeValue = attr.Value;
end
args(attr.Name) = attributeValue;
end
end
26 changes: 19 additions & 7 deletions +io/parseCompound.m
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
subtids = cell(1, ncol);
ref_i = false(1, ncol);
char_i = false(1, ncol);
bool_i = false(1,ncol);
for i = 1:ncol
subtid = H5T.get_member_type(tid, i-1);
subtids{i} = subtid;
Expand All @@ -19,17 +20,20 @@
%if not variable len (which would make it a cell array)
%then mark for transpose
char_i(i) = ~H5T.is_variable_str(subtid);
case H5ML.get_constant_value('H5T_ENUM')
bool_i(i) = io.isBool(subtid);
otherwise
%do nothing
end
end
propnames = fieldnames(data);

fields = fieldnames(data);
if any(ref_i)
%resolve references by column
reftids = subtids(ref_i);
refPropNames = propnames(ref_i);
for j=1:length(refPropNames)
rpname = refPropNames{j};
refFields = fields(ref_i);
for j=1:length(refFields)
rpname = refFields{j};
refdata = data.(rpname);
reflist = cell(size(refdata, 2), 1);
for k=1:size(refdata, 2)
Expand All @@ -43,11 +47,19 @@
if any(char_i)
%transpose character arrays because they are column-ordered
%when read
charPropNames = propnames(char_i);
for j=1:length(charPropNames)
cpname = charPropNames{j};
charFields = fields(char_i);
for j=1:length(charFields)
cpname = charFields{j};
data.(cpname) = data.(cpname) .';
end
end

if any(bool_i)
% convert column data to proper logical arrays/matrices
for f=fields{bool_i}
data.(f) = strcmp('TRUE', data.(f));
end
end

data = struct2table(data);
end
35 changes: 22 additions & 13 deletions +io/parseDataset.m
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,28 @@
H5T.close(tid);
elseif ~strcmp(dataspace.Type, 'simple')
data = H5D.read(did);
if iscellstr(data) && 1 == length(data)
data = data{1};
elseif ischar(data)
if datetime(version('-date')) < datetime('25-Feb-2020')
% MATLAB 2020a fixed string support for HDF5, making reading strings
% "consistent"
data = data .';
end
datadim = size(data);
if datadim(1) > 1
%multidimensional strings should become cellstr
data = strtrim(mat2cell(data, ones(datadim(1), 1), datadim(2)));
end

switch datatype.Class
case 'H5T_STRING'
if datetime(version('-date')) < datetime('25-Feb-2020')
% MATLAB 2020a fixed string support for HDF5, making
% reading strings "consistent" with regular use.
data = data .';
end
datadim = size(data);
if datadim(1) > 1
%multidimensional strings should become cellstr
data = strtrim(mat2cell(data, ones(datadim(1), 1), datadim(2)));
end
case 'H5T_ENUM'
if io.isBool(datatype.Type)
data = strcmp('TRUE', data);
else
warning('MatNWB:Dataset:UnknownEnum', ...
['Encountered unknown enum under field `%s` with %d members. ' ...
'Will be saved as cell array of characters.'], ...
info.Name, length(datatype.Type.Member));
end
end
else
sid = H5D.get_space(did);
Expand Down
20 changes: 13 additions & 7 deletions +io/writeCompound.m
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ function writeCompound(fid, fullpath, data, varargin)
elseif isa(data, 'containers.Map')
names = keys(data);
vals = values(data, names);

s = struct();
for i=1:length(names)
s.(misc.str2validName(names{i})) = vals{i};
end
data = s;
end

%convert to scalar struct
names = fieldnames(data);
if isempty(names)
Expand Down Expand Up @@ -42,7 +42,7 @@ function writeCompound(fid, fullpath, data, varargin)
data.(names{i}) = [val{:}];
val = val{1};
end

classes{i} = class(val);
tids{i} = io.getBaseType(classes{i});
sizes(i) = H5T.get_size(tids{i});
Expand All @@ -64,6 +64,12 @@ function writeCompound(fid, fullpath, data, varargin)
ref_i = strcmp(classes, 'types.untyped.ObjectView') |...
strcmp(classes, 'types.untyped.RegionView');

% convert logical values
boolNames = names(strcmp(classes, 'logical'));
for iField = 1:length(boolNames)
data.(boolNames{iField}) = strcmp('TRUE', data.(boolNames{iField}));
end

%transpose numeric column arrays to row arrays
% reference and str arrays are handled below
transposeNames = names(~ref_i);
Expand All @@ -81,8 +87,8 @@ function writeCompound(fid, fullpath, data, varargin)
end

try
sid = H5S.create_simple(1, numrows, []);
did = H5D.create(fid, fullpath, tid, sid, 'H5P_DEFAULT');
sid = H5S.create_simple(1, numrows, []);
did = H5D.create(fid, fullpath, tid, sid, 'H5P_DEFAULT');
catch ME
if contains(ME.message, 'name already exists')
did = H5D.open(fid, fullpath);
Expand All @@ -97,8 +103,8 @@ function writeCompound(fid, fullpath, data, varargin)
if is_chunked
H5D.set_extent(did, dims);
else
warning('Attempted to change size of continuous compound `%s`. Skipping.',...
fullpath);
warning('Attempted to change size of continuous compound `%s`. Skipping.',...
fullpath);
end
end
H5P.close(create_plist);
Expand Down
12 changes: 11 additions & 1 deletion +tests/+system/DynamicTableTest.m
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
methods
function addContainer(~, file)
colnames = {'start_time', 'stop_time', 'randomvalues', ...
'random_multi', 'stringdata', 'compound_data'};
'random_multi', 'stringdata', 'compound_data', 'compound_data_struct'};
%add trailing nulls to columnames
for c =1:length(colnames)
colnames{c} = char([double(colnames{c}) zeros(1,randi(10))]);
Expand Down Expand Up @@ -44,6 +44,9 @@ function addContainer(~, file)
);
compound_data = types.hdmf_common.VectorData(...
'description', 'compound data column', ...
'data', table(rand(nrows, 1), rand(nrows, 1), 'VariableNames', {'a', 'b'}));
compound_data_struct = types.hdmf_common.VectorData(...
'description', 'compound data using struct', ...
'data', struct('a', rand(nrows, 1), 'b', rand(nrows, 1)));
file.intervals_trials = types.core.TimeIntervals(...
'description', 'test dynamic table column',...
Expand All @@ -56,6 +59,7 @@ function addContainer(~, file)
'random_multi', multi_col, ...
'stringdata', str_col, ...
'compound_data', compound_data, ...
'compound_data_struct', compound_data_struct, ...
'id', types.hdmf_common.ElementIdentifiers('data', ids(1:nrows)) ...
);
% check table configuration.
Expand Down Expand Up @@ -299,6 +303,12 @@ function getRowRoundtripTest(testCase)
ActualTable = ActualFile.intervals_trials;
ExpectedTable = testCase.file.intervals_trials;

% even if struct is passed in. It is still read back as a
% table. So we cheat a bit here since this is expected a2a.
CompoundStructVector = ExpectedTable.vectordata.get('compound_data_struct');
ExpectedCompoundStruct = CompoundStructVector.data;
CompoundStructVector.data = struct2table(ExpectedCompoundStruct);

testCase.verifyEqual(ExpectedTable.getRow(5), ActualTable.getRow(5));
testCase.verifyEqual(ExpectedTable.getRow([5 6]), ActualTable.getRow([5 6]));
testCase.verifyEqual(ExpectedTable.getRow([13, 19], 'useId', true),...
Expand Down
12 changes: 12 additions & 0 deletions +tests/+unit/boolSchema/bool.bools.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
groups:
- neurodata_type_def: BoolContainer
neurodata_type_inc: NWBDataInterface
datasets:
- name: data
dtype: bool
shape:
- null
attributes:
- name: attribute
dtype: bool
doc: bool as attribute
7 changes: 7 additions & 0 deletions +tests/+unit/boolSchema/bool.namespace.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespaces:
- full_name: Boolean Values Schema Test
name: bool
schema:
- namespace: core
- source: bool.bools.yaml
version: 1.0.0
35 changes: 35 additions & 0 deletions +tests/+unit/boolTest.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
function tests = boolTest()
tests = functiontests(localfunctions);
end

function setupOnce(testCase)
rootPath = fullfile(fileparts(mfilename('fullpath')), '..', '..');
testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath));
end

function setup(testCase)
testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture);
generateCore('savedir', '.');
schemaPath = fullfile(misc.getMatnwbDir(),...
'+tests', '+unit', 'boolSchema', 'bool.namespace.yaml');
generateExtension(schemaPath, 'savedir', '.');
rehash();
end

function testIo(testCase)
nwb = NwbFile(...
'identifier', 'BOOL',...
'session_description', 'test bool',...
'session_start_time', datetime());
boolContainer = types.bool.BoolContainer(...
'data', logical(randi([0,1], 100, 1)), ...
'attribute', false);
scalarBoolContainer = types.bool.BoolContainer(...
'data', false, ...
'attribute', true);
nwb.acquisition.set('bool', boolContainer);
nwb.acquisition.set('scalarbool', scalarBoolContainer);
nwb.export('test.nwb');
nwbActual = nwbRead('test.nwb', 'ignorecache');
tests.util.verifyContainerEqual(testCase, nwbActual, nwb);
end
Loading