Skip to content

Commit

Permalink
Merge pull request #263 from NeurodataWithoutBorders/update-basic-tut…
Browse files Browse the repository at this point in the history 
…orials

Update Basic Usage and Conversion Tutorials
  • Loading branch information
@lawrence-mbf
lawrence-mbf authored Feb 9, 2021
2 parents e748549 + be650ab commit c166b1a
Show file tree
Hide file tree
Showing 11 changed files with 567 additions and 549 deletions.
6 changes: 6 additions & 0 deletions +tests/+unit/dataStubTest.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,10 @@ function testRegionRead(testCase)
testCase.verifyEqual(stub(primeInd, :, 1), data(primeInd, :, 1));
testCase.verifyEqual(stub(primeInd, [1 2 5]), data(primeInd, [1 2 5]));
testCase.verifyEqual(stub([1 25], [1 5], [1 4], [1 2], [1 5]), data([1 25], [1 5], [1 4], [1 2], [1 5]));

% test duplicate indices
testCase.verifyEqual(stub([1 1 1 1]), data([1 1 1 1]));

% test out of order indices
testCase.verifyEqual(stub([5 4 3 2 2]), data([5 4 3 2 2]));
end
15 changes: 8 additions & 7 deletions +types/+untyped/+datastub/findShapes.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
assert(isvector(indices),...
'MatNwb:DataStub:FindShapes:InvalidShape',...
'Indices cannot be matrices.');
indices = sort(indices);
indices = unique(indices);
shapes = {};
while ~isempty(indices)
BlockSelection = findOptimalBlock(indices);
Expand All @@ -37,7 +37,7 @@
end
stop = 1;
start = 1;
step = 0;
step = 1;
count = 0;
for i = 1:length(indices)
tempStart = indices(i);
Expand All @@ -47,19 +47,20 @@
end
for j = 1:(length(indices)-i)
tempStep = indices(i+j) - indices(i);
for k = fliplr(i:length(indices))
for k = length(indices):-1:i
tempStop = indices(k);
idealRange = tempStart:tempStep:tempStop;
tempRange = intersect(indices, idealRange, 'stable');
if length(tempRange) <= count
rangeMatches = ismembc(idealRange, indices);
numMatches = sum(rangeMatches);
if numMatches <= count
% number of intersected items is shorter than what we have.
break;
end
if isequal(tempRange, idealRange)
if all(rangeMatches)
start = tempStart;
step = tempStep;
stop = tempStop;
count = length(tempRange);
count = numMatches;
break;
end
end
Expand Down
133 changes: 84 additions & 49 deletions +types/+untyped/DataStub.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,10 +176,11 @@
dims = obj.dims;
rank = length(dims);
for i = 1:length(varargin)
if ischar(varargin{i})
ind = varargin{i};
if ischar(ind) || isempty(ind)
continue;
end
validateattributes(varargin{i}, {'numeric'}, {'vector', '<=', dims(i)});
validateattributes(ind, {'numeric'}, {'vector', '<=', dims(i)});
end
shapes = getShapes(varargin, dims);

Expand Down Expand Up @@ -208,8 +209,17 @@
shapeInd(iterateInd) = shapeInd(iterateInd) + 1;
shapeInd(1:(iterateInd-1)) = 1;
end

memSize = getMemSize(varargin, dims);
memSize = zeros(1, rank);
for i = 1:rank
for j = 1:length(shapes{i})
Selection = shapes{i}{j};
if isa(Selection, 'types.untyped.datastub.shape.Point')
memSize(i) = memSize(i) + 1;
else
memSize(i) = memSize(i) + Selection.length;
end
end
end
memSid = H5S.create_simple(length(memSize), fliplr(memSize), []);
% read data.
fid = H5F.open(obj.filename);
Expand All @@ -220,6 +230,76 @@
H5S.close(memSid);
H5S.close(sid);

expectedSize = dims;
for i = 1:length(varargin)
if ~ischar(varargin{i})
expectedSize(i) = length(varargin{i});
end
end

if ischar(varargin{end})
% dangling ':' where leftover dimensions are folded into
% the last selection.
selDimInd = length(varargin);
expectedSize = [expectedSize(1:(selDimInd-1)) prod(dims(selDimInd:end))];
else
expectedSize = expectedSize(1:length(varargin));
end

if isscalar(expectedSize)
expectedSize = [1 expectedSize];
end

selections = varargin;
openSelInd = find(cellfun('isclass', selections, 'char'));
for i = 1:length(openSelInd)
selections{i} = 1:dims(i);
end
data = reorderLoadedData(data, selections);
data = reshape(data, expectedSize);

function reordered = reorderLoadedData(data, selections)
% dataset loading does not account for duplicate or unordered
% indices so we have to re-order everything here.
% we presume data is the indexed values of a unique(ind)
if isempty(data)
reordered = data;
return;
end

indKey = cell(size(selections));
isSelectionNormal = false(size(selections)); % that is, without duplicates or out of order.
for i = 1:length(indKey)
indKey{i} = unique(selections{i});
isSelectionNormal = isequal(indKey{i}, selections{i});
end
if all(isSelectionNormal)
reordered = data;
return;
end
indKeyIndMax = cellfun('length', indKey);
if isscalar(indKeyIndMax)
reordered = repmat(data(1), indKeyIndMax, 1);
else
reordered = repmat(data(1), indKeyIndMax);
end
indKeyInd = ones(size(selections));
while true
selInd = cell(size(selections));
for i = 1:length(selections)
selInd{i} = selections{i} == indKey{i}(indKeyInd(i));
end
indKeyIndArgs = num2cell(indKeyInd);
reordered(selInd{:}) = data(indKeyIndArgs{:});
indKeyIndNextInd = find(indKeyIndMax ~= indKeyInd, 1, 'last');
if isempty(indKeyIndNextInd)
break;
end
indKeyInd(indKeyIndNextInd) = indKeyInd(indKeyIndNextInd) + 1;
indKeyInd((indKeyIndNextInd+1):end) = 1;
end
end

function shapes = getShapes(selections, dims)
rank = length(dims);
shapes = cell(1, rank); % cell array of cell arrays of shapes
Expand All @@ -240,27 +320,6 @@
end
end
end

function memSize = getMemSize(selections, dims)
% replace dims with number of selections
indexSelections = find(~cellfun('isclass', selections, 'char'));
vararginSizes = cellfun('length', selections);
dims(indexSelections) = vararginSizes(indexSelections);

% case where varargin rank is smaller than actual data
% space rank.
selectionRank = length(selections);
fileSpaceRank = length(dims);
isOpenEnded = ischar(selections{end});
if fileSpaceRank > selectionRank && ~isOpenEnded
% when there are no trailing ':' then the remainder
% dims are scalar. Otherwise, just load the rest of the
% data.
scalarDims = selectionRank + 1;
dims(scalarDims:end) = 1;
end
memSize = dims;
end
end

function refs = export(obj, fid, fullpath, refs)
Expand Down Expand Up @@ -351,30 +410,6 @@
'Cannot index into %d dimensions when max rank is %d',...
selectionRank, rank);
data = obj.load_mat_style(CurrentSubRef.subs{:});
expectedSize = dims;
for i = 1:length(CurrentSubRef.subs)
if ~ischar(CurrentSubRef.subs{i})
expectedSize(i) = length(CurrentSubRef.subs{i});
end
end

if ischar(CurrentSubRef.subs{end})
% dangling ':' where leftover dimensions are folded into
% the last selection.
selDimInd = length(CurrentSubRef.subs);
expectedSize = [expectedSize(1:(selDimInd-1)) prod(dims(selDimInd:end))];
else
expectedSize = expectedSize(1:length(CurrentSubRef.subs));
end

if isscalar(expectedSize)
expectedSize = [1 expectedSize];
end

if ~isequal(size(data), expectedSize)
data = reshape(data, expectedSize);
end

if isscalar(S)
B = data;
else
Expand Down
4 changes: 4 additions & 0 deletions +types/+untyped/MetaClass.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,11 @@
end
end

refLen = length(refs);
refs = obj.write_base(fid, fullpath, refs);
if refLen ~= length(refs)
return;
end

uuid = char(java.util.UUID.randomUUID().toString());
if isa(obj, 'NwbFile')
Expand Down
53 changes: 42 additions & 11 deletions +types/+util/+dynamictable/addRow.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ function addRow(DynamicTable, varargin)
rv = p.Results.(rn);

if isKey(TypeMap, rn)
TypeStruct = TypeMap(rn);
validateattributes(rv, {TypeStruct.type}, {'size', [NaN TypeStruct.dims(2:end)]});
rv = validateType(TypeMap(rn), rv);
else
assert(iscellstr(rv) || ~iscell(rv),...
'MatNWB:DynamicTable:AddRow:InvalidCellArray',...
Expand Down Expand Up @@ -132,6 +131,19 @@ function addRow(DynamicTable, varargin)
end
end

function rv = validateType(TypeStruct, rv)
if strcmp(TypeStruct.type, 'cellstr')
assert(iscellstr(rv) || (ischar(rv) && 1 == size(rv, 1)),...
'MatNWB:DynamicTable:AddRow:InvalidType',...
'Type of value must be a cell array of character vectors or a scalar character');
if ischar(rv)
rv = {rv};
end
else
validateattributes(rv, {TypeStruct.type}, {'size', [NaN TypeStruct.dims(2:end)]});
end
end

function TypeMap = constructTypeMap(DynamicTable)
TypeMap = containers.Map;
if isempty(DynamicTable.id.data)...
Expand All @@ -152,13 +164,19 @@ function addRow(DynamicTable, varargin)
else
colval = colVecData.data(1);
end
TypeStruct.type = class(colval);

if iscellstr(colval)
TypeStruct.type = 'cellstr';
else
TypeStruct.type = class(colval);
end

if isa(colVecData.data, 'types.untyped.DataPipe')
TypeStruct.dims = colVecData.data.internal.maxSize;
else
TypeStruct.dims = size(colVecData.data);
end

TypeMap(colnm) = TypeStruct;
end
end
Expand Down Expand Up @@ -187,27 +205,40 @@ function appendData(DynamicTable, column, data, index)
end

if ~isempty(index)
if isa(VecData.data, 'types.untyped.DataPipe')
raggedIndex = VecData.data.offset;
else
raggedIndex = size(VecData.data, 1);
end

if isprop(DynamicTable, index)
VecInd = DynamicTable.(index);
else
VecInd = DynamicTable.vectorindex.get(index);
end

if isa(VecInd.data, 'types.untyped.DataPipe')
VecInd.data.append(raggedIndex);
if 0 == VecInd.data.dims
raggedOffset = 0;
else
raggedOffset = VecInd.data.load(VecInd.data.dims);
end
else
VecInd.data = [VecInd.data; raggedIndex];
if isempty(VecInd.data)
raggedOffset = 0;
else
raggedOffset = VecInd.data(end);
end
end

raggedValue = raggedOffset + size(data, 1);
if isa(VecInd.data, 'types.untyped.DataPipe')
VecInd.data.append(raggedValue);
else
VecInd.data = [VecInd.data; raggedValue];
end
end

if isa(VecData.data, 'types.untyped.DataPipe')
VecData.data.append(data);
else
if ischar(data)
data = {data};
end
VecData.data = [VecData.data; data];
end
end
24 changes: 22 additions & 2 deletions +types/+util/+dynamictable/getIndex.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,34 @@
'MatNWB:GetIndex:InvalidColumn',...
'Column name not found `%s`', column);
indexName = '';

vecIndKeys = keys(DynamicTable.vectorindex);
for i = 1:length(vecIndKeys)
vik = vecIndKeys{i};
VecInd = DynamicTable.vectorindex.get(vik);
if endsWith(VecInd.target.path, ['/' column])
if isVecIndColumn(DynamicTable.vectorindex.get(vik), column)
indexName = vik;
return;
end
end

DynamicTableProps = properties(DynamicTable);
isPropVecInd = false(size(DynamicTableProps));
for i = 1:length(DynamicTableProps)
isPropVecInd(i) = isa(DynamicTable.(DynamicTableProps{i}), 'types.hdmf_common.VectorIndex');
end

DynamicTableProps = DynamicTableProps(isPropVecInd);
for i = 1:length(DynamicTableProps)
vik = DynamicTableProps{i};
VecInd = DynamicTable.(vik);
if isVecIndColumn(VecInd, column)
indexName = vik;
return;
end
end
end

function tf = isVecIndColumn(VectorIndex, column)
tf = endsWith(VectorIndex.target.path, ['/' column]);
end

Loading

0 comments on commit c166b1a

Please sign in to comment.