From eefccf3efeb8e3c309b2262f7d683f42fffb3d99 Mon Sep 17 00:00:00 2001 From: Qianqian Fang Date: Sat, 6 Jun 2020 13:48:41 -0400 Subject: [PATCH] call jsonencode/decode in jsave/jload, parse embedded jdata struct --- jdatadecode.m | 33 +++++++++++- jdataencode.m | 143 +++++++++++++++++++++++++++++--------------------- jload.m | 22 ++++++-- jsave.m | 33 ++++++++++-- 4 files changed, 161 insertions(+), 70 deletions(-) diff --git a/jdatadecode.m b/jdatadecode.m index 3e465bc..3d5efee 100644 --- a/jdatadecode.m +++ b/jdatadecode.m @@ -4,7 +4,7 @@ % % Convert all JData object (in the form of a struct array) into an array % (accepts JData objects loaded from either loadjson/loadubjson or -% jsondecode for MATLAB R2018a or later) +% jsondecode for MATLAB R2016b or later) % % This function implements the JData Specification Draft 3 (Jun. 2020) % see http://github.com/fangq/jdata for details @@ -113,6 +113,9 @@ for j=1:len if(isfield(data,N_('_ArrayZipSize_')) && isfield(data,N_('_ArrayZipData_'))) zipmethod='zip'; + if(isstruct(data(j).(N_('_ArrayZipSize_')))) + data(j).(N_('_ArrayZipSize_'))=jdatadecode(data(j).(N_('_ArrayZipSize_')),opt); + end dims=data(j).(N_('_ArrayZipSize_'))(:)'; if(length(dims)==1) dims=[1 dims]; @@ -140,6 +143,9 @@ error('compression method is not supported'); end else + if(isstruct(data(j).(N_('_ArrayData_')))) + data(j).(N_('_ArrayData_'))=jdatadecode(data(j).(N_('_ArrayData_')),opt); + end if(isstruct(data(j).(N_('_ArrayData_'))) && isfield(data(j).(N_('_ArrayData_')),N_('_ArrayType_'))) data(j).(N_('_ArrayData_'))=jdatadecode(data(j).(N_('_ArrayData_')),varargin{:}); end @@ -149,6 +155,9 @@ ndata=cast(data(j).(N_('_ArrayData_')),char(data(j).(N_('_ArrayType_')))); end if(isfield(data,N_('_ArrayZipSize_'))) + if(isstruct(data(j).(N_('_ArrayZipSize_')))) + data(j).(N_('_ArrayZipSize_'))=jdatadecode(data(j).(N_('_ArrayZipSize_')),opt); + end dims=data(j).(N_('_ArrayZipSize_'))(:)'; if(iscell(dims)) dims=cell2mat(dims); @@ -160,6 +169,9 @@ ndata=permute(ndata,ndims(ndata):-1:1); end iscpx=0; + if(isfield(data,N_('_ArrayIsComplex_')) && isstruct(data(j).(N_('_ArrayIsComplex_'))) ) + data(j).(N_('_ArrayIsComplex_'))=jdatadecode(data(j).(N_('_ArrayIsComplex_')),opt); + end if(isfield(data,N_('_ArrayIsComplex_')) && data(j).(N_('_ArrayIsComplex_')) ) iscpx=1; end @@ -170,8 +182,14 @@ iscol=1; end end + if(isfield(data,N_('_ArrayIsSparse_')) && isstruct(data(j).(N_('_ArrayIsSparse_'))) ) + data(j).(N_('_ArrayIsSparse_'))=jdatadecode(data(j).(N_('_ArrayIsSparse_')),opt); + end if(isfield(data,N_('_ArrayIsSparse_')) && data(j).(N_('_ArrayIsSparse_'))) if(isfield(data,N_('_ArraySize_'))) + if(isstruct(data(j).(N_('_ArraySize_')))) + data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt); + end dim=data(j).(N_('_ArraySize_'))(:)'; if(iscell(dim)) dim=cell2mat(dim); @@ -203,6 +221,9 @@ ndata=sparse(ndata(1,:),ndata(2,:),ndata(3,:)); end elseif(isfield(data,N_('_ArrayShape_'))) + if(isstruct(data(j).(N_('_ArrayShape_')))) + data(j).(N_('_ArrayShape_'))=jdatadecode(data(j).(N_('_ArrayShape_')),opt); + end if(iscpx) if(size(ndata,1)==2) dim=size(ndata); @@ -227,7 +248,11 @@ else datasize=size(arraydata); end + if(isstruct(data(j).(N_('_ArraySize_')))) + data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt); + end arraysize=data.(N_('_ArraySize_')); + if(iscell(arraysize)) arraysize=cell2mat(arraysize); end @@ -278,12 +303,16 @@ end ndata=spdiags(reshape(arraydata,min(arraysize),datasize(1)),double(shapeid{2}):-1:-double(shapeid{3}),arraysize(1),arraysize(2)); elseif(strcmpi(shapeid{1},'toeplitz')) - ndata=toeplitz(arraydata(:,1),arraydata(:,2)).'; + arraydata=reshape(arraydata,flipud(datasize(:))'); + ndata=toeplitz(arraydata(1:arraysize(1),2),arraydata(1:arraysize(2),1)); end if(opt.fullarrayshape && issparse(ndata)) ndata=cast(full(ndata),data(j).(N_('_ArrayType_'))); end elseif(isfield(data,N_('_ArraySize_'))) + if(isstruct(data(j).(N_('_ArraySize_')))) + data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt); + end if(iscpx) ndata=complex(ndata(1,:),ndata(2,:)); end diff --git a/jdataencode.m b/jdataencode.m index 386a5ba..7ff00e1 100644 --- a/jdataencode.m +++ b/jdataencode.m @@ -5,8 +5,11 @@ % jdata=jdataencode(data, options) % jdata=jdataencode(data, 'Param1',value1, 'Param2',value2,...) % -% Serialize a MATLAB struct or cell array into a JData-compliant -% structure as defined in the JData spec: http://github.com/fangq/jdata +% Annotate a MATLAB struct or cell array into a JData-compliant data +% structure as defined in the JData spec: http://github.com/fangq/jdata. +% This encoded form servers as an intermediate format that allows unambiguous +% storage, exchange of complex data structures and easy-to-serialize by +% json encoders such as savejson and jsonencode (MATLAB R2016b or newer) % % This function implements the JData Specification Draft 3 (Jun. 2020) % see http://github.com/fangq/jdata for details @@ -17,6 +20,11 @@ % data: a structure (array) or cell (array) to be encoded. % options: (optional) a struct or Param/value pairs for user % specified options (first in [.|.] is the default) +% AnnotateArray: [0|1] - if set to 1, convert all 1D/2D matrices +% to the annotated JData array format to preserve data types; +% N-D (N>2), complex and sparse arrays are encoded using the +% annotated format by default. Please set this option to 1 if +% you intend to use MATLAB's jsonencode to convert to JSON. % Base64: [0|1] if set to 1, _ArrayZipData_ is assumed to % be encoded with base64 format and need to be % decoded first. This is needed for JSON but not @@ -50,6 +58,14 @@ % example: % jd=jdataencode(struct('a',rand(5)+1i*rand(5),'b',[],'c',sparse(5,5))) % +% encodedmat=jdataencode(single(magic(5)),'annotatearray',1,'prefix','x') +% jdatadecode(jsondecode(jsonencode(encodedmat))) % serialize by jsonencode +% jdatadecode(loadjson(savejson('',encodedmat))) % serialize by savejson +% +% encodedtoeplitz=jdataencode(uint8(toeplitz([1,2,3,4],[1,5,6])),'usearrayshape',1,'prefix','x') +% jdatadecode(jsondecode(jsonencode(encodedtoeplitz))) % serialize by jsonencode +% jdatadecode(loadjson(savejson('',encodedtoeplitz))) % serialize by savejson +% % license: % BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details % @@ -77,6 +93,7 @@ opt.usearrayzipsize=jsonopt('UseArrayZipSize',1,opt); opt.messagepack=jsonopt('MessagePack',0,opt); opt.usearrayshape=jsonopt('UseArrayShape',0,opt) && exist('bandwidth'); +opt.annotatearray=jsonopt('AnnotateArray',0,opt); jdata=obj2jd(data,opt); @@ -166,14 +183,9 @@ zipmethod=varargin{1}.compression; minsize=varargin{1}.compressarraysize; -% no encoding for char arrays or non-sparse real vectors -if(isempty(item) || isa(item,'string') || ischar(item) || varargin{1}.nestarray || ... - ((isvector(item) || (ndims(item)==2 && ~varargin{1}.usearrayshape)) ... - && isreal(item) && ~issparse(item))) - newitem=item; - return; % 2d numerical (real/complex/sparse) arrays with _ArrayShape_ encoding enabled -elseif(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item)) +if(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item)) + encoded=1; if(~isreal(item)) newitem.(N('_ArrayIsComplex_'))=true; end @@ -214,11 +226,12 @@ newitem.(N('_ArrayData_'))(1,1:size(item,2))=item(1,:); newitem.(N('_ArrayData_'))(2,1:size(item,1))=item(:,1).'; else % full matrix - newitem=item; + newitem=rmfield(newitem,N('_ArrayZipSize_')); + encoded=0; end % serialize complex data at last - if(isstruct(newitem) && ~isreal(newitem.(N('_ArrayData_')))) + if(encoded && isstruct(newitem) && ~isreal(newitem.(N('_ArrayData_')))) item=squeeze(zeros([2, size(newitem.(N('_ArrayData_')))])); item(1,:)=real(newitem.(N('_ArrayData_'))(:)); item(2,:)=imag(newitem.(N('_ArrayData_'))(:)); @@ -228,64 +241,74 @@ % wrap _ArrayData_ into a single row vector, and store preprocessed % size to _ArrayZipSize_ (force varargin{1}.usearrayzipsize=true) - if(isstruct(newitem) && ~isvector(newitem.(N('_ArrayData_')))) - item=newitem.(N('_ArrayData_')); - item=permute(item,ndims(item):-1:1); - newitem.(N('_ArrayData_'))=item(:).'; - else - newitem=rmfield(newitem,N('_ArrayZipSize_')); - end - - newitem.(N('_ArrayData_'))=full(newitem.(N('_ArrayData_'))); -else - if(isa(item,'logical')) - item=uint8(item); + if(encoded) + if(isstruct(newitem) && ~isvector(newitem.(N('_ArrayData_')))) + item=newitem.(N('_ArrayData_')); + item=permute(item,ndims(item):-1:1); + newitem.(N('_ArrayData_'))=item(:).'; + else + newitem=rmfield(newitem,N('_ArrayZipSize_')); + end + newitem.(N('_ArrayData_'))=full(newitem.(N('_ArrayData_'))); + return end +end - if(isreal(item)) - if(issparse(item)) - fulldata=full(item(find(item))); - newitem.(N('_ArrayIsSparse_'))=true; - newitem.(N('_ArrayZipSize_'))=[2+(~isvector(item)),length(fulldata)]; - if(isvector(item)) - newitem.(N('_ArrayData_'))=[find(item(:))', fulldata(:)']; - else - [ix,iy]=find(item); - newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)', fulldata(:)']; - end +% no encoding for char arrays or non-sparse real vectors +if(isempty(item) || isa(item,'string') || ischar(item) || varargin{1}.nestarray || ... + ((isvector(item) || ndims(item)==2) && isreal(item) && ~issparse(item) && ... + ~varargin{1}.annotatearray)) + newitem=item; + return; +end + +if(isa(item,'logical')) + item=uint8(item); +end + +if(isreal(item)) + if(issparse(item)) + fulldata=full(item(item~=0)); + newitem.(N('_ArrayIsSparse_'))=true; + newitem.(N('_ArrayZipSize_'))=[2+(~isvector(item)),length(fulldata)]; + if(isvector(item)) + newitem.(N('_ArrayData_'))=[find(item(:))', fulldata(:)']; else - if(varargin{1}.formatversion>1.9) - item=permute(item,ndims(item):-1:1); - end - newitem.(N('_ArrayData_'))=item(:)'; + [ix,iy]=find(item); + newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)', fulldata(:)']; end else - newitem.(N('_ArrayIsComplex_'))=true; - if(issparse(item)) - fulldata=full(item(find(item))); - newitem.(N('_ArrayIsSparse_'))=true; - newitem.(N('_ArrayZipSize_'))=[3+(~isvector(item)),length(fulldata)]; - if(isvector(item)) - newitem.(N('_ArrayData_'))=[find(item(:))', real(fulldata(:))', imag(fulldata(:))']; - else - [ix,iy]=find(item); - newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)' , real(fulldata(:))', imag(fulldata(:))']; - end + if(varargin{1}.formatversion>1.9) + item=permute(item,ndims(item):-1:1); + end + newitem.(N('_ArrayData_'))=item(:)'; + end +else + newitem.(N('_ArrayIsComplex_'))=true; + if(issparse(item)) + fulldata=full(item(item~=0)); + newitem.(N('_ArrayIsSparse_'))=true; + newitem.(N('_ArrayZipSize_'))=[3+(~isvector(item)),length(fulldata)]; + if(isvector(item)) + newitem.(N('_ArrayData_'))=[find(item(:))', real(fulldata(:))', imag(fulldata(:))']; else - if(varargin{1}.formatversion>1.9) - item=permute(item,ndims(item):-1:1); - end - newitem.(N('_ArrayZipSize_'))=[2,numel(item)]; - newitem.(N('_ArrayData_'))=[real(item(:))', imag(item(:))']; + [ix,iy]=find(item); + newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)' , real(fulldata(:))', imag(fulldata(:))']; + end + else + if(varargin{1}.formatversion>1.9) + item=permute(item,ndims(item):-1:1); end + newitem.(N('_ArrayZipSize_'))=[2,numel(item)]; + newitem.(N('_ArrayData_'))=[real(item(:))', imag(item(:))']; end +end - if(varargin{1}.usearrayzipsize==0 && isfield(newitem,N('_ArrayZipSize_'))) - data=newitem.(N('_ArrayData_')); - data=reshape(data,fliplr(newitem.(N('_ArrayZipSize_')))); - newitem.(N('_ArrayData_'))=permute(data,ndims(data):-1:1); - newitem=rmfield(newitem,N('_ArrayZipSize_')); - end +if(varargin{1}.usearrayzipsize==0 && isfield(newitem,N('_ArrayZipSize_'))) + data=newitem.(N('_ArrayData_')); + data=reshape(data,fliplr(newitem.(N('_ArrayZipSize_')))); + newitem.(N('_ArrayData_'))=permute(data,ndims(data):-1:1); + newitem=rmfield(newitem,N('_ArrayZipSize_')); end if(~isempty(zipmethod) && numel(item)>minsize) diff --git a/jload.m b/jload.m index e58f580..783a7fb 100644 --- a/jload.m +++ b/jload.m @@ -23,8 +23,11 @@ % ws ['base'|'wsname']: the name of the workspace in which the % variables are to be saved % vars [{'var1','var2',...}]: list of variables to be saved -% Header [0|1]: if set to 1, return the metadata of the variables +% header [0|1]: if set to 1, return the metadata of the variables % stored in the file +% matlab [0|1] if set to 1, use matlab's built-in jsondecode to +% parse the json file and then decode the output by +% jdatadecode; input file must have a suffix of .jdt % % all options for loadubjson/loadjson (depends on file suffix) % can be used to adjust the parsing options @@ -61,7 +64,16 @@ loadfun=@loadmsgpack; end -header=loadfun(filename,'ObjectID',1, varargin{:}); +if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin')) + jsonstr=fileread(filename); + pos=regexp(jsonstr,'}\n\n\n{"WorkspaceData":','once'); + if(isempty(pos)) + error('the json file is not generated using matlab''s jsonencode'); + end + header=jsondecode(jsonstr(1:pos+1)); +else + header=loadfun(filename,'ObjectID',1, varargin{:}); +end if(jsonopt('Header',0,opt)) varargout{1}=header; @@ -78,7 +90,11 @@ error('specified variable is not found'); end -body=loadfun(filename,'ObjectID',2, varargin{:}); +if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin')) + body=jdatadecode(jsondecode(jsonstr(pos+4:end))); +else + body=jsondecode(filename,'ObjectID',2, varargin{:}); +end for i=1:length(varlist) assignin(ws, varlist{i}, body.WorkspaceData.(varlist{i})); diff --git a/jsave.m b/jsave.m index e2b9738..b360c49 100644 --- a/jsave.m +++ b/jsave.m @@ -23,9 +23,12 @@ % ws ['base'|'wsname']: the name of the workspace in which the % variables are to be saved % vars [{'var1','var2',...}]: cell array of variable names to be saved +% matlab [0|1] if set to 1, use matlab's built-in jsonencode to +% store encoded data to a json file; output file +% must have a suffix of .jdt % % all options for saveubjson/savejson (depends on file suffix) -% can be used to adjust the output +% can be used to adjust the output unless "'matlab',1" is used % % output: % varlist: a list of variables loaded @@ -57,7 +60,6 @@ error('specified variable is not found'); end -metadata=struct; header=struct; body=struct; @@ -100,6 +102,27 @@ varargout{1}=header; end -savefun('WorkspaceHeader',header,'filename',filename,varargin{:}); -savefun('WorkspaceData',body,'filename',filename,'append',1,... - 'compression','zlib','keeptype',1,'array2struct',1,varargin{:}); +if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin')) + if(isempty(regexp(filename,'\.[jJ][sS][oO][nN]$', 'once'))) + filename=regexprep(filename,'\.[a-zA-Z]+$','.jdt'); + end + output.WorkspaceHeader=jdataencode(header,'prefix','x','base64',1,varargin{:}); + headerjson=jsonencode(output); + clear output; + + output.WorkspaceData=jdataencode(body,'AnnotateArray',1,'base64',1,... + 'Compression','zlib','UseArrayZipSize',1,'MapAsStruct',1,... + 'prefix','x',varargin{:}); + bodyjson=jsonencode(output); + clear output; + + fid=fopen(filename,jsonopt('writemode','w',opt)); + fwrite(fid,headerjson); + fwrite(fid,sprintf('\n\n\n')); + fwrite(fid,bodyjson); + fclose(fid); +else + savefun('WorkspaceHeader',header,'filename',filename,varargin{:}); + savefun('WorkspaceData',body,'filename',filename,'append',1,... + 'compression','zlib','keeptype',1,'array2struct',1,varargin{:}); +end