diff --git a/README.rst b/README.rst index 9678bc0..4e91946 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,7 @@ * URL: https://neurojson.org/jsonlab * JData Specification Version: V1 Draft-3 (https://neurojson.org/jdata/draft3) * Binary JData Specification Version: V1 Draft-2 (https://neurojson.org/bjdata/draft2) +* JSON-Mmap Specification Version: V1 Draft-1 (https://neurojson.org/jsonmmap/draft1) * Compatibility: MATLAB R2008 or newer, GNU Octave 3.8 or newer * Acknowledgement: This project is supported by US National Institute of Health (NIH) grant `U24-NS124027 `_ @@ -44,7 +45,7 @@ JSONLab v3.0 - containing a number of key feature enhancement and bug fixes. The new features include 1. exporting JSON Memory-Map for rapid disk-map like JSON/binary JSON reading - and writing, + and writing, implementing `JSON-Mmap Spec v1 Draft 1 `_ 2. supporting JSONPath query to MATLAB data and JSON/binary JSON file and streams, 3. (**breaking**) upgrading the supported BJData spec to `V1 Draft 2 `_ where the default numerical data byte order changed from Big-Endian to **Little-Endian**, @@ -180,7 +181,7 @@ flexibility and generality similar to other more sophisticated formats such as `HDF5 `_, but are significantly simpler with a much greater software ecosystem. -Towards this goal, we have developed the JData Specification (http://github.com/fangq/jdata) +Towards this goal, we have developed the JData Specification (http://github.com/NeuroJSON/jdata) to standardize serializations of complex scientific data structures, such as N-D arrays, sparse/complex-valued arrays, trees, maps, tables and graphs using JSON/binary JSON constructs. The text and binary formatted JData files are @@ -325,7 +326,7 @@ for reading and writing below files types: ``.bnii`` (binary JNIfTI file), ``.bnirs`` (binary JSNIRF file), ``.jamm`` (MATLAB session file) - UBJSON based files: ``.ubj`` - MessagePack based files: ``.msgpack`` -- HDF5 based files: ``.h5``, ``.hdf5``, ``.snirf`` (SNIRF fNIRS data files) - require `EasyH5 toolbox `_ +- HDF5 based files: ``.h5``, ``.hdf5``, ``.snirf`` (SNIRF fNIRS data files) - require `EasyH5 toolbox `_ In the below section, we provide a few examples on how to us each of the @@ -508,8 +509,8 @@ In short, to conveniently read/write data files created by JSONLab into Python, whether they are JSON based or binary JData/UBJSON based, one just need to download the below two light-weight python modules: -* **jdata**: PyPi: https://pypi.org/project/jdata/ ; Github: https://github.com/fangq/pyjdata -* **bjdata** PyPi: https://pypi.org/project/bjdata/ ; Github: https://github.com/fangq/pybj +* **jdata**: PyPi: https://pypi.org/project/jdata/ ; Github: https://github.com/NeuroJSON/pyjdata +* **bjdata** PyPi: https://pypi.org/project/bjdata/ ; Github: https://github.com/NeuroJSON/pybj To install these modules on Python 2.x, please first check if your system has ``pip`` and ``numpy``, if not, please install it by running (using Ubuntu/Debian as example) diff --git a/README.txt b/README.txt index b945c76..eceea9f 100644 --- a/README.txt +++ b/README.txt @@ -9,6 +9,7 @@ * URL: https://neurojson.org/jsonlab * JData Specification Version: V1 Draft-3 (http://github.com/NeuroJSON/jdata) * Binary JData Specification Version: V1 Draft-2 (http://github.com/NeuroJSON/bjdata) +* JSON-Mmap Specification Version: V1 Draft-1 (https://neurojson.org/jsonmmap/draft1) * Compatibility: MATLAB R2008 or newer, GNU Octave 3.8 or newer * Acknowledgement: This project is supported by US National Institute of Health (NIH) \ grant [https://reporter.nih.gov/project-details/10308329 U24-NS124027] @@ -45,7 +46,7 @@ JSONLab v2.9.8 - code named "Micronus - beta" - is the beta-release of the next JSONLab v3.0 - containing a number of key feature enhancement and bug fixes. The major new features include -# exporting JSON Memory-Map for rapid disk-map like JSON/binary JSON reading and writing, +# exporting JSON Memory-Map for rapid disk-map like JSON/binary JSON reading and writing, implementing [https://github.com/NeuroJSON/jsonmmap JSON-Mmap Spec v1 Draft 1] # supporting JSONPath query to MATLAB data and JSON/binary JSON file and streams, # ('''breaking''') upgrading the supported BJData spec to [https://neurojson.org/bjdata/draft2 V1 Draft 2] \ where the default numerical data byte order changed from Big-Endian to '''Little-Endian''' diff --git a/decodevarname.m b/decodevarname.m index f1ab141..5a24d66 100644 --- a/decodevarname.m +++ b/decodevarname.m @@ -25,9 +25,9 @@ % decodevarname('a_') % returns a_ as it is a valid variable name % decodevarname('x0xE58F98__0xE9878F_') % returns '变量' % -% this file is part of EasyH5 Toolbox: https://github.com/fangq/easyh5 +% this file is part of EasyH5 Toolbox: https://github.com/NeuroJSON/easyh5 % -% License: GPLv3 or 3-clause BSD license, see https://github.com/fangq/easyh5 for details +% License: GPLv3 or 3-clause BSD license, see https://github.com/NeuroJSON/easyh5 for details % newname=name; diff --git a/encodevarname.m b/encodevarname.m index ef5c5ff..7719e67 100644 --- a/encodevarname.m +++ b/encodevarname.m @@ -32,9 +32,9 @@ % encodevarname('a_') % returns a_ as it is a valid variable name % encodevarname('变量') % returns 'x0xE58F98__0xE9878F_' % -% this file is part of EasyH5 Toolbox: https://github.com/fangq/easyh5 +% this file is part of EasyH5 Toolbox: https://github.com/NeuroJSON/easyh5 % -% License: GPLv3 or 3-clause BSD license, see https://github.com/fangq/easyh5 for details +% License: GPLv3 or 3-clause BSD license, see https://github.com/NeuroJSON/easyh5 for details % if(~isvarname(str(1))) diff --git a/filterjsonmmap.m b/filterjsonmmap.m index 1070e10..d774a19 100644 --- a/filterjsonmmap.m +++ b/filterjsonmmap.m @@ -23,7 +23,7 @@ % str='{"arr":[[1,2],"a",{"c":2}],"obj":{"k":"test"}}'; % [dat, mmap]=loadjson(str); % savejson('',mmap) -% newmmap=filterjsonmmap(mmap,{'arr.[1]', 'obj.k'}); +% newmmap=filterjsonmmap(mmap,{'arr[1]', 'obj.k'}); % savejson('',newmmap) % % license: diff --git a/getfromjsonpath.m b/getfromjsonpath.m index e22b42e..924e0d4 100644 --- a/getfromjsonpath.m +++ b/getfromjsonpath.m @@ -14,7 +14,7 @@ % obj: if the specified element exist, obj returns the result % % example: -% getfromjsonpath(struct('a',[1,2,3]), '$.a.[1]') % returns 2 +% getfromjsonpath(struct('a',[1,2,3]), '$.a[1]') % returns 2 % % license: % BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details @@ -23,6 +23,7 @@ % obj=root; +jsonpath=regexprep(jsonpath,'([^.])(\[\d+\])','$1.$2'); [pat,paths]=regexp(jsonpath,'\.*([^\s\.]+)\.*','match','tokens'); if(~isempty(pat) && ~isempty(paths)) for i=1:length(paths) diff --git a/jdatadecode.m b/jdatadecode.m index a7be4aa..4604912 100644 --- a/jdatadecode.m +++ b/jdatadecode.m @@ -7,7 +7,7 @@ % jsondecode for MATLAB R2016b or later) % % This function implements the JData Specification Draft 3 (Jun. 2020) -% see http://github.com/fangq/jdata for details +% see http://github.com/NeuroJSON/jdata for details % % authors:Qianqian Fang (q.fang neu.edu) % diff --git a/jdataencode.m b/jdataencode.m index 7ff00e1..ccee220 100644 --- a/jdataencode.m +++ b/jdataencode.m @@ -6,13 +6,13 @@ % jdata=jdataencode(data, 'Param1',value1, 'Param2',value2,...) % % Annotate a MATLAB struct or cell array into a JData-compliant data -% structure as defined in the JData spec: http://github.com/fangq/jdata. +% structure as defined in the JData spec: http://github.com/NeuroJSON/jdata. % This encoded form servers as an intermediate format that allows unambiguous % storage, exchange of complex data structures and easy-to-serialize by % json encoders such as savejson and jsonencode (MATLAB R2016b or newer) % % This function implements the JData Specification Draft 3 (Jun. 2020) -% see http://github.com/fangq/jdata for details +% see http://github.com/NeuroJSON/jdata for details % % author: Qianqian Fang (q.fang neu.edu) % diff --git a/jsonget.m b/jsonget.m index 14b60d9..64d41fd 100644 --- a/jsonget.m +++ b/jsonget.m @@ -24,8 +24,8 @@ % str='[[1,2],"a",{"c":2}]{"k":"test"}'; % [dat, mmap]=loadjson(str); % savejson('',dat,'filename','mydata.json','compact',1); -% json=jsonget(str,mmap,'$.[0]','$.[2].c') -% json=jsonget('mydata.json',mmap,'$.[0]','$.[2].c') +% json=jsonget(str,mmap,'$[0]','$[2].c') +% json=jsonget('mydata.json',mmap,'$[0]','$[2].c') % % license: % BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details diff --git a/jsonset.m b/jsonset.m index 27a5fc9..4ca6d15 100644 --- a/jsonset.m +++ b/jsonset.m @@ -31,11 +31,11 @@ % % display mmap entries % savejson('',mmap) % % replace value using mmap -% json=jsonset(str,mmap,'$.arr.[2].c','5') +% json=jsonset(str,mmap,'$.arr[2].c','5') % % save same json string to file (must set savebinary 1) % savejson('',d,'filename','file.json','compact',1,'savebinary',1); % % fast write to file -% json=jsonset('file.json',mmap,'$.arr.[2].c','5') +% json=jsonset('file.json',mmap,'$.arr[2].c','5') % % license: % BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details diff --git a/loadbj.m b/loadbj.m index 8c5f70e..4dfaa99 100644 --- a/loadbj.m +++ b/loadbj.m @@ -78,6 +78,10 @@ % of the stream, and length is the JSON object string length. % For more details, please see the help section of loadjson.m % +% The format of the mmap table retruned from this function +% follows the JSON-Mmap Specification Draft 1 [3] defined by the +% NeuroJSON project, see https://neurojson.org/jsonmmap/draft1/ +% % examples: % obj=struct('string','value','array',[1 2 3]); % ubjdata=savebj('obj',obj); @@ -277,7 +281,7 @@ if cc ~= ']' while 1 if(nargout>2) - varargin{1}.jsonpath_=[origpath '.' sprintf('[%d]',length(object))]; + varargin{1}.jsonpath_=[origpath sprintf('[%d]',length(object))]; mmap{end+1}={varargin{1}.jsonpath_, pos}; [val, pos, newmmap] = parse_value(inputstr, pos, [], varargin{:}); mmap{end}{2}=[mmap{end}{2}, pos-mmap{end}{2}]; diff --git a/loadjd.m b/loadjd.m index f58b80f..7009397 100644 --- a/loadjd.m +++ b/loadjd.m @@ -20,7 +20,7 @@ % options: (optional) for JSON/JData files, these are optional 'param',value pairs % supported by loadjson.m; for BJData/UBJSON/MessagePack files, these are % options supported by loadbj.m; for HDF5 files, these are options -% supported by loadh5.m (part of EasyH5 toolbox, http://github.com/fangq/easyh5/) +% supported by loadh5.m (part of EasyH5 toolbox, http://github.com/NeuroJSON/easyh5/) % % output: % data: a structure (array) or cell (array) storing the hierarchical data @@ -53,7 +53,7 @@ [varargout{1:nargout}]=loadmsgpack(filename,varargin{:}); elseif(regexpi(filename,'\.h5$|\.hdf5$|\.snirf$')) if(~exist('loadh5','file')) - error('you must first install EasyH5 from http://github.com/fangq/easyh5/'); + error('you must first install EasyH5 from http://github.com/NeuroJSON/easyh5/'); end [varargout{1:nargout}]=loadh5(filename,varargin{:}); else diff --git a/loadjson.m b/loadjson.m index 3ca2abb..3713e09 100644 --- a/loadjson.m +++ b/loadjson.m @@ -81,15 +81,19 @@ % dat: a cell array, where {...} blocks are converted into cell arrays, % and [...] are converted to arrays % mmap: (optional) a cell array as memory-mapping table in the form of -% {{jsonpath1,[start,length,]}, -% {jsonpath2,[start,length,]}, ...} +% {{jsonpath1,[start,length,]}, +% {jsonpath2,[start,length,]}, ...} % where jsonpath_i is a string in the JSONPath [1,2] format, and % "start" is an integer referring to the offset from the begining % of the stream, and "length" is the JSON object string length. -% An optional 3rd integer "whitespace" may appear to record the -% preceding whitespace length in case expansion of the data +% An optional 3rd integer "whitespace_pre" may appear to record +% the preceding whitespace length in case expansion of the data % record is needed when using the mmap. % +% The format of the mmap table retruned from this function +% follows the JSON-Mmap Specification Draft 1 [3] defined by the +% NeuroJSON project, see https://neurojson.org/jsonmmap/draft1/ +% % Memory-mapping table (mmap) is useful when fast reading/writing % specific data records inside a large JSON file without needing % to load/parse/overwrite the entire file. @@ -101,10 +105,10 @@ % In the mmap jsonpath key, a '$' denotes the root object, a '.' % denotes a child of the preceding element; '.key' points to the % value segment of the child named "key" of the preceding -% object; '.[i]' denotes the (i+1)th member of the preceding +% object; '[i]' denotes the (i+1)th member of the preceding % element, which must be an array. For example, a key % -% $.obj1.obj2.[0].obj3 +% $.obj1.obj2[0].obj3 % % defines the memory-map of the "value" section in the below % hierarchy: @@ -134,6 +138,7 @@ % % [1] https://goessner.net/articles/JsonPath/ % [2] http://jsonpath.herokuapp.com/ +% [3] https://neurojson.org/jsonmmap/draft1/ % % examples: % dat=loadjson('{"obj":{"string":"value","array":[1,2,3]}}') @@ -363,7 +368,7 @@ while 1 varargin{1}.arraydepth_=arraydepth+1; if(nargout>3) - varargin{1}.jsonpath_=[origpath '.' sprintf('[%d]',length(object))]; + varargin{1}.jsonpath_=[origpath sprintf('[%d]',length(object))]; mmap{end+1}={varargin{1}.jsonpath_, [pos, 0, w2]}; [val, pos, index_esc, newmmap] = parse_value(inputstr, pos, esc, index_esc,varargin{:}); mmap{end}{2}(2)=pos-mmap{end}{2}(1); diff --git a/savejd.m b/savejd.m index cd3e05a..c934af1 100644 --- a/savejd.m +++ b/savejd.m @@ -28,7 +28,7 @@ % opt: (optional) for JSON/JData files, these are optional 'param',value pairs % supported by loadjson.m; for BJData/UBJSON/MessagePack files, these are % options supported by loadbj.m; for HDF5 files, these are options -% supported by loadh5.m (part of EasyH5 toolbox, http://github.com/fangq/easyh5/) +% supported by loadh5.m (part of EasyH5 toolbox, http://github.com/NeuroJSON/easyh5/) % % output: % data: a structure (array) or cell (array) storing the hierarchical data @@ -72,7 +72,7 @@ [varargout{1:nargout}]=savemsgpack(varargin{:}); elseif(regexpi(filename,'\.h5$|\.hdf5$|\.snirf$')) if(~exist('saveh5','file')) - error('you must first install EasyH5 from http://github.com/fangq/easyh5/'); + error('you must first install EasyH5 from http://github.com/NeuroJSON/easyh5/'); end [varargout{1:nargout}]=saveh5(varargin{:}); else diff --git a/test/run_jsonlab_test.m b/test/run_jsonlab_test.m index dae16fa..f5bcb1e 100644 --- a/test/run_jsonlab_test.m +++ b/test/run_jsonlab_test.m @@ -310,17 +310,17 @@ function run_jsonlab_test(tests) test_jsonlab('mmap of concatenated json',@savejson,loadjson('[1,2,3][4,5,6]','mmaponly',1),'[["$",[1,7]],["$1",[8,7]]]','compact',1); test_jsonlab('mmap of concatenated json objects',@savejson,loadjson('[1,2,3]{"a":[4,5]}','mmaponly',1),'[["$",[1,7]],["$1",[8,11]],["$1.a",[13,5]]]','compact',1); test_jsonlab('mmap of an array with an object',@savejson,loadjson('[1,2,{"a":3}]','mmaponly',1),... - '[["$",[1,13]],["$.[0]",[2,1]],["$.[1]",[4,1]],["$.[2]",[6,7]],["$.[2].a",[11,1]]]','compact',1); + '[["$",[1,13]],["$[0]",[2,1]],["$[1]",[4,1]],["$[2]",[6,7]],["$[2].a",[11,1]]]','compact',1); test_jsonlab('mmap of an object',@savejson,loadjson('{"a":1,"b":[2,3]}','mmaponly',1),... '[["$",[1,17]],["$.a",[6,1]],["$.b",[12,5]]]','compact',1); test_jsonlab('mmap of object with white-space',@savejson,loadjson('{"a":1 , "b" : [2,3]}','mmaponly',1),... '[["$",[1,23]],["$.a",[6,1]],["$.b",[18,5,2]]]','compact',1); test_jsonlab('mmapinclude option',@savejson,loadjson('[[1,2,3],{"a":[4,5]}]','mmaponly',1,'mmapinclude','.a'),... - '[["$.[1].a",[15,5]]]','compact',1); + '[["$[1].a",[15,5]]]','compact',1); test_jsonlab('mmapexclude option',@savejson,loadjson('[[1,2,3],{"a":[4,5]}]','mmaponly',1,'mmapexclude',{'[0]','[1]','[2]'}),... '[["$",[1,21]]]','compact',1); test_jsonlab('json with indentation',@savejson,loadjson(savejson({[1,2,3],struct('a',[4,5])}),'mmaponly',1,'mmapinclude','.a'),... - '[["$.[1].a",[22,7]]]','compact',1); + '[["$[1].a",[22,7]]]','compact',1); end %% @@ -330,14 +330,14 @@ function run_jsonlab_test(tests) fprintf(sprintf('%s\n',char(ones(1,79)*61))); test_jsonlab('mmap of a 1D numerical array',@savejson,loadbj(savebj([1,2,3]),'mmaponly',1),'[["$",[1,9]]]','compact',1); - test_jsonlab('mmap of a 1D mixed array',@savejson,loadbj(savebj({1,'2',3}),'mmaponly',1),'[["$",[1,8]],["$.[0]",[2,2]],["$.[1]",[4,2]],["$.[2]",[6,2]]]','compact',1); + test_jsonlab('mmap of a 1D mixed array',@savejson,loadbj(savebj({1,'2',3}),'mmaponly',1),'[["$",[1,8]],["$[0]",[2,2]],["$[1]",[4,2]],["$[2]",[6,2]]]','compact',1); test_jsonlab('mmap of a 2D array',@savejson,loadbj(savebj([[1,2,3],[4,5,6]]),'mmaponly',1),'[["$",[1,12]]]','compact',1); test_jsonlab('mmap of an array with an object',@savejson,loadbj(savebj({1,2,struct('a',3)}),'mmaponly',1),... - '[["$",[1,13]],["$.[0]",[2,2]],["$.[1]",[4,2]],["$.[2]",[6,7]],["$.[2].a",[10,2]]]','compact',1); + '[["$",[1,13]],["$[0]",[2,2]],["$[1]",[4,2]],["$[2]",[6,7]],["$[2].a",[10,2]]]','compact',1); test_jsonlab('mmap of an object',@savejson,loadbj(savebj(struct('a',1,'b',[2,3])),'mmaponly',1),... '[["$",[1,18]],["$.a",[5,2]],["$.b",[10,8]]]','compact',1); test_jsonlab('mmapinclude option',@savejson,loadbj(savebj({[1,2,3],struct('a',[4,5])}),'mmaponly',1,'mmapinclude','.a'),... - '[["$.[1].a",[15,8]]]','compact',1); + '[["$[1].a",[15,8]]]','compact',1); test_jsonlab('mmapexclude option',@savejson,loadbj(savebj({[1,2,3],struct('a',[4,5])}),'mmaponly',1,'mmapexclude',{'[0]','[1]','[2]'}),... '[["$",[1,24]]]','compact',1); test_jsonlab('test multiple root objects with N padding',@savejson,loadbj([savebj({[1,2,3],struct('a',[4,5])}) 'NNN' savebj(struct('b',[4,5]))],'mmaponly',1,'mmapinclude','.b'),...