Skip to content

Commit

Permalink
fixed bad bloom filter for JSON root object; added regression to test…
Browse files Browse the repository at this point in the history 
… 318
  • Loading branch information
@tomatolog
tomatolog committed Apr 23, 2023
1 parent c95ddde commit 6dd3964
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 4 deletions.
9 changes: 6 additions & 3 deletions src/sphinxjson.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -887,9 +887,12 @@ bool sphJsonParse ( CSphVector<BYTE> &dData, char * sData, bool bAutoconv, bool
DWORD sphJsonKeyMask ( const char * sKey, int iLen )
{
DWORD uCrc = sphCRC32 ( sKey, iLen );
return
( 1UL<<( uCrc & 31 ) ) +
( 1UL<<( ( uCrc>>8 ) & 31 ) );

DWORD uBloom1 = ( 1UL<<( uCrc & 31 ) );
DWORD uBloom2 = ( 1UL<<( ( uCrc>>8 ) & 31 ) );

// DWORD overflow protection as 0 at bloom mask means not the ROOT node
return ( ( uBloom1 + uBloom2 ) ? ( uBloom1 + uBloom2 ) : ( uBloom1 | uBloom2 ) );
}

static inline DWORD KeyMask ( const char * s )
Expand Down
2 changes: 1 addition & 1 deletion test/test_318/model.bin
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
a:1:{i:0;a:5:{i:0;a:3:{s:8:"sphinxql";s:48:"select * from test where json_data.price is null";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 1";}i:1;a:4:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:9:"json_data";s:14:"{"price":null}";s:5:"title";s:6:"test 4";}i:2;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 5";}i:3;a:4:{s:2:"id";s:1:"6";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"[]";s:5:"title";s:6:"test 6";}}}i:1;a:3:{s:8:"sphinxql";s:52:"select * from test where json_data.price is not null";s:10:"total_rows";i:2;s:4:"rows";a:2:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":20}";s:5:"title";s:6:"test 2";}i:1;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":30}";s:5:"title";s:6:"test 3";}}}i:2;a:3:{s:8:"sphinxql";s:45:"select * from test where json_data.price < 25";s:10:"total_rows";i:2;s:4:"rows";a:2:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":20}";s:5:"title";s:6:"test 2";}i:1;a:4:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:9:"json_data";s:14:"{"price":null}";s:5:"title";s:6:"test 4";}}}i:3;a:3:{s:8:"sphinxql";s:46:"select * from test where json_data is not null";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 1";}i:1;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":20}";s:5:"title";s:6:"test 2";}i:2;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":30}";s:5:"title";s:6:"test 3";}i:3;a:4:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:9:"json_data";s:14:"{"price":null}";s:5:"title";s:6:"test 4";}i:4;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 5";}i:5;a:4:{s:2:"id";s:1:"6";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"[]";s:5:"title";s:6:"test 6";}}}i:4;a:3:{s:8:"sphinxql";s:57:"select id i, json_data as j from test where j is not null";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:1:"i";s:1:"2";s:1:"j";s:12:"{"price":20}";}i:1;a:2:{s:1:"i";s:1:"3";s:1:"j";s:12:"{"price":30}";}i:2;a:2:{s:1:"i";s:1:"4";s:1:"j";s:14:"{"price":null}";}i:3;a:2:{s:1:"i";s:1:"6";s:1:"j";s:2:"[]";}}}}}
a:1:{i:0;a:8:{i:0;a:3:{s:8:"sphinxql";s:48:"select * from test where json_data.price is null";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 1";}i:1;a:4:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:9:"json_data";s:14:"{"price":null}";s:5:"title";s:6:"test 4";}i:2;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 5";}i:3;a:4:{s:2:"id";s:1:"6";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"[]";s:5:"title";s:6:"test 6";}}}i:1;a:3:{s:8:"sphinxql";s:52:"select * from test where json_data.price is not null";s:10:"total_rows";i:2;s:4:"rows";a:2:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":20}";s:5:"title";s:6:"test 2";}i:1;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":30}";s:5:"title";s:6:"test 3";}}}i:2;a:3:{s:8:"sphinxql";s:45:"select * from test where json_data.price < 25";s:10:"total_rows";i:2;s:4:"rows";a:2:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":20}";s:5:"title";s:6:"test 2";}i:1;a:4:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:9:"json_data";s:14:"{"price":null}";s:5:"title";s:6:"test 4";}}}i:3;a:3:{s:8:"sphinxql";s:46:"select * from test where json_data is not null";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 1";}i:1;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":20}";s:5:"title";s:6:"test 2";}i:2;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:1:"1";s:9:"json_data";s:12:"{"price":30}";s:5:"title";s:6:"test 3";}i:3;a:4:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:9:"json_data";s:14:"{"price":null}";s:5:"title";s:6:"test 4";}i:4;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"{}";s:5:"title";s:6:"test 5";}i:5;a:4:{s:2:"id";s:1:"6";s:3:"gid";s:1:"1";s:9:"json_data";s:2:"[]";s:5:"title";s:6:"test 6";}}}i:4;a:3:{s:8:"sphinxql";s:57:"select id i, json_data as j from test where j is not null";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:1:"i";s:1:"2";s:1:"j";s:12:"{"price":20}";}i:1;a:2:{s:1:"i";s:1:"3";s:1:"j";s:12:"{"price":30}";}i:2;a:2:{s:1:"i";s:1:"4";s:1:"j";s:14:"{"price":null}";}i:3;a:2:{s:1:"i";s:1:"6";s:1:"j";s:2:"[]";}}}i:5;a:3:{s:8:"sphinxql";s:19:"select * from test1";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:5:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:2:"j1";s:10:"{"1234":0}";s:2:"j2";s:10:"{"1416":0}";s:5:"title";s:6:"test 2";}i:1;a:5:{s:2:"id";s:1:"3";s:3:"gid";s:1:"1";s:2:"j1";s:7:"{"1":1}";s:2:"j2";s:17:"{"1416":{"1":12}}";s:5:"title";s:6:"test 3";}i:2;a:5:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:2:"j1";s:8:"{"12":1}";s:2:"j2";s:9:"{"141":1}";s:5:"title";s:6:"test 4";}}}i:6;a:3:{s:8:"sphinxql";s:45:"select * from test1 where j2.1416 is not null";s:10:"total_rows";i:2;s:4:"rows";a:2:{i:0;a:5:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:2:"j1";s:10:"{"1234":0}";s:2:"j2";s:10:"{"1416":0}";s:5:"title";s:6:"test 2";}i:1;a:5:{s:2:"id";s:1:"3";s:3:"gid";s:1:"1";s:2:"j1";s:7:"{"1":1}";s:2:"j2";s:17:"{"1416":{"1":12}}";s:5:"title";s:6:"test 3";}}}i:7;a:3:{s:8:"sphinxql";s:39:"select *, j2.1416, j2.1416.1 from test1";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:7:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:2:"j1";s:10:"{"1234":0}";s:2:"j2";s:10:"{"1416":0}";s:5:"title";s:6:"test 2";s:7:"j2.1416";s:1:"0";s:9:"j2.1416.1";N;}i:1;a:7:{s:2:"id";s:1:"3";s:3:"gid";s:1:"1";s:2:"j1";s:7:"{"1":1}";s:2:"j2";s:17:"{"1416":{"1":12}}";s:5:"title";s:6:"test 3";s:7:"j2.1416";s:8:"{"1":12}";s:9:"j2.1416.1";s:2:"12";}i:2;a:7:{s:2:"id";s:1:"4";s:3:"gid";s:1:"1";s:2:"j1";s:8:"{"12":1}";s:2:"j2";s:9:"{"141":1}";s:5:"title";s:6:"test 4";s:7:"j2.1416";N;s:9:"j2.1416.1";N;}}}}}
39 changes: 39 additions & 0 deletions test/test_318/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,22 @@ index test
path = <data_path/>/test
}


source test1
{
type = mysql
<sql_settings/>
sql_query = select * from test_table1
sql_attr_uint = gid
sql_attr_json = j1
sql_attr_json = j2
}

index test1
{
source = test1
path = <data_path/>/test1
}
</config>

<db_create>
Expand All @@ -63,11 +79,34 @@ INSERT INTO `test_table` VALUES
(6,1,'test 6','[]')
</db_insert>

<db_create>
CREATE TABLE `test_table1` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`gid` int(11) NOT NULL,
`title` varchar(255) NOT NULL,
`j1` varchar(255) NOT NULL,
`j2` varchar(255) NOT NULL,
PRIMARY KEY (`id`)
);
</db_create>
<db_drop>DROP TABLE IF EXISTS `test_table1`;</db_drop>
<db_insert>
INSERT INTO `test_table1` VALUES
(2,1,'test 2','{"1234":0}', '{ "1416" : 0 }'),
(3,1,'test 3','{ "1" : 1 }', '{ "1416" : { "1" : 12 } }'),
(4,1,'test 4','{"12":1}', '{ "141" : 1 }')
</db_insert>

<queries><sphinxql>
select * from test where json_data.price is null;
select * from test where json_data.price is not null;
select * from test where json_data.price &lt; 25;
select * from test where json_data is not null;
select id i, json_data as j from test where j is not null;

<!-- regression crash on access of json atribute with empty bloom -->
select * from test1;
select * from test1 where j2.1416 is not null;
select *, j2.1416, j2.1416.1 from test1;
</sphinxql></queries>
</test>

0 comments on commit 6dd3964

Please sign in to comment.