Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updated Filters to support predicate logic #21

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

updated Filters to support predicate logic #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
93649af
updated Filters to support predicate logic
May 3, 2019
4ee3fb6
updated README
May 3, 2019
62b53af
added support for regex filters \n added support for . keyword
May 23, 2019
23c2f64
removed excess print
May 24, 2019
2492593
updated README
May 24, 2019
8cacb50
updated unittests
Jun 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ Extensions
+==============+==============================================+
| len | - $.objects.`len` |
+--------------+----------------------------------------------+
| keys | - $.dict_field.`keys`(returns a list of keys)|
+--------------+----------------------------------------------+
| sub | - $.field.`sub(/foo\\\\+(.*)/, \\\\1)` |
+--------------+----------------------------------------------+
| split | - $.field.`split(+, 2, -1)` |
Expand All @@ -203,7 +205,10 @@ Extensions
+--------------+----------------------------------------------+
| filter | - $.objects[?(@some_field > 5)] |
| | - $.objects[?some_field = "foobar")] |
| | - $.objects[?some_field > 5 & other < 2)] |
| | - $.objects[?some_field > 5 & other < 2)] and|
| | - $.objects[?some_field>5 |some_field<2)] or |
| | - $.objects[?(!(field>5 | field<2))] not|
| | - $.objects[[email protected] ~= "a.+a"] regex|
+--------------+----------------------------------------------+
| arithmetic | - $.foo + "_" + $.bar |
| (-+*/) | - $.foo * 12 |
Expand Down
47 changes: 40 additions & 7 deletions jsonpath_ng/ext/filter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,15 @@

import operator
from six import moves
import re

from .. import JSONPath, DatumInContext, Index

def contains(a,b):
if re.search(b,a):
return True
return False


OPERATOR_MAP = {
'!=': operator.ne,
Expand All @@ -25,8 +31,37 @@
'<': operator.lt,
'>=': operator.ge,
'>': operator.gt,
'~=': contains
}

def eval_exp(expressions,val):
for expression in expressions:
if type(expression)==tuple and expression[0]=='|':
val1=eval_exp(expression[1],val)
val2=eval_exp(expression[2],val)
if (val1 or val2):
return True
else:
return False
if type(expression)==tuple and expression[0]=='&':
val1=eval_exp(expression[1],val)
val2=eval_exp(expression[2],val)
if (val1 and val2):
return True
else:
return False
if type(expression)==tuple and expression[0]=='!':
val1=eval_exp(expression[1],val)
if (val1):
return False
else:
return True
else:
if(len([expression])==len(list(filter(lambda x: x.find(val),[expression])))):
return True
else:
return False


class Filter(JSONPath):
"""The JSONQuery filter"""
Expand All @@ -41,12 +76,11 @@ def find(self, datum):
datum = DatumInContext.wrap(datum)
if not isinstance(datum.value, list):
return []

return [DatumInContext(datum.value[i], path=Index(i), context=datum)
for i in moves.range(0, len(datum.value))
if (len(self.expressions) ==
len(list(filter(lambda x: x.find(datum.value[i]),
self.expressions))))]
res=[]
for i in moves.range(0,len(datum.value)):
if eval_exp(self.expressions,datum.value[i]):
res.append(DatumInContext(datum.value[i], path=Index(i), context=datum))
return(res)

def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self.expressions)
Expand All @@ -65,7 +99,6 @@ def __init__(self, target, op, value):

def find(self, datum):
datum = self.target.find(DatumInContext.wrap(datum))

if not datum:
return []
if self.op is None:
Expand Down
26 changes: 26 additions & 0 deletions jsonpath_ng/ext/iterable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,29 @@ def __str__(self):

def __repr__(self):
return 'Len()'

class Keys(JSONPath):
"""The JSONPath referring to the keys of the current object.

Concrete syntax is '`keys`'.
"""

def find(self, datum):
datum = DatumInContext.wrap(datum)
try:
value = datum.value.keys()
except Exception as e:
return []
else:
return [DatumInContext(value[i],
context=None,
path=Keys()) for i in range (0, len(datum.value))]

def __eq__(self, other):
return isinstance(other, Keys)

def __str__(self):
return '`keys`'

def __repr__(self):
return 'Keys()'
21 changes: 14 additions & 7 deletions jsonpath_ng/ext/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from .. import lexer
from .. import parser
from .. import Fields, This, Child

from . import arithmetic as _arithmetic
from . import filter as _filter
from . import iterable as _iterable
Expand All @@ -23,12 +22,11 @@

class ExtendedJsonPathLexer(lexer.JsonPathLexer):
"""Custom LALR-lexer for JsonPath"""
literals = lexer.JsonPathLexer.literals + ['?', '@', '+', '*', '/', '-']
literals = lexer.JsonPathLexer.literals + ['?', '@', '+', '*', '/', '-', '!','~']
tokens = (['BOOL'] +
parser.JsonPathLexer.tokens +
['FILTER_OP', 'SORT_DIRECTION', 'FLOAT'])

t_FILTER_OP = r'==?|<=|>=|!=|<|>'
t_FILTER_OP = r'==?|<=|>=|!=|<|>|~='

def t_BOOL(self, t):
r'true|false'
Expand Down Expand Up @@ -94,6 +92,8 @@ def p_jsonpath_named_operator(self, p):
"jsonpath : NAMED_OPERATOR"
if p[1] == 'len':
p[0] = _iterable.Len()
elif p[1] == 'keys':
p[0] = _iterable.Keys()
elif p[1] == 'sorted':
p[0] = _iterable.SortedThis()
elif p[1].startswith("split("):
Expand All @@ -119,11 +119,18 @@ def p_expression(self, p):
def p_expressions_expression(self, p):
"expressions : expression"
p[0] = [p[1]]


def p_expressions_not(self, p):
"expressions : '!' expressions"
p[0]=[('!',p[2])]

def p_expressions_and(self, p):
"expressions : expressions '&' expressions"
# TODO(sileht): implements '|'
p[0] = p[1] + p[3]
p[0] = [('&',p[1],p[3])]

def p_expressions_or(self, p):
"expressions : expressions '|' expressions"
p[0] = [('|',p[1],p[3])]

def p_expressions_parens(self, p):
"expressions : '(' expressions ')'"
Expand Down
152 changes: 151 additions & 1 deletion tests/test_jsonpath_rw_ext.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,53 @@
import testscenarios

from jsonpath_ng.ext import parser

rest_response1={
"items":
[
{
"status": "UP",
"kind": "compute#region",
"description": "us-central1",
"quotas": [
{"usage": 3.0, "metric": "CPUS", "limit": 72.0},
{"usage": 261.0, "metric": "DISKS", "limit": 40960.0},
{"usage": 0.0, "metric": "STATIC", "limit": 21.0},
{"usage": 0.0, "metric": "IN_USE", "limit": 69.0},
{"usage": 0.0, "metric": "SSD", "limit": 20480.0}
],
"id": "1000",
"name": "us-central1"
},
{
"status": "UP",
"kind": "compute#region",
"description": "us-central2",
"quotas": [
{"usage": 0.0, "metric": "CPUS", "limit": 72.0},
{"usage": 0.0, "metric": "DISKS", "limit": 40960.0},
{"usage": 0.0, "metric": "STATIC", "limit": 21.0},
{"usage": 0.0, "metric": "IN_USE", "limit": 69.0},
{"usage": 0.0, "metric": "SSD", "limit": 20480.0}
],
"id": "1001",
"name": "us-central2"
},
{
"status": "UP",
"kind": "compute#region",
"description": "us-central3",
"quotas": [
{"usage": 0.0, "metric": "CPUS", "limit": 90.0},
{"usage": 0.0, "metric": "DISKS", "limit": 2040.0},
{"usage": 0.0, "metric": "STATIC", "limit": 46.0},
{"usage": 0.0, "metric": "IN_USE", "limit": 80.0},
{"usage": 500.0, "metric": "SSD", "limit": 20480.0}
],
"id": "1002",
"name": "us-central3"
}
]
}

class Testjsonpath_ng_ext(testscenarios.WithScenarios,
base.BaseTestCase):
Expand All @@ -50,12 +96,31 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios,
('len_list', dict(string='objects.`len`',
data={'objects': ['alpha', 'gamma', 'beta']},
target=3)),
('filter_list', dict(string='objects[?@="alpha"]',
data={'objects': ['alpha', 'gamma', 'beta']},
target=['alpha'])),
('filter_list_2', dict(string='objects[?@ ~= "a.+"]',
data={'objects': ['alpha', 'gamma', 'beta']},
target=['alpha','gamma'])),
('keys_list', dict(string='objects.`keys`',
data={'objects': ['alpha', 'gamma', 'beta']},
target=[])),
('len_dict', dict(string='objects.`len`',
data={'objects': {'cow': 'moo', 'cat': 'neigh'}},
target=2)),
('keys_dict', dict(string='objects.`keys`',
data={'objects': {'cow': 'moo', 'cat': 'neigh'}},
target=['cow','cat'])),
#('filter_keys_dict', dict(string='objects.`keys`[?`this`="cow"]',
# data={'objects': {'cow': 'moo', 'cat': 'neigh'}},
# target=['cow'])),
#TODO make keys dictionaries filterable
('len_str', dict(string='objects[0].`len`',
data={'objects': ['alpha', 'gamma']},
target=5)),
('contains_filter', dict(string='objects[?id ~= "v.*[1-9]"].id',
data={'objects': [{'id':'vasll1'},{'id':'v2'},{'id':'vaal3'},{'id':'other'},{'id':'val'}]},
target=['vasll1','v2','vaal3'])),

('filter_exists_syntax1', dict(string='objects[?cow]',
data={'objects': [{'cow': 'moo'},
Expand Down Expand Up @@ -94,6 +159,12 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios,
{'cow': 5},
{'cow': 'neigh'}]},
target=[{'cow': 8}, {'cow': 7}])),
('filter_gt_negation', dict(string='objects[?!cow<=5]',
data={'objects': [{'cow': 8},
{'cow': 7},
{'cow': 5},
{'cow': 'neigh'}]},
target=[{'cow': 8}, {'cow': 7},{'cow':'neigh'}])),
('filter_and', dict(string='objects[?cow>5&cat=2]',
data={'objects': [{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2},
Expand All @@ -102,6 +173,85 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios,
{'cow': 8, 'cat': 3}]},
target=[{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2}])),
('filter_and_demorgans', dict(string='objects[?!(cow<=5|cat!=2)]',
data={'objects': [{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2},
{'cow': 2, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}]},
target=[{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2}])),
('filter_or', dict(string='objects[?cow=8|cat=3]',
data={'objects': [{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2},
{'cow': 2, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}]},
target=[{'cow': 8, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}])),
('filter_or_demorgans', dict(string='objects[?!(cow!=8&cat!=3)]',
data={'objects': [{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2},
{'cow': 2, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}]},
target=[{'cow': 8, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}])),
('filter_or_and', dict(string='objects[?cow=8&cat=2|cat=3]',
data={'objects': [{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2},
{'cow': 2, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}]},
target=[{'cow': 8, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}])),
('filter_or_and_overide', dict(string='objects[?cow=8&(cat=2|cat=3)]',
data={'objects': [{'cow': 8, 'cat': 2},
{'cow': 7, 'cat': 2},
{'cow': 2, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}]},
target=[{'cow': 8, 'cat': 2},
{'cow': 8, 'cat': 3}])),
('filter_or_and', dict(string='objects[?dog=1|cat=3&cow=8]',
data={'objects': [{'cow': 8, 'cat': 2, 'dog':1},
{'cow': 7, 'cat': 2},
{'cow': 2, 'cat': 2},
{'cow': 5, 'cat': 3},
{'cow': 8, 'cat': 3}]},
target=[{'cow': 8, 'cat': 2, 'dog':1},
{'cow': 8, 'cat': 3}])),
('filter_complex', dict(string='$.items[?((!(val==4))&(id==2))|(!((id!=1)&(id!=3)))]',
data={"items":[{"id":1, "val":1, "info":1},{"id":2, "val":4},{"id":2,"val":2},{"id":3,"val":3}]},
target=[{'info': 1, 'id': 1, 'val': 1},
{'id': 2, 'val': 2},
{'id': 3, 'val': 3}])),
('filter_complex2', dict(string="$.items[?(@.quotas[?((@.metric='SSD' & @.usage>0) | (@.metric='CPU' & @.usage>0) | (@.metric='DISKS' & @.usage>0))])]",
data=rest_response1,
target=[{'description': 'us-central1',
'id': '1000',
'kind': 'compute#region',
'name': 'us-central1',
'quotas': [{'limit': 72.0, 'metric': 'CPUS', 'usage': 3.0},
{'limit': 40960.0, 'metric': 'DISKS', 'usage': 261.0},
{'limit': 21.0, 'metric': 'STATIC', 'usage': 0.0},
{'limit': 69.0, 'metric': 'IN_USE', 'usage': 0.0},
{'limit': 20480.0, 'metric': 'SSD', 'usage': 0.0}],
'status': 'UP'},
{'description': 'us-central3',
'id': '1002',
'kind': 'compute#region',
'name': 'us-central3',
'quotas': [{'limit': 90.0, 'metric': 'CPUS', 'usage': 0.0},
{'limit': 2040.0, 'metric': 'DISKS', 'usage': 0.0},
{'limit': 46.0, 'metric': 'STATIC', 'usage': 0.0},
{'limit': 80.0, 'metric': 'IN_USE', 'usage': 0.0},
{'limit': 20480.0, 'metric': 'SSD', 'usage': 500.0}],
'status': 'UP'}])),

('filter_float_gt', dict(
string='objects[?confidence>=0.5].prediction',
data={
Expand Down