From 93649afdbb9a78c30fc7946716b09bc5d8555a96 Mon Sep 17 00:00:00 2001 From: dlacher Date: Fri, 3 May 2019 12:06:55 -0400 Subject: [PATCH 1/9] updated Filters to support predicate logic --- jsonpath_ng/ext/filter.py | 40 ++++++++-- jsonpath_ng/ext/parser.py | 14 +++- jsonpath_ng/lexer.py | 2 +- tests/test_jsonpath_rw_ext.py | 133 +++++++++++++++++++++++++++++++++- 4 files changed, 176 insertions(+), 13 deletions(-) diff --git a/jsonpath_ng/ext/filter.py b/jsonpath_ng/ext/filter.py index 4be473c..939cce2 100644 --- a/jsonpath_ng/ext/filter.py +++ b/jsonpath_ng/ext/filter.py @@ -27,6 +27,34 @@ '>': operator.gt, } +def eval_exp(expressions,val): + for expression in expressions: + if type(expression)==tuple and expression[0]=='|': + val1=eval_exp(expression[1],val) + val2=eval_exp(expression[2],val) + if (val1 or val2): + return True + else: + return False + if type(expression)==tuple and expression[0]=='&': + val1=eval_exp(expression[1],val) + val2=eval_exp(expression[2],val) + if (val1 and val2): + return True + else: + return False + if type(expression)==tuple and expression[0]=='!': + val1=eval_exp(expression[1],val) + if (val1): + return False + else: + return True + else: + if(len([expression])==len(list(filter(lambda x: x.find(val),[expression])))): + return True + else: + return False + class Filter(JSONPath): """The JSONQuery filter""" @@ -41,12 +69,11 @@ def find(self, datum): datum = DatumInContext.wrap(datum) if not isinstance(datum.value, list): return [] - - return [DatumInContext(datum.value[i], path=Index(i), context=datum) - for i in moves.range(0, len(datum.value)) - if (len(self.expressions) == - len(list(filter(lambda x: x.find(datum.value[i]), - self.expressions))))] + res=[] + for i in moves.range(0,len(datum.value)): + if eval_exp(self.expressions,datum.value[i]): + res.append(DatumInContext(datum.value[i], path=Index(i), context=datum)) + return(res) def __repr__(self): return '%s(%r)' % (self.__class__.__name__, self.expressions) @@ -65,7 +92,6 @@ def __init__(self, target, op, value): def find(self, datum): datum = self.target.find(DatumInContext.wrap(datum)) - if not datum: return [] if self.op is None: diff --git a/jsonpath_ng/ext/parser.py b/jsonpath_ng/ext/parser.py index c7d7cae..77ebe57 100644 --- a/jsonpath_ng/ext/parser.py +++ b/jsonpath_ng/ext/parser.py @@ -14,7 +14,6 @@ from .. import lexer from .. import parser from .. import Fields, This, Child - from . import arithmetic as _arithmetic from . import filter as _filter from . import iterable as _iterable @@ -119,11 +118,18 @@ def p_expression(self, p): def p_expressions_expression(self, p): "expressions : expression" p[0] = [p[1]] - + + def p_expressions_not(self, p): + "expressions : '!' expressions" + p[0]=[('!',p[2])] + def p_expressions_and(self, p): "expressions : expressions '&' expressions" - # TODO(sileht): implements '|' - p[0] = p[1] + p[3] + p[0] = [('&',p[1],p[3])] + + def p_expressions_or(self, p): + "expressions : expressions '|' expressions" + p[0] = [('|',p[1],p[3])] def p_expressions_parens(self, p): "expressions : '(' expressions ')'" diff --git a/jsonpath_ng/lexer.py b/jsonpath_ng/lexer.py index 8d53ffc..5d8e3d4 100644 --- a/jsonpath_ng/lexer.py +++ b/jsonpath_ng/lexer.py @@ -48,7 +48,7 @@ def tokenize(self, string): # # Anyhow, it is pythonic to give some rope to hang oneself with :-) - literals = ['*', '.', '[', ']', '(', ')', '$', ',', ':', '|', '&'] + literals = ['*', '.', '[', ']', '(', ')', '$', ',', ':', '|', '&', '!'] reserved_words = { 'where': 'WHERE' } diff --git a/tests/test_jsonpath_rw_ext.py b/tests/test_jsonpath_rw_ext.py index fb3c886..f1af844 100644 --- a/tests/test_jsonpath_rw_ext.py +++ b/tests/test_jsonpath_rw_ext.py @@ -25,7 +25,53 @@ import testscenarios from jsonpath_ng.ext import parser - +rest_response1={ + "items": + [ + { + "status": "UP", + "kind": "compute#region", + "description": "us-central1", + "quotas": [ + {"usage": 3.0, "metric": "CPUS", "limit": 72.0}, + {"usage": 261.0, "metric": "DISKS", "limit": 40960.0}, + {"usage": 0.0, "metric": "STATIC", "limit": 21.0}, + {"usage": 0.0, "metric": "IN_USE", "limit": 69.0}, + {"usage": 0.0, "metric": "SSD", "limit": 20480.0} + ], + "id": "1000", + "name": "us-central1" + }, + { + "status": "UP", + "kind": "compute#region", + "description": "us-central2", + "quotas": [ + {"usage": 0.0, "metric": "CPUS", "limit": 72.0}, + {"usage": 0.0, "metric": "DISKS", "limit": 40960.0}, + {"usage": 0.0, "metric": "STATIC", "limit": 21.0}, + {"usage": 0.0, "metric": "IN_USE", "limit": 69.0}, + {"usage": 0.0, "metric": "SSD", "limit": 20480.0} + ], + "id": "1001", + "name": "us-central2" + }, + { + "status": "UP", + "kind": "compute#region", + "description": "us-central3", + "quotas": [ + {"usage": 0.0, "metric": "CPUS", "limit": 90.0}, + {"usage": 0.0, "metric": "DISKS", "limit": 2040.0}, + {"usage": 0.0, "metric": "STATIC", "limit": 46.0}, + {"usage": 0.0, "metric": "IN_USE", "limit": 80.0}, + {"usage": 500.0, "metric": "SSD", "limit": 20480.0} + ], + "id": "1002", + "name": "us-central3" + } + ] +} class Testjsonpath_ng_ext(testscenarios.WithScenarios, base.BaseTestCase): @@ -94,6 +140,12 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, {'cow': 5}, {'cow': 'neigh'}]}, target=[{'cow': 8}, {'cow': 7}])), + ('filter_gt_negation', dict(string='objects[?!cow<=5]', + data={'objects': [{'cow': 8}, + {'cow': 7}, + {'cow': 5}, + {'cow': 'neigh'}]}, + target=[{'cow': 8}, {'cow': 7},{'cow':'neigh'}])), ('filter_and', dict(string='objects[?cow>5&cat=2]', data={'objects': [{'cow': 8, 'cat': 2}, {'cow': 7, 'cat': 2}, @@ -102,6 +154,85 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, {'cow': 8, 'cat': 3}]}, target=[{'cow': 8, 'cat': 2}, {'cow': 7, 'cat': 2}])), + ('filter_and_demorgans', dict(string='objects[?!(cow<=5|cat!=2)]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}])), + ('filter_or', dict(string='objects[?cow=8|cat=3]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}])), + ('filter_or_demorgans', dict(string='objects[?!(cow!=8&cat!=3)]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}])), + ('filter_or_and', dict(string='objects[?cow=8&cat=2|cat=3]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}])), + ('filter_or_and_overide', dict(string='objects[?cow=8&(cat=2|cat=3)]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 8, 'cat': 3}])), + ('filter_or_and', dict(string='objects[?dog=1|cat=3&cow=8]', + data={'objects': [{'cow': 8, 'cat': 2, 'dog':1}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2, 'dog':1}, + {'cow': 8, 'cat': 3}])), + ('filter_complex', dict(string='$.items[?((!(val==4))&(id==2))|(!((id!=1)&(id!=3)))]', + data={"items":[{"id":1, "val":1, "info":1},{"id":2, "val":4},{"id":2,"val":2},{"id":3,"val":3}]}, + target=[{'info': 1, 'id': 1, 'val': 1}, + {'id': 2, 'val': 2}, + {'id': 3, 'val': 3}])), + ('filter_complex2', dict(string="$.items[?(@.quotas[?((@.metric='SSD' & @.usage>0) | (@.metric='CPU' & @.usage>0) | (@.metric='DISKS' & @.usage>0))])]", + data=rest_response1, + target=[{'description': 'us-central1', + 'id': '1000', + 'kind': 'compute#region', + 'name': 'us-central1', + 'quotas': [{'limit': 72.0, 'metric': 'CPUS', 'usage': 3.0}, + {'limit': 40960.0, 'metric': 'DISKS', 'usage': 261.0}, + {'limit': 21.0, 'metric': 'STATIC', 'usage': 0.0}, + {'limit': 69.0, 'metric': 'IN_USE', 'usage': 0.0}, + {'limit': 20480.0, 'metric': 'SSD', 'usage': 0.0}], + 'status': 'UP'}, + {'description': 'us-central3', + 'id': '1002', + 'kind': 'compute#region', + 'name': 'us-central3', + 'quotas': [{'limit': 90.0, 'metric': 'CPUS', 'usage': 0.0}, + {'limit': 2040.0, 'metric': 'DISKS', 'usage': 0.0}, + {'limit': 46.0, 'metric': 'STATIC', 'usage': 0.0}, + {'limit': 80.0, 'metric': 'IN_USE', 'usage': 0.0}, + {'limit': 20480.0, 'metric': 'SSD', 'usage': 500.0}], + 'status': 'UP'}])), + ('filter_float_gt', dict( string='objects[?confidence>=0.5].prediction', data={ From 4ee3fb669becea15c6592d4b464a3b4a9374f155 Mon Sep 17 00:00:00 2001 From: dlacher Date: Fri, 3 May 2019 12:41:56 -0400 Subject: [PATCH 2/9] updated README --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ba40f3b..bb406a6 100644 --- a/README.rst +++ b/README.rst @@ -203,7 +203,9 @@ Extensions +--------------+----------------------------------------------+ | filter | - $.objects[?(@some_field > 5)] | | | - $.objects[?some_field = "foobar")] | -| | - $.objects[?some_field > 5 & other < 2)] | +| | - $.objects[?some_field > 5 & other < 2)] and| +| | - $.objects[?some_field>5 |some_field<2)] or | +| | - $.objects[?(!(field>5 | field<2))] not| +--------------+----------------------------------------------+ | arithmetic | - $.foo + "_" + $.bar | | (-+*/) | - $.foo * 12 | From 62b53af010ddd683a1783f13a922ab49d542f46a Mon Sep 17 00:00:00 2001 From: dlacher Date: Thu, 23 May 2019 10:54:57 -0400 Subject: [PATCH 3/9] added support for regex filters \n added support for . keyword --- jsonpath_ng/ext/filter.py | 7 +++++++ jsonpath_ng/ext/iterable.py | 26 ++++++++++++++++++++++++++ jsonpath_ng/ext/parser.py | 8 +++++--- jsonpath_ng/lexer.py | 2 +- tests/test_jsonpath_rw_ext.py | 9 +++++++++ 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/jsonpath_ng/ext/filter.py b/jsonpath_ng/ext/filter.py index 939cce2..7a1ef64 100644 --- a/jsonpath_ng/ext/filter.py +++ b/jsonpath_ng/ext/filter.py @@ -13,9 +13,15 @@ import operator from six import moves +import re from .. import JSONPath, DatumInContext, Index +def contains(a,b): + if re.search(b,a): + return True + return False + OPERATOR_MAP = { '!=': operator.ne, @@ -25,6 +31,7 @@ '<': operator.lt, '>=': operator.ge, '>': operator.gt, + '~=': contains } def eval_exp(expressions,val): diff --git a/jsonpath_ng/ext/iterable.py b/jsonpath_ng/ext/iterable.py index 92ece5f..e5ddbfc 100644 --- a/jsonpath_ng/ext/iterable.py +++ b/jsonpath_ng/ext/iterable.py @@ -90,3 +90,29 @@ def __str__(self): def __repr__(self): return 'Len()' + +class Keys(JSONPath): + """The JSONPath referring to the keys of the current object. + + Concrete syntax is '`keys`'. + """ + + def find(self, datum): + datum = DatumInContext.wrap(datum) + try: + value = datum.value.keys() + except Exception as e: + return [] + else: + return [DatumInContext(value[i], + context=None, + path=Keys()) for i in range (0, len(datum.value))] + + def __eq__(self, other): + return isinstance(other, Keys) + + def __str__(self): + return '`keys`' + + def __repr__(self): + return 'Keys()' diff --git a/jsonpath_ng/ext/parser.py b/jsonpath_ng/ext/parser.py index 77ebe57..5ba4158 100644 --- a/jsonpath_ng/ext/parser.py +++ b/jsonpath_ng/ext/parser.py @@ -22,12 +22,12 @@ class ExtendedJsonPathLexer(lexer.JsonPathLexer): """Custom LALR-lexer for JsonPath""" - literals = lexer.JsonPathLexer.literals + ['?', '@', '+', '*', '/', '-'] + literals = lexer.JsonPathLexer.literals + ['?', '@', '+', '*', '/', '-', '!','~'] tokens = (['BOOL'] + parser.JsonPathLexer.tokens + ['FILTER_OP', 'SORT_DIRECTION', 'FLOAT']) - - t_FILTER_OP = r'==?|<=|>=|!=|<|>' + print "tokens {}".format(tokens) + t_FILTER_OP = r'==?|<=|>=|!=|<|>|~=' def t_BOOL(self, t): r'true|false' @@ -93,6 +93,8 @@ def p_jsonpath_named_operator(self, p): "jsonpath : NAMED_OPERATOR" if p[1] == 'len': p[0] = _iterable.Len() + elif p[1] == 'keys': + p[0] = _iterable.Keys() elif p[1] == 'sorted': p[0] = _iterable.SortedThis() elif p[1].startswith("split("): diff --git a/jsonpath_ng/lexer.py b/jsonpath_ng/lexer.py index 5d8e3d4..8d53ffc 100644 --- a/jsonpath_ng/lexer.py +++ b/jsonpath_ng/lexer.py @@ -48,7 +48,7 @@ def tokenize(self, string): # # Anyhow, it is pythonic to give some rope to hang oneself with :-) - literals = ['*', '.', '[', ']', '(', ')', '$', ',', ':', '|', '&', '!'] + literals = ['*', '.', '[', ']', '(', ')', '$', ',', ':', '|', '&'] reserved_words = { 'where': 'WHERE' } diff --git a/tests/test_jsonpath_rw_ext.py b/tests/test_jsonpath_rw_ext.py index f1af844..5a03172 100644 --- a/tests/test_jsonpath_rw_ext.py +++ b/tests/test_jsonpath_rw_ext.py @@ -96,12 +96,21 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, ('len_list', dict(string='objects.`len`', data={'objects': ['alpha', 'gamma', 'beta']}, target=3)), + ('keys_list', dict(string='objects.`keys`', + data={'objects': ['alpha', 'gamma', 'beta']}, + target=[])), ('len_dict', dict(string='objects.`len`', data={'objects': {'cow': 'moo', 'cat': 'neigh'}}, target=2)), + ('keys_dict', dict(string='objects.`keys`', + data={'objects': {'cow': 'moo', 'cat': 'neigh'}}, + target=['cow','cat'])), ('len_str', dict(string='objects[0].`len`', data={'objects': ['alpha', 'gamma']}, target=5)), + ('contains_filter', dict(string='objects[?id ~= "v.*[1-9]"].id', + data={'objects': [{'id':'vasll1'},{'id':'v2'},{'id':'vaal3'},{'id':'other'},{'id':'val'}]}, + target=['vasll1','v2','vaal3'])), ('filter_exists_syntax1', dict(string='objects[?cow]', data={'objects': [{'cow': 'moo'}, From 23c2f64e14a36bcb4d647bc7529533e5e5b24197 Mon Sep 17 00:00:00 2001 From: dlacher Date: Fri, 24 May 2019 09:41:23 -0400 Subject: [PATCH 4/9] removed excess print --- jsonpath_ng/ext/parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jsonpath_ng/ext/parser.py b/jsonpath_ng/ext/parser.py index 5ba4158..9da08a7 100644 --- a/jsonpath_ng/ext/parser.py +++ b/jsonpath_ng/ext/parser.py @@ -26,7 +26,6 @@ class ExtendedJsonPathLexer(lexer.JsonPathLexer): tokens = (['BOOL'] + parser.JsonPathLexer.tokens + ['FILTER_OP', 'SORT_DIRECTION', 'FLOAT']) - print "tokens {}".format(tokens) t_FILTER_OP = r'==?|<=|>=|!=|<|>|~=' def t_BOOL(self, t): From 24925934bb4a5229abf52006e5270cf9a7bf7de0 Mon Sep 17 00:00:00 2001 From: dlacher Date: Fri, 24 May 2019 09:49:29 -0400 Subject: [PATCH 5/9] updated README --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index bb406a6..0d1701e 100644 --- a/README.rst +++ b/README.rst @@ -192,6 +192,8 @@ Extensions +==============+==============================================+ | len | - $.objects.`len` | +--------------+----------------------------------------------+ +| keys | - $.dict_field.`keys`(returns a list of keys)| ++--------------+----------------------------------------------+ | sub | - $.field.`sub(/foo\\\\+(.*)/, \\\\1)` | +--------------+----------------------------------------------+ | split | - $.field.`split(+, 2, -1)` | @@ -206,6 +208,7 @@ Extensions | | - $.objects[?some_field > 5 & other < 2)] and| | | - $.objects[?some_field>5 |some_field<2)] or | | | - $.objects[?(!(field>5 | field<2))] not| +| | - $.objects[?@.field ~= "a.+a"] regex| +--------------+----------------------------------------------+ | arithmetic | - $.foo + "_" + $.bar | | (-+*/) | - $.foo * 12 | From 8cacb50796e250a4f9fab0ca0ccf3b20cf082fd8 Mon Sep 17 00:00:00 2001 From: dlacher Date: Thu, 27 Jun 2019 10:12:57 -0400 Subject: [PATCH 6/9] updated unittests --- tests/test_jsonpath_rw_ext.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_jsonpath_rw_ext.py b/tests/test_jsonpath_rw_ext.py index 5a03172..192e5c6 100644 --- a/tests/test_jsonpath_rw_ext.py +++ b/tests/test_jsonpath_rw_ext.py @@ -96,6 +96,12 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, ('len_list', dict(string='objects.`len`', data={'objects': ['alpha', 'gamma', 'beta']}, target=3)), + ('filter_list', dict(string='objects[?@="alpha"]', + data={'objects': ['alpha', 'gamma', 'beta']}, + target=['alpha'])), + ('filter_list_2', dict(string='objects[?@ ~= "a.+"]', + data={'objects': ['alpha', 'gamma', 'beta']}, + target=['alpha','gamma'])), ('keys_list', dict(string='objects.`keys`', data={'objects': ['alpha', 'gamma', 'beta']}, target=[])), @@ -105,6 +111,10 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, ('keys_dict', dict(string='objects.`keys`', data={'objects': {'cow': 'moo', 'cat': 'neigh'}}, target=['cow','cat'])), + #('filter_keys_dict', dict(string='objects.`keys`[?`this`="cow"]', + # data={'objects': {'cow': 'moo', 'cat': 'neigh'}}, + # target=['cow'])), + #TODO make keys dictionaries filterable ('len_str', dict(string='objects[0].`len`', data={'objects': ['alpha', 'gamma']}, target=5)), From a00b2184557fd18975cf0a87d26f11af61dcfec2 Mon Sep 17 00:00:00 2001 From: Laurent PINSON Date: Thu, 12 May 2022 11:02:39 +0200 Subject: [PATCH 7/9] new rule idx_comma + change Index() to handle indices instead of index --- jsonpath_ng/jsonpath.py | 43 +++++++++++++++++++++++++---------------- jsonpath_ng/parser.py | 10 +++++++--- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/jsonpath_ng/jsonpath.py b/jsonpath_ng/jsonpath.py index f4f9d4b..14a9bc4 100644 --- a/jsonpath_ng/jsonpath.py +++ b/jsonpath_ng/jsonpath.py @@ -606,8 +606,8 @@ class Index(JSONPath): NOTE: For the concrete syntax of `[*]`, the abstract syntax is a Slice() with no parameters (equiv to `[:]` """ - def __init__(self, index): - self.index = index + def __init__(self, *indices): + self.indices = indices def find(self, datum): return self._find_base(datum, create=False) @@ -621,10 +621,12 @@ def _find_base(self, datum, create): if datum.value == {}: datum.value = _create_list_key(datum.value) self._pad_value(datum.value) - if datum.value and len(datum.value) > self.index: - return [DatumInContext(datum.value[self.index], path=self, context=datum)] - else: - return [] + rv = [] + for index in self.indices: + # invalid indices do not crash, return [] instead + if datum.value and len(datum.value) > index: + rv += [DatumInContext(datum.value[index], path=self, context=datum)] + return rv def update(self, data, val): return self._update_base(data, val, create=False) @@ -638,31 +640,38 @@ def _update_base(self, data, val, create): data = _create_list_key(data) self._pad_value(data) if hasattr(val, '__call__'): - val.__call__(data[self.index], data, self.index) - elif len(data) > self.index: - data[self.index] = val + for index in self.indices: + val.__call__(data[index], data, index) + else: + if not isinstance(val, list): + val = [val] + # allows somelist[5,1,2] = [some_value, another_value, third_value] + for index in self.indices: + if len(data) > index: + data[index] = val.pop(0) return data def filter(self, fn, data): - if fn(data[self.index]): - data.pop(self.index) # relies on mutation :( + for index in self.indices: + if fn(data[index]): + data.pop(index) # relies on mutation :( return data def __eq__(self, other): - return isinstance(other, Index) and self.index == other.index + return isinstance(other, Index) and sorted(self.indices) == sorted(other.indices) def __str__(self): - return '[%i]' % self.index + return '[%i]' % self.indices def __repr__(self): - return '%s(index=%r)' % (self.__class__.__name__, self.index) + return '%s(indices=%r)' % (self.__class__.__name__, self.indices) def _pad_value(self, value): - if len(value) <= self.index: - pad = self.index - len(value) + 1 + _max = max(self.indices) + if len(value) <= _max: + pad = _max - len(value) + 1 value += [{} for __ in range(pad)] - class Slice(JSONPath): """ JSONPath matching a slice of an array. diff --git a/jsonpath_ng/parser.py b/jsonpath_ng/parser.py index 72333c2..d74b590 100644 --- a/jsonpath_ng/parser.py +++ b/jsonpath_ng/parser.py @@ -120,7 +120,7 @@ def p_jsonpath_root(self, p): def p_jsonpath_idx(self, p): "jsonpath : '[' idx ']'" - p[0] = p[2] + p[0] = Index(*p[2]) def p_jsonpath_slice(self, p): "jsonpath : '[' slice ']'" @@ -136,7 +136,7 @@ def p_jsonpath_child_fieldbrackets(self, p): def p_jsonpath_child_idxbrackets(self, p): "jsonpath : jsonpath '[' idx ']'" - p[0] = Child(p[1], p[3]) + p[0] = Child(p[1], Index(*p[3])) def p_jsonpath_child_slicebrackets(self, p): "jsonpath : jsonpath '[' slice ']'" @@ -165,8 +165,12 @@ def p_fields_comma(self, p): def p_idx(self, p): "idx : NUMBER" - p[0] = Index(p[1]) + p[0] = [p[1]] + def p_idx_comma(self, p): + "idx : idx ',' idx " + p[0] = p[1] + p[3] + def p_slice_any(self, p): "slice : '*'" p[0] = Slice() From 5f7e5aee1f6fa41f97dd2f8d8fcf9dd961ebf2bc Mon Sep 17 00:00:00 2001 From: Laurent PINSON Date: Thu, 12 May 2022 18:04:46 +0200 Subject: [PATCH 8/9] all tests passed --- jsonpath_ng/jsonpath.py | 7 ++-- tests/test_create.py | 50 +++++++++++++++++++++++ tests/test_examples.py | 5 ++- tests/test_jsonpath.py | 88 ++++++++++++++++++++++------------------- tests/test_parser.py | 1 + 5 files changed, 105 insertions(+), 46 deletions(-) diff --git a/jsonpath_ng/jsonpath.py b/jsonpath_ng/jsonpath.py index 14a9bc4..1617330 100644 --- a/jsonpath_ng/jsonpath.py +++ b/jsonpath_ng/jsonpath.py @@ -625,7 +625,7 @@ def _find_base(self, datum, create): for index in self.indices: # invalid indices do not crash, return [] instead if datum.value and len(datum.value) > index: - rv += [DatumInContext(datum.value[index], path=self, context=datum)] + rv += [DatumInContext(datum.value[index], path=Index(index), context=datum)] return rv def update(self, data, val): @@ -646,6 +646,7 @@ def _update_base(self, data, val, create): if not isinstance(val, list): val = [val] # allows somelist[5,1,2] = [some_value, another_value, third_value] + # skip the indices that are too high but the value will be applied to the next index for index in self.indices: if len(data) > index: data[index] = val.pop(0) @@ -661,7 +662,7 @@ def __eq__(self, other): return isinstance(other, Index) and sorted(self.indices) == sorted(other.indices) def __str__(self): - return '[%i]' % self.indices + return str(list(self.indices)) def __repr__(self): return '%s(indices=%r)' % (self.__class__.__name__, self.indices) @@ -718,7 +719,7 @@ def find(self, datum): return [DatumInContext(datum.value[i], path=Index(i), context=datum) for i in xrange(0, len(datum.value))] else: return [DatumInContext(datum.value[i], path=Index(i), context=datum) for i in range(0, len(datum.value))[self.start:self.end:self.step]] - + def update(self, data, val): for datum in self.find(data): datum.path.update(data, val) diff --git a/tests/test_create.py b/tests/test_create.py index 579fcd2..cb91289 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -31,6 +31,11 @@ insert_val=42, target={'foo': [{}, 42]}), + Params(string='$.foo[1,3]', + initial_data={}, + insert_val=[42, 51], + target={'foo': [{}, 42, {}, 51]}), + Params(string='$.foo[0].bar', initial_data={}, insert_val=42, @@ -41,6 +46,12 @@ insert_val=42, target={'foo': [{}, {'bar': 42}]}), + # Note that each field will received the full + Params(string='$.foo[1,3].bar', + initial_data={}, + insert_val=[42, 51], + target={'foo': [{}, {'bar': [42, 51]}, {}, {'bar': [42, 51]}]}), + Params(string='$.foo[0][0]', initial_data={}, insert_val=42, @@ -51,6 +62,22 @@ insert_val=42, target={'foo': [{}, [{}, 42]]}), + # But here Note that each index will received one value of + Params(string='$.foo[1,3][1]', + initial_data={}, + insert_val=[42, 51], + target={'foo': [{}, [{}, 42], {}, [{}, 51]]}), + + Params(string='$.foo[1,3][1]', + initial_data={}, + insert_val=[[42, 51], [42, 51]], + target={'foo': [{}, [{}, [42, 51]], {}, [{}, [42, 51]]]}), + + Params(string='$.foo[1,3][0,2]', + initial_data={}, + insert_val=[42, 51, 42, 51], + target={'foo': [{}, [42, {}, 51], {}, [42, {}, 51]]}), + Params(string='foo[0]', initial_data={}, insert_val=42, @@ -61,6 +88,11 @@ insert_val=42, target={'foo': [{}, 42]}), + Params(string='foo[1,3]', + initial_data={}, + insert_val=[42, 51], + target={'foo': [{}, 42, {}, 51]}), + Params(string='foo', initial_data={}, insert_val=42, @@ -77,6 +109,11 @@ insert_val=42, target=[{}, 42]), + Params(string='[1,3]', + initial_data=[], + insert_val=[42, 51], + target=[{}, 42, {}, 51]), + # Converts initial data to a list if necessary Params(string='[0]', initial_data={}, @@ -88,6 +125,11 @@ insert_val=42, target=[{}, 42]), + Params(string='[1,3]', + initial_data={}, + insert_val=[42, 51], + target=[{}, 42, {}, 51]), + Params(string='foo[?bar="baz"].qux', initial_data={'foo': [ {'bar': 'baz'}, @@ -127,6 +169,14 @@ def test_update_or_create(string, initial_data, insert_val, target): target={'foo': 42}), # raises TypeError + # more indices than values to insert + Params(string='$.foo[1,2,3]', + initial_data={}, + insert_val=[42, 51], + target={'foo': [{}, 42, 51, {}]}), + # raises IndexError + + ]) @pytest.mark.xfail def test_unsupported_classes(string, initial_data, insert_val, target): diff --git a/tests/test_examples.py b/tests/test_examples.py index 1b08123..b9e3ad6 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -32,8 +32,9 @@ Child(Descendants(Root(), Fields('book')), Slice(start=-1))), # The first two books - # ("$..book[0,1]", # Not implemented - # Child(Descendants(Root(), Fields('book')), Slice(end=2))), + ("$..book[0,1]", + Child(Descendants(Root(), Fields('book')), Index(0,1))), + ("$..book[:2]", Child(Descendants(Root(), Fields('book')), Slice(end=2))), diff --git a/tests/test_jsonpath.py b/tests/test_jsonpath.py index 60670b1..0728283 100644 --- a/tests/test_jsonpath.py +++ b/tests/test_jsonpath.py @@ -86,10 +86,13 @@ def setup_class(cls): # # Check that the data value returned is good # - def check_cases(self, test_cases): + def check_cases(self, test_cases, auto_id_field=None): # Note that just manually building an AST would avoid this dep and isolate the tests, but that would suck a bit # Also, we coerce iterables, etc, into the desired target type + # This is a global parameter, it should be updated with each call of this function automatically + # Not manually outside this function, or someone might forget to switch it back to None + jsonpath.auto_id_field = auto_id_field for string, data, target in test_cases: print('parse("%s").find(%s) =?= %s' % (string, data, target)) result = parse(string).find(data) @@ -101,17 +104,16 @@ def check_cases(self, test_cases): assert result.value == target def test_fields_value(self): - jsonpath.auto_id_field = None self.check_cases([ ('foo', {'foo': 'baz'}, ['baz']), ('foo,baz', {'foo': 1, 'baz': 2}, [1, 2]), ('@foo', {'@foo': 1}, [1]), - ('*', {'foo': 1, 'baz': 2}, set([1, 2])) ]) + ('*', {'foo': 1, 'baz': 2}, set([1, 2])) + ]) - jsonpath.auto_id_field = 'id' - self.check_cases([ ('*', {'foo': 1, 'baz': 2}, set([1, 2, '`this`'])) ]) + self.check_cases([ ('*', {'foo': 1, 'baz': 2}, set([1, 2, '`this`'])) + ], 'id') def test_root_value(self): - jsonpath.auto_id_field = None self.check_cases([ ('$', {'foo': 'baz'}, [{'foo':'baz'}]), ('foo.$', {'foo': 'baz'}, [{'foo':'baz'}]), @@ -119,7 +121,6 @@ def test_root_value(self): ]) def test_this_value(self): - jsonpath.auto_id_field = None self.check_cases([ ('`this`', {'foo': 'baz'}, [{'foo':'baz'}]), ('foo.`this`', {'foo': 'baz'}, ['baz']), @@ -131,6 +132,7 @@ def test_index_value(self): ('[0]', [42], [42]), ('[5]', [42], []), ('[2]', [34, 65, 29, 59], [29]), + ('[0,2,5]', [34, 65, 29, 59, 17, 3], [34, 29, 3]), ('[0]', None, []) ]) @@ -138,7 +140,8 @@ def test_slice_value(self): self.check_cases([('[*]', [1, 2, 3], [1, 2, 3]), ('[*]', xrange(1, 4), [1, 2, 3]), ('[1:]', [1, 2, 3, 4], [2, 3, 4]), - ('[:2]', [1, 2, 3, 4], [1, 2])]) + ('[:2]', [1, 2, 3, 4], [1, 2]) + ]) # Funky slice hacks self.check_cases([ @@ -151,7 +154,8 @@ def test_slice_value(self): def test_child_value(self): self.check_cases([('foo.baz', {'foo': {'baz': 3}}, [3]), ('foo.baz', {'foo': {'baz': [3]}}, [[3]]), - ('foo.baz.bizzle', {'foo': {'baz': {'bizzle': 5}}}, [5])]) + ('foo.baz.bizzle', {'foo': {'baz': {'bizzle': 5}}}, [5]) + ]) def test_descendants_value(self): self.check_cases([ @@ -161,28 +165,28 @@ def test_descendants_value(self): def test_parent_value(self): self.check_cases([('foo.baz.`parent`', {'foo': {'baz': 3}}, [{'baz': 3}]), - ('foo.`parent`.foo.baz.`parent`.baz.bizzle', {'foo': {'baz': {'bizzle': 5}}}, [5])]) + ('foo.`parent`.foo.baz.`parent`.baz.bizzle', {'foo': {'baz': {'bizzle': 5}}}, [5]) + ]) def test_hyphen_key(self): self.check_cases([('foo.bar-baz', {'foo': {'bar-baz': 3}}, [3]), - ('foo.[bar-baz,blah-blah]', {'foo': {'bar-baz': 3, 'blah-blah':5}}, - [3,5])]) + ('foo.[bar-baz,blah-blah]', {'foo': {'bar-baz': 3, 'blah-blah':5}}, [3,5]) + ]) self.assertRaises(JsonPathLexerError, self.check_cases, [('foo.-baz', {'foo': {'-baz': 8}}, [8])]) - - # # Check that the paths for the data are correct. # FIXME: merge these tests with the above, since the inputs are the same anyhow # - def check_paths(self, test_cases): + def check_paths(self, test_cases, auto_id_field=None): # Note that just manually building an AST would avoid this dep and isolate the tests, but that would suck a bit # Also, we coerce iterables, etc, into the desired target type + jsonpath.auto_id_field = auto_id_field for string, data, target in test_cases: - print('parse("%s").find(%s).paths =?= %s' % (string, data, target)) + print('parse("%s").find(%s).path =?= %s' % (string, data, target)) result = parse(string).find(data) if isinstance(target, list): assert [str(r.full_path) for r in result] == target @@ -192,16 +196,15 @@ def check_paths(self, test_cases): assert str(result.path) == target def test_fields_paths(self): - jsonpath.auto_id_field = None self.check_paths([ ('foo', {'foo': 'baz'}, ['foo']), ('foo,baz', {'foo': 1, 'baz': 2}, ['foo', 'baz']), - ('*', {'foo': 1, 'baz': 2}, set(['foo', 'baz'])) ]) + ('*', {'foo': 1, 'baz': 2}, set(['foo', 'baz'])) + ], None) - jsonpath.auto_id_field = 'id' - self.check_paths([ ('*', {'foo': 1, 'baz': 2}, set(['foo', 'baz', 'id'])) ]) + self.check_paths([ ('*', {'foo': 1, 'baz': 2}, set(['foo', 'baz', 'id'])) + ], 'id') def test_root_paths(self): - jsonpath.auto_id_field = None self.check_paths([ ('$', {'foo': 'baz'}, ['$']), ('foo.$', {'foo': 'baz'}, ['$']), @@ -209,7 +212,6 @@ def test_root_paths(self): ]) def test_this_paths(self): - jsonpath.auto_id_field = None self.check_paths([ ('`this`', {'foo': 'baz'}, ['`this`']), ('foo.`this`', {'foo': 'baz'}, ['foo']), @@ -218,70 +220,72 @@ def test_this_paths(self): def test_index_paths(self): self.check_paths([('[0]', [42], ['[0]']), - ('[2]', [34, 65, 29, 59], ['[2]'])]) + ('[2]', [34, 65, 29, 59], ['[2]']), + ('[1,2,3]', [34, 65, 29, 59], ['[1]', '[2]', '[3]']), + ]) def test_slice_paths(self): self.check_paths([ ('[*]', [1, 2, 3], ['[0]', '[1]', '[2]']), - ('[1:]', [1, 2, 3, 4], ['[1]', '[2]', '[3]']) ]) + ('[1:]', [1, 2, 3, 4], ['[1]', '[2]', '[3]']) + ]) def test_child_paths(self): self.check_paths([('foo.baz', {'foo': {'baz': 3}}, ['foo.baz']), ('foo.baz', {'foo': {'baz': [3]}}, ['foo.baz']), - ('foo.baz.bizzle', {'foo': {'baz': {'bizzle': 5}}}, ['foo.baz.bizzle'])]) + ('foo.baz.bizzle', {'foo': {'baz': {'bizzle': 5}}}, ['foo.baz.bizzle']) + ]) def test_descendants_paths(self): - self.check_paths([('foo..baz', {'foo': {'baz': 1, 'bing': {'baz': 2}}}, ['foo.baz', 'foo.bing.baz'] )]) + self.check_paths([('foo..baz', {'foo': {'baz': 1, 'bing': {'baz': 2}}}, ['foo.baz', 'foo.bing.baz'] ) + ]) # # Check the "auto_id_field" feature # def test_fields_auto_id(self): - jsonpath.auto_id_field = "id" self.check_cases([ ('foo.id', {'foo': 'baz'}, ['foo']), ('foo.id', {'foo': {'id': 'baz'}}, ['baz']), ('foo,baz.id', {'foo': 1, 'baz': 2}, ['foo', 'baz']), ('*.id', {'foo':{'id': 1}, 'baz': 2}, - set(['1', 'baz'])) ]) + set(['1', 'baz'])) + ], 'id') def test_root_auto_id(self): - jsonpath.auto_id_field = 'id' self.check_cases([ ('$.id', {'foo': 'baz'}, ['$']), # This is a wonky case that is not that interesting ('foo.$.id', {'foo': 'baz', 'id': 'bizzle'}, ['bizzle']), ('foo.$.baz.id', {'foo': 4, 'baz': 3}, ['baz']), - ]) + ], 'id') def test_this_auto_id(self): - jsonpath.auto_id_field = 'id' self.check_cases([ ('id', {'foo': 'baz'}, ['`this`']), # This is, again, a wonky case that is not that interesting ('foo.`this`.id', {'foo': 'baz'}, ['foo']), ('foo.`this`.baz.id', {'foo': {'baz': 3}}, ['foo.baz']), - ]) + ], 'id') def test_index_auto_id(self): - jsonpath.auto_id_field = "id" self.check_cases([('[0].id', [42], ['[0]']), - ('[2].id', [34, 65, 29, 59], ['[2]'])]) + ('[2].id', [34, 65, 29, 59], ['[2]']) + ], 'id') def test_slice_auto_id(self): - jsonpath.auto_id_field = "id" self.check_cases([ ('[*].id', [1, 2, 3], ['[0]', '[1]', '[2]']), - ('[1:].id', [1, 2, 3, 4], ['[1]', '[2]', '[3]']) ]) + ('[1:].id', [1, 2, 3, 4], ['[1]', '[2]', '[3]']) + ], 'id') def test_child_auto_id(self): - jsonpath.auto_id_field = "id" self.check_cases([('foo.baz.id', {'foo': {'baz': 3}}, ['foo.baz']), ('foo.baz.id', {'foo': {'baz': [3]}}, ['foo.baz']), ('foo.baz.id', {'foo': {'id': 'bizzle', 'baz': 3}}, ['bizzle.baz']), ('foo.baz.id', {'foo': {'baz': {'id': 'hi'}}}, ['foo.hi']), - ('foo.baz.bizzle.id', {'foo': {'baz': {'bizzle': 5}}}, ['foo.baz.bizzle'])]) + ('foo.baz.bizzle.id', {'foo': {'baz': {'bizzle': 5}}}, ['foo.baz.bizzle']) + ], 'id') def test_descendants_auto_id(self): - jsonpath.auto_id_field = "id" self.check_cases([('foo..baz.id', {'foo': { 'baz': 1, @@ -290,9 +294,11 @@ def test_descendants_auto_id(self): } } }, ['foo.baz', - 'foo.bing.baz'] )]) + 'foo.bing.baz'] ) + ], 'id') - def check_update_cases(self, test_cases): + def check_update_cases(self, test_cases, auto_id_field=None): + jsonpath.auto_id_field = auto_id_field for original, expr_str, value, expected in test_cases: print('parse(%r).update(%r, %r) =?= %r' % (expr_str, original, value, expected)) diff --git a/tests/test_parser.py b/tests/test_parser.py index b728590..820bb16 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -24,6 +24,7 @@ def test_atomic(self): ('*', Fields('*')), ('baz,bizzle', Fields('baz','bizzle')), ('[1]', Index(1)), + ('[0,2,3]', Index(0,2,3)), ('[1:]', Slice(start=1)), ('[:]', Slice()), ('[*]', Slice()), From abc73d5d090aaae691f6fd81c09d6d96a4b57ee9 Mon Sep 17 00:00:00 2001 From: Laurent PINSON Date: Fri, 13 May 2022 10:03:58 +0200 Subject: [PATCH 9/9] added step in slice --- jsonpath_ng/parser.py | 5 +++-- tests/test_jsonpath.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/jsonpath_ng/parser.py b/jsonpath_ng/parser.py index d74b590..7e9693a 100644 --- a/jsonpath_ng/parser.py +++ b/jsonpath_ng/parser.py @@ -176,8 +176,9 @@ def p_slice_any(self, p): p[0] = Slice() def p_slice(self, p): # Currently does not support `step` - "slice : maybe_int ':' maybe_int" - p[0] = Slice(start=p[1], end=p[3]) + """slice : maybe_int ':' maybe_int + | maybe_int ':' maybe_int ':' maybe_int """ + p[0] = Slice(*p[1::2]) def p_maybe_int(self, p): """maybe_int : NUMBER diff --git a/tests/test_jsonpath.py b/tests/test_jsonpath.py index 0728283..07c6d9b 100644 --- a/tests/test_jsonpath.py +++ b/tests/test_jsonpath.py @@ -140,7 +140,11 @@ def test_slice_value(self): self.check_cases([('[*]', [1, 2, 3], [1, 2, 3]), ('[*]', xrange(1, 4), [1, 2, 3]), ('[1:]', [1, 2, 3, 4], [2, 3, 4]), - ('[:2]', [1, 2, 3, 4], [1, 2]) + ('[:2]', [1, 2, 3, 4], [1, 2]), + ('[:3:2]', [1, 2, 3, 4], [1, 3]), + ('[1::2]', [1, 2, 3, 4], [2, 4]), + ('[1:5:3]', [1, 2, 3, 4, 5], [2, 5]), + ('[::-2]', [1, 2, 3, 4, 5], [5, 3, 1]), ]) # Funky slice hacks @@ -226,7 +230,11 @@ def test_index_paths(self): def test_slice_paths(self): self.check_paths([ ('[*]', [1, 2, 3], ['[0]', '[1]', '[2]']), - ('[1:]', [1, 2, 3, 4], ['[1]', '[2]', '[3]']) + ('[1:]', [1, 2, 3, 4], ['[1]', '[2]', '[3]']), + ('[1:3]', [1, 2, 3, 4], ['[1]', '[2]']), + ('[1::2]', [1, 2, 3, 4], ['[1]', '[3]']), + ('[::-1]', [1, 2, 3], ['[2]', '[1]', '[0]']), + ('[1:6:3]', xrange(10), ['[1]', '[4]']), ]) def test_child_paths(self):