diff --git a/CHANGELOG b/CHANGELOG index f03b717..668b953 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,9 @@ jello changelog +20230114 v1.5.5 +- Fix schema output to ensure invalid variable names are enclosed in bracket notation +- Fix to allow blank lines when slurping JSON Lines objects + 20220730 v1.5.4 - Add `__main__.py` to package for `python -m jello` use cases diff --git a/jello/__init__.py b/jello/__init__.py index a791442..a2c98b0 100644 --- a/jello/__init__.py +++ b/jello/__init__.py @@ -1,8 +1,8 @@ """jello - query JSON at the command line with python syntax""" -__version__ = '1.5.4' +__version__ = '1.5.5' AUTHOR = 'Kelly Brazil' WEBSITE = 'https://github.com/kellyjonbrazil/jello' -COPYRIGHT = '© 2020-2022 Kelly Brazil' +COPYRIGHT = '© 2020-2023 Kelly Brazil' LICENSE = 'MIT License' diff --git a/jello/lib.py b/jello/lib.py index cd4830e..ae177f0 100644 --- a/jello/lib.py +++ b/jello/lib.py @@ -5,6 +5,7 @@ import ast import json import shutil +from keyword import iskeyword from textwrap import TextWrapper from jello.dotmap import DotMap @@ -22,6 +23,21 @@ PYGMENTS_INSTALLED = False +def is_valid_variable_name(name: str) -> bool: + dict_methods = [ + '__class__', '__class_getitem__', '__contains__', '__delattr__', + '__delitem__', '__dir__', '__eq__', '__format__', '__ge__', + '__getattribute__', '__getitem__', '__getstate__', '__gt__', + '__init__', '__init_subclass__', '__ior__', '__iter__', '__le__', + '__len__', '__lt__', '__ne__', '__new__', '__or__', '__reduce__', + '__reduce_ex__', '__repr__', '__reversed__', '__ror__', '__setattr__', + '__setitem__', '__sizeof__', '__str__', '__subclasshook__', 'clear', + 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', + 'setdefault', 'update', 'values' + ] + return name.isidentifier() and not iskeyword(name) and name not in dict_methods + + class opts: initialize = None version_info = None @@ -177,11 +193,11 @@ def _schema_gen(self, src, path='_'): self._schema_list.append(f'{path} = {val};{padding}{val_type}') for k, v in src.items(): - # encapsulate key in brackets if it includes spaces - if ' ' in k: - k = f'["{k}"]' - else: + # encapsulate key in brackets if it is not a valid variable name + if is_valid_variable_name(k): k = f'.{k}' + else: + k = f'["{k}"]' if isinstance(v, list): # print empty brackets as first list definition @@ -335,7 +351,7 @@ def load_json(data): except Exception as e: try: # if json.loads fails, try loading as json lines - json_dict = [json.loads(i) for i in data.splitlines()] + json_dict = [json.loads(i) for i in data.splitlines() if i.strip()] except Exception: # raise original JSON exception instead of JSON Lines exception raise e diff --git a/setup.py b/setup.py index 79848e2..bcc5e8f 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='jello', - version='1.5.4', + version='1.5.5', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Filter JSON and JSON Lines data with Python syntax.', diff --git a/tests/test_create_schema.py b/tests/test_create_schema.py index b96ec78..169e622 100644 --- a/tests/test_create_schema.py +++ b/tests/test_create_schema.py @@ -139,6 +139,16 @@ def setUp(self): self.deep_nest_sample = [[[[{"foo":[[[[1,2,3]]]]}]]]] + self.reserved_or_invalid_keynames_sample = { + "True": 1, + "get": 1, + "fromkeys": 1, + "aquaero-hid-3-1cb1": 1, + "if": 1, + "return": 1, + "__class__": 1 + } + # ------------ Tests ------------ # @@ -534,5 +544,18 @@ def test_deep_nest_m(self): expected = '_ = [];\n_[0] = [];\n_[0][0] = [];\n_[0][0][0] = [];\n_[0][0][0][0] = {};\n_[0][0][0][0].foo = [];\n_[0][0][0][0].foo[0] = [];\n_[0][0][0][0].foo[0][0] = [];\n_[0][0][0][0].foo[0][0][0] = [];\n_[0][0][0][0].foo[0][0][0][0] = 1;\n_[0][0][0][0].foo[0][0][0][1] = 2;\n_[0][0][0][0].foo[0][0][0][2] = 3;' self.assertEqual(self.schema.create_schema(data_in), expected) + # + # Handle invalid or reserved key names + # + + def test_dict_reserved_or_invalid_keynames(self): + """ + Test self.reserved_or_invalid_keynames_sample + """ + data_in = self.reserved_or_invalid_keynames_sample + expected = '_ = {};\n_["True"] = 1;\n_["get"] = 1;\n_["fromkeys"] = 1;\n_["aquaero-hid-3-1cb1"] = 1;\n_["if"] = 1;\n_["return"] = 1;\n_["__class__"] = 1;' + output = self.schema.create_schema(data_in) + self.assertEqual(output, expected) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_load_json.py b/tests/test_load_json.py new file mode 100644 index 0000000..270f737 --- /dev/null +++ b/tests/test_load_json.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import unittest +from jello.lib import load_json + + +class MyTests(unittest.TestCase): + pretty_json = '''\ +{ + "foo": 1, + "bar": 2, + "baz": 3 +}''' + + compact_json = '''{"foo": 1,"bar": 2,"baz": 3}''' + + json_lines = '''\ +{"foo": 1,"bar": 2,"baz": 3} +{"foo": 4,"bar": 5,"baz": 6} +{"foo": 7,"bar": 8,"baz": 9}''' + + json_lines_extra_spaces = '''\ + +{"foo": 1,"bar": 2,"baz": 3} + +{"foo": 4,"bar": 5,"baz": 6} + +{"foo": 7,"bar": 8,"baz": 9} + + ''' + + def test_load_pretty_json(self): + """ + Test with pretty JSON + """ + expected = {'foo': 1, 'bar': 2, 'baz': 3} + self.assertEqual(load_json(self.pretty_json), expected) + + def test_load_compact_json(self): + """ + Test with compact JSON + """ + expected = {'foo': 1, 'bar': 2, 'baz': 3} + self.assertEqual(load_json(self.compact_json), expected) + + def test_load_json_lines(self): + """ + Test with JSON Lines + """ + expected = [{'foo': 1, 'bar': 2, 'baz': 3}, {'foo': 4, 'bar': 5, 'baz': 6}, {'foo': 7, 'bar': 8, 'baz': 9}] + self.assertEqual(load_json(self.json_lines), expected) + + def test_load_json_lines_with_extra_whitespace(self): + """ + Test with JSON Lines with extra blank lines and whitespace + """ + expected = [{'foo': 1, 'bar': 2, 'baz': 3}, {'foo': 4, 'bar': 5, 'baz': 6}, {'foo': 7, 'bar': 8, 'baz': 9}] + self.assertEqual(load_json(self.json_lines_extra_spaces), expected) + + +if __name__ == '__main__': + unittest.main()