From a0e669aad9f2f5b43fa3259d639c5ba3b03f5cc9 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Wed, 20 Nov 2019 06:38:57 -0600 Subject: [PATCH] Introduce RegularArray for rectilinear blocks of any type of awkward array. (#23) This PR only implements `RegularArray` in C++ and Numba, with all the `__getitem__` cases. The next PR will actually use it in inflating arrays for `__getitem__` of `tuple` and for `newaxis`. We should also have some actual tests for `__getitem__` of `Ellipsis` and `newaxis`. * Fewest tests in matrix and rename 'test-requirements.txt' to 'requirements-test.txt'. * Infrastructure for RegularArray. * Minimal implementation of RegularArray; can print it out. * Checkpoint before replacing RegularArray/RegularType into TileArray/TileType. * Changed my mind about renaming. Switched RegularType to 'size' rather than 'shape'. * Switched RegularArray to 'size' rather than 'shape'. * RegularArray::getitem_at. * Renamed 'unsafe' methods to 'nowrap' and 'check_for_iteration' (more descriptive). * RegularArray::getitem_range. * RegularArray::getitem_next of null and at. * RegularArray::getitem_next of range (advanced and non-advanced). * RegularArray::getitem_next of array (non-advanced). * RegularArray::getitem_next of array (advanced). * Consolidated dynamic_casts of getitem so it will be easier to update. * Gave NumpyArray::getitem_strides and getitem_next the same treatment: dynamic_casts are consolidated. * Working on test_numpy. * test_numpy completely works. * First RegularArray::getitem_next loop moved to cpu-kernels. * RegularArray::getitem_next(range) loops moved to cpu-kernels. * RegularArray::getitem_next(array) loops moved to cpu-kernels; now these are all done. * RegularArray::setid written, compiles, but untested. * Testing RegularArray::setid; doesn't work yet. * RegularArray::setid works now. * Prepared for Numbafication of RegularArray. * Boxing and unboxing RegularArray. * RegularArray's lower_len works. * RegularArray's lower_getitem_int works. * RegularArray's lower_getitem_range works. * Fix 32-bit error in lower_getitem_range. * Forgot to externalize awkward_regulararray_getitem_carry_64. * Fixed a bug in lower_getitem_range (don't multiply the step size). * RegularArray lower_getitem_next(int) works. * [skip ci] RegularArray lower_getitem_next(range, not advanced) works. * RegularArray lower_getitem_next(range) works. * RegularArray lower_getitem_next(array, not advanced) works; also fixed 32-bit bug. * RegularArray lower_getitem_next(array, advanced) works. * Try to fix 32-bit again. * RegularArray (and this PR) is done; the next PR will apply it in various places. --- .ci/azure-buildtest-awkward.yml | 34 +- MANIFEST.in | 2 +- README.md | 2 +- VERSION_INFO | 2 +- awkward1/_numba/__init__.py | 3 +- .../_numba/array/{empty.py => emptyarray.py} | 0 awkward1/_numba/array/regulararray.py | 415 ++++++++++++++ awkward1/operations/describe.py | 7 +- awkward1/signatures/Content_8cpp.xml | 6 + awkward1/signatures/EmptyArray_8cpp.xml | 48 ++ awkward1/signatures/ListArray_8cpp.xml | 12 +- awkward1/signatures/ListOffsetArray_8cpp.xml | 16 +- awkward1/signatures/NumpyArray_8cpp.xml | 26 +- awkward1/signatures/RegularArray_8cpp.xml | 60 ++ .../signatures/cpu-kernels_2util_8cpp.xml | 4 +- awkward1/signatures/getitem_8cpp.xml | 513 ++++++++++++++++- awkward1/signatures/identity_8cpp.xml | 136 ++++- include/awkward/Content.h | 20 +- include/awkward/Identity.h | 8 +- include/awkward/Index.h | 6 +- include/awkward/Iterator.h | 2 +- include/awkward/array/EmptyArray.h | 14 +- include/awkward/array/ListArray.h | 14 +- include/awkward/array/ListOffsetArray.h | 14 +- include/awkward/array/NumpyArray.h | 29 +- include/awkward/array/RawArray.h | 136 ++--- include/awkward/array/RegularArray.h | 55 ++ include/awkward/cpu-kernels/getitem.h | 8 + include/awkward/cpu-kernels/identity.h | 5 +- include/awkward/fillable/GrowableBuffer.h | 2 +- include/awkward/type/ArrayType.h | 4 +- include/awkward/type/RegularType.h | 6 +- ...-requirements.txt => requirements-test.txt | 0 setup.py | 2 +- src/cpu-kernels/getitem.cpp | 105 +++- src/cpu-kernels/identity.cpp | 26 +- src/libawkward/Content.cpp | 34 +- src/libawkward/Identity.cpp | 10 +- src/libawkward/Index.cpp | 16 +- src/libawkward/Iterator.cpp | 2 +- src/libawkward/Slice.cpp | 6 +- src/libawkward/array/EmptyArray.cpp | 56 +- src/libawkward/array/ListArray.cpp | 314 +++++----- src/libawkward/array/ListOffsetArray.cpp | 313 +++++----- src/libawkward/array/NumpyArray.cpp | 534 ++++++++++-------- src/libawkward/array/RegularArray.cpp | 291 ++++++++++ src/libawkward/fillable/FillableArray.cpp | 2 +- src/libawkward/io/root.cpp | 10 +- src/libawkward/type/ArrayType.cpp | 2 +- src/libawkward/type/RegularType.cpp | 23 +- src/pyawkward.cpp | 33 +- ...test_PR016_finish_getitem_for_rawarray.cpp | 4 +- tests/test_PR021_emptyarray.py | 20 +- tests/test_PR023_regular_array.py | 291 ++++++++++ 54 files changed, 2848 insertions(+), 855 deletions(-) rename awkward1/_numba/array/{empty.py => emptyarray.py} (100%) create mode 100644 awkward1/_numba/array/regulararray.py create mode 100644 awkward1/signatures/EmptyArray_8cpp.xml create mode 100644 awkward1/signatures/RegularArray_8cpp.xml create mode 100644 include/awkward/array/RegularArray.h rename test-requirements.txt => requirements-test.txt (100%) create mode 100644 src/libawkward/array/RegularArray.cpp create mode 100644 tests/test_PR023_regular_array.py diff --git a/.ci/azure-buildtest-awkward.yml b/.ci/azure-buildtest-awkward.yml index 3f763f9a00..6d04442c50 100644 --- a/.ci/azure-buildtest-awkward.yml +++ b/.ci/azure-buildtest-awkward.yml @@ -33,26 +33,14 @@ jobs: python.version: "2.7" python.architecture: "x64" numpy.version: "1.16.5" - "py35-32bit": - python.version: "3.5" - python.architecture: "x86" - numpy.version: "latest" "py35-64bit": python.version: "3.5" python.architecture: "x64" numpy.version: "latest" - "py36-32bit": - python.version: "3.6" - python.architecture: "x86" - numpy.version: "latest" "py36-64bit": python.version: "3.6" python.architecture: "x64" numpy.version: "latest" - "py37-32bit": - python.version: "3.7" - python.architecture: "x86" - numpy.version: "latest" "py37-64bit": python.version: "3.7" python.architecture: "x64" @@ -79,7 +67,7 @@ jobs: - script: | python -m pip install --upgrade pip python -m pip install -r requirements.txt - python -m pip install -r test-requirements.txt + python -m pip install -r requirements-test.txt python -c "import numpy; print(numpy.__version__)" displayName: "Install" @@ -129,7 +117,7 @@ jobs: - script: | python -m pip install --upgrade pip python -m pip install -r requirements.txt - python -m pip install -r test-requirements.txt + python -m pip install -r requirements-test.txt python -c "import numpy; print(numpy.__version__)" displayName: "Install" @@ -157,34 +145,18 @@ jobs: python.version: "3.4" python.architecture: "x64" numpy.version: "1.13.1" - "py34-np16": - python.version: "3.4" - python.architecture: "x64" - numpy.version: "1.16.5" "py35-np13": python.version: "3.5" python.architecture: "x64" numpy.version: "1.13.1" - "py35-np*": - python.version: "3.5" - python.architecture: "x64" - numpy.version: "latest" "py36-np14": python.version: "3.6" python.architecture: "x64" numpy.version: "1.14.5" - "py36-np*": - python.version: "3.6" - python.architecture: "x64" - numpy.version: "latest" "py37-np14": python.version: "3.7" python.architecture: "x64" numpy.version: "1.14.5" - "py37-np*": - python.version: "3.7" - python.architecture: "x64" - numpy.version: "latest" "py38-np*": python.version: "3.8" python.architecture: "x64" @@ -208,7 +180,7 @@ jobs: sed 's/numpy>=1.13.1/numpy==$(numpy.version)/' requirements.txt > requirements2.txt; python -m pip install -r requirements2.txt; fi - python -m pip install -r test-requirements.txt + python -m pip install -r requirements-test.txt python -c "import numpy; print(numpy.__version__)" displayName: "Install" diff --git a/MANIFEST.in b/MANIFEST.in index 5293371a44..2234d217e2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include requirements.txt test-requirements.txt VERSION_INFO CMakeLists.txt LICENSE README.md +include requirements.txt requirements-test.txt VERSION_INFO CMakeLists.txt LICENSE README.md recursive-include include *.h recursive-include src *.cpp recursive-include awkward1 *.py diff --git a/README.md b/README.md index 0436af5bc1..f5b216a9ac 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit- * [X] `NumpyArray`: rectilinear, N-dimensional array type without Python/pybind11 dependencies, but intended for Numpy. * [X] `ListArray`: the new `JaggedArray`, based on `starts` and `stops` (i.e. fully general). * [X] `ListOffsetArray`: the `JaggedArray` case with no unreachable data between reachable data (gaps). - * [ ] `RegularArray`: rectilinear, N-dimensional array of arbitrary contents, for putting jagged dimensions inside fixed dimensions (for example). + * [X] `RegularArray`: for building rectilinear, N-dimensional arrays of arbitrary contents, e.g. putting jagged dimensions inside fixed dimensions. * [ ] `RecordArray`: the new `Table` _without_ lazy-slicing. * [ ] `MaskedArray`, `BitMaskedArray`, `IndexedMaskedArray`: same as the old versions. * [ ] `UnionArray`: same as the old version; `SparseUnionArray`: the additional case found in Apache Arrow. diff --git a/VERSION_INFO b/VERSION_INFO index 7e72641b4b..001d752878 100644 --- a/VERSION_INFO +++ b/VERSION_INFO @@ -1 +1 @@ -0.1.22 +0.1.23 diff --git a/awkward1/_numba/__init__.py b/awkward1/_numba/__init__.py index 19e7293d00..e4037026dc 100644 --- a/awkward1/_numba/__init__.py +++ b/awkward1/_numba/__init__.py @@ -16,4 +16,5 @@ import awkward1._numba.array.numpyarray import awkward1._numba.array.listarray import awkward1._numba.array.listoffsetarray - import awkward1._numba.array.empty + import awkward1._numba.array.emptyarray + import awkward1._numba.array.regulararray diff --git a/awkward1/_numba/array/empty.py b/awkward1/_numba/array/emptyarray.py similarity index 100% rename from awkward1/_numba/array/empty.py rename to awkward1/_numba/array/emptyarray.py diff --git a/awkward1/_numba/array/regulararray.py b/awkward1/_numba/array/regulararray.py new file mode 100644 index 0000000000..6369e7e829 --- /dev/null +++ b/awkward1/_numba/array/regulararray.py @@ -0,0 +1,415 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import operator + +import numpy +import numba +import numba.typing.arraydecl + +import awkward1.layout +from ..._numba import cpu, util, content + +@numba.extending.typeof_impl.register(awkward1.layout.RegularArray) +def typeof(val, c): + return RegularArrayType(numba.typeof(val.content), numba.typeof(val.id)) + +class RegularArrayType(content.ContentType): + def __init__(self, contenttpe, idtpe): + super(RegularArrayType, self).__init__(name="RegularArrayType({}, id={})".format(contenttpe.name, idtpe.name)) + self.contenttpe = contenttpe + self.idtpe = idtpe + + @property + def ndim(self): + return 1 + self.contenttpe.ndim + + def getitem_int(self): + return self.contenttpe + + def getitem_range(self): + return self + + def getitem_tuple(self, wheretpe): + nexttpe = RegularArrayType(self, numba.none) + out = nexttpe.getitem_next(wheretpe, False) + return out.getitem_int() + + def getitem_next(self, wheretpe, isadvanced): + if len(wheretpe.types) == 0: + return self + headtpe = wheretpe.types[0] + tailtpe = numba.types.Tuple(wheretpe.types[1:]) + + if isinstance(headtpe, numba.types.Integer): + return self.contenttpe.carry().getitem_next(tailtpe, isadvanced) + + elif isinstance(headtpe, numba.types.SliceType): + contenttpe = self.contenttpe.carry().getitem_next(tailtpe, isadvanced) + return RegularArrayType(contenttpe, self.idtpe) + + elif isinstance(headtpe, numba.types.EllipsisType): + raise NotImplementedError("ellipsis") + + elif isinstance(headtpe, type(numba.typeof(numpy.newaxis))): + raise NotImplementedError("newaxis") + + elif isinstance(headtpe, numba.types.Array): + if headtpe.ndim != 1: + raise NotImplementedError("array.ndim != 1") + contenttpe = self.contenttpe.carry().getitem_next(tailtpe, True) + if not isadvanced: + return RegularArrayType(contenttpe, self.idtpe) + else: + return contenttpe + + else: + raise AssertionError(headtpe) + + def carry(self): + return RegularArrayType(self.contenttpe.carry(), self.idtpe) + + @property + def lower_len(self): + return lower_len + + @property + def lower_getitem_int(self): + return lower_getitem_int + + @property + def lower_getitem_range(self): + return lower_getitem_range + + @property + def lower_getitem_next(self): + return lower_getitem_next + + @property + def lower_carry(self): + return lower_carry + +@numba.extending.register_model(RegularArrayType) +class RegularArrayModel(numba.datamodel.models.StructModel): + def __init__(self, dmm, fe_type): + members = [("content", fe_type.contenttpe), + ("size", numba.int64)] + if fe_type.idtpe != numba.none: + members.append(("id", fe_type.idtpe)) + super(RegularArrayModel, self).__init__(dmm, fe_type, members) + +@numba.extending.unbox(RegularArrayType) +def unbox(tpe, obj, c): + content_obj = c.pyapi.object_getattr_string(obj, "content") + size_obj = c.pyapi.object_getattr_string(obj, "size") + proxyout = numba.cgutils.create_struct_proxy(tpe)(c.context, c.builder) + proxyout.content = c.pyapi.to_native_value(tpe.contenttpe, content_obj).value + proxyout.size = c.pyapi.to_native_value(numba.int64, size_obj).value + c.pyapi.decref(content_obj) + c.pyapi.decref(size_obj) + if tpe.idtpe != numba.none: + id_obj = c.pyapi.object_getattr_string(obj, "id") + proxyout.id = c.pyapi.to_native_value(tpe.idtpe, id_obj).value + c.pyapi.decref(id_obj) + is_error = numba.cgutils.is_not_null(c.builder, c.pyapi.err_occurred()) + return numba.extending.NativeValue(proxyout._getvalue(), is_error) + +@numba.extending.box(RegularArrayType) +def box(tpe, val, c): + RegularArray_obj = c.pyapi.unserialize(c.pyapi.serialize_object(awkward1.layout.RegularArray)) + proxyin = numba.cgutils.create_struct_proxy(tpe)(c.context, c.builder, value=val) + content_obj = c.pyapi.from_native_value(tpe.contenttpe, proxyin.content, c.env_manager) + size_obj = c.pyapi.long_from_longlong(proxyin.size) + if tpe.idtpe != numba.none: + id_obj = c.pyapi.from_native_value(tpe.idtpe, proxyin.id, c.env_manager) + out = c.pyapi.call_function_objargs(RegularArray_obj, (content_obj, size_obj, id_obj)) + c.pyapi.decref(id_obj) + else: + out = c.pyapi.call_function_objargs(RegularArray_obj, (content_obj, size_obj)) + c.pyapi.decref(RegularArray_obj) + c.pyapi.decref(content_obj) + c.pyapi.decref(size_obj) + return out + +@numba.extending.lower_builtin(len, RegularArrayType) +def lower_len(context, builder, sig, args): + rettpe, (tpe,) = sig.return_type, sig.args + val, = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + innerlen = tpe.contenttpe.lower_len(context, builder, rettpe(tpe.contenttpe), (proxyin.content,)) + size = util.cast(context, builder, numba.int64, numba.intp, proxyin.size) + return builder.sdiv(innerlen, size) + +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.Integer) +def lower_getitem_int(context, builder, sig, args): + rettpe, (tpe, wheretpe) = sig.return_type, sig.args + val, whereval = args + whereval, wheretpe = util.cast(context, builder, wheretpe, numba.int64, whereval), numba.int64 + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + + start = builder.mul(whereval, proxyin.size) + stop = builder.mul(builder.add(whereval, context.get_constant(numba.int64, 1)), proxyin.size) + proxyslice = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder) + proxyslice.start = util.cast(context, builder, numba.int64, numba.intp, start) + proxyslice.stop = util.cast(context, builder, numba.int64, numba.intp, stop) + proxyslice.step = context.get_constant(numba.intp, 1) + + outtpe = tpe.contenttpe.getitem_range() + return tpe.contenttpe.lower_getitem_range(context, builder, outtpe(tpe.contenttpe, numba.types.slice2_type), (proxyin.content, proxyslice._getvalue())) + +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.slice2_type) +def lower_getitem_range(context, builder, sig, args): + import awkward1._numba.identity + + rettpe, (tpe, wheretpe) = sig.return_type, sig.args + val, whereval = args + + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + size = util.cast(context, builder, numba.int64, numba.intp, proxyin.size) + + proxyslicein = context.make_helper(builder, wheretpe, value=whereval) + numba.targets.slicing.guard_invalid_slice(context, builder, wheretpe, proxyslicein) + numba.targets.slicing.fix_slice(builder, proxyslicein, util.arraylen(context, builder, tpe, val)) + + proxysliceout = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder) + proxysliceout.start = builder.mul(proxyslicein.start, size) + proxysliceout.stop = builder.mul(proxyslicein.stop, size) + proxysliceout.step = context.get_constant(numba.intp, 1) + + proxyout = numba.cgutils.create_struct_proxy(tpe)(context, builder) + proxyout.content = tpe.contenttpe.lower_getitem_range(context, builder, rettpe.contenttpe(tpe.contenttpe, numba.types.slice2_type), (proxyin.content, proxysliceout._getvalue())) + proxyout.size = proxyin.size + if tpe.idtpe != numba.none: + proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, tpe.idtpe, wheretpe, proxyin.id, whereval) + + out = proxyout._getvalue() + if context.enable_nrt: + context.nrt.incref(builder, rettpe, out) + return out + +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.BaseTuple) +def lower_getitem_tuple(context, builder, sig, args): + rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args + arrayval, whereval = args + + wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval) + nexttpe, nextval = util.wrap_for_slicetuple(context, builder, arraytpe, arrayval) + + outtpe = nexttpe.getitem_next(wheretpe, False) + outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None) + + return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0))) + +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.Array) +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.List) +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.ArrayCompatible) +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.EllipsisType) +@numba.extending.lower_builtin(operator.getitem, RegularArrayType, type(numba.typeof(numpy.newaxis))) +def lower_getitem_other(context, builder, sig, args): + rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args + arrayval, whereval = args + wrappedtpe = numba.types.Tuple((wheretpe,)) + wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,)) + return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval)) + +def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, advanced): + if len(wheretpe.types) == 0: + return arrayval + + headtpe = wheretpe.types[0] + tailtpe = numba.types.Tuple(wheretpe.types[1:]) + headval = numba.cgutils.unpack_tuple(builder, whereval)[0] + tailval = context.make_tuple(builder, tailtpe, numba.cgutils.unpack_tuple(builder, whereval)[1:]) + + proxyin = numba.cgutils.create_struct_proxy(arraytpe)(context, builder, value=arrayval) + leng = util.arraylen(context, builder, arraytpe, arrayval, totpe=numba.int64) + + if isinstance(headtpe, numba.types.Integer): + assert advanced is None + nextcarry = util.newindex64(context, builder, numba.int64, leng) + util.call(context, builder, cpu.kernels.awkward_regulararray_getitem_next_at_64, + (util.arrayptr(context, builder, util.index64tpe, nextcarry), + util.cast(context, builder, headtpe, numba.int64, headval), + leng, + proxyin.size), + "in {}, indexing error".format(arraytpe.shortname)) + nextcontenttpe = arraytpe.contenttpe.carry() + nextcontentval = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry) + return nextcontenttpe.lower_getitem_next(context, builder, nextcontenttpe, tailtpe, nextcontentval, tailval, advanced) + + elif isinstance(headtpe, numba.types.SliceType): + proxyslicein = context.make_helper(builder, headtpe, value=headval) + numba.targets.slicing.guard_invalid_slice(context, builder, headtpe, proxyslicein) + numba.targets.slicing.fix_slice(builder, proxyslicein, util.cast(context, builder, numba.int64, numba.intp, proxyin.size)) + nextsize = util.cast(context, builder, numba.intp, numba.int64, numba.targets.slicing.get_slice_length(builder, proxyslicein)) + + nextcarry = util.newindex64(context, builder, numba.int64, builder.mul(leng, nextsize)) + util.call(context, builder, cpu.kernels.awkward_regulararray_getitem_next_range_64, + (util.arrayptr(context, builder, util.index64tpe, nextcarry), + util.cast(context, builder, numba.intp, numba.int64, proxyslicein.start), + util.cast(context, builder, numba.intp, numba.int64, proxyslicein.step), + leng, + proxyin.size, + nextsize), + "in {}, indexing error".format(arraytpe.shortname)) + + nextcontenttpe = arraytpe.contenttpe.carry() + nextcontentval = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry) + + if advanced is None: + outcontenttpe = nextcontenttpe.getitem_next(tailtpe, False) + outcontentval = nextcontenttpe.lower_getitem_next(context, builder, nextcontenttpe, tailtpe, nextcontentval, tailval, advanced) + + else: + nextadvanced = util.newindex64(context, builder, numba.int64, builder.mul(leng, nextsize)) + util.call(context, builder, cpu.kernels.awkward_regulararray_getitem_next_range_spreadadvanced_64, + (util.arrayptr(context, builder, util.index64tpe, nextadvanced), + util.arrayptr(context, builder, util.index64tpe, advanced), + leng, + nextsize), + "in {}, indexing error".format(arraytpe.shortname)) + + outcontenttpe = nextcontenttpe.getitem_next(tailtpe, True) + outcontentval = nextcontenttpe.lower_getitem_next(context, builder, nextcontenttpe, tailtpe, nextcontentval, tailval, nextadvanced) + + outtpe = RegularArrayType(outcontenttpe, arraytpe.idtpe) + proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder) + proxyout.content = outcontentval + proxyout.size = nextsize + if arraytpe.idtpe != numba.none: + proxyout.id = proxyin.id + return proxyout._getvalue() + + elif isinstance(headtpe, numba.types.EllipsisType): + raise NotImplementedError("RegularArray.getitem_next(ellipsis)") + + elif isinstance(headtpe, type(numba.typeof(numpy.newaxis))): + raise NotImplementedError("RegularArray.getitem_next(newaxis)") + + elif isinstance(headtpe, numba.types.Array): + if headtpe.ndim != 1: + raise NotImplementedError("array.ndim != 1") + + flathead = numba.targets.arrayobj.array_ravel(context, builder, util.index64tpe(headtpe), (headval,)) + lenflathead = util.arraylen(context, builder, util.index64tpe, flathead, totpe=numba.int64) + regular_flathead = util.newindex64(context, builder, numba.int64, lenflathead) + + util.call(context, builder, cpu.kernels.awkward_regulararray_getitem_next_array_regularize_64, + (util.arrayptr(context, builder, util.index64tpe, regular_flathead), + util.arrayptr(context, builder, util.index64tpe, flathead), + lenflathead, + proxyin.size), + "in {}, indexing error".format(arraytpe.shortname)) + + if advanced is None: + lencarry = builder.mul(leng, lenflathead) + nextcarry = util.newindex64(context, builder, numba.int64, lencarry) + nextadvanced = util.newindex64(context, builder, numba.int64, lencarry) + + util.call(context, builder, cpu.kernels.awkward_regulararray_getitem_next_array_64, + (util.arrayptr(context, builder, util.index64tpe, nextcarry), + util.arrayptr(context, builder, util.index64tpe, nextadvanced), + util.arrayptr(context, builder, util.index64tpe, regular_flathead), + leng, + lenflathead, + proxyin.size), + "in {}, indexing error".format(arraytpe.shortname)) + + nexttpe = arraytpe.contenttpe.carry() + nextval = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry) + + contenttpe = nexttpe.getitem_next(tailtpe, True) + contentval = nexttpe.lower_getitem_next(context, builder, nexttpe, tailtpe, nextval, tailval, nextadvanced) + + outtpe = RegularArrayType(contenttpe, arraytpe.idtpe) + proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder) + proxyout.content = contentval + proxyout.size = lenflathead + if outtpe.idtpe != numba.none: + proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, outtpe.idtpe, util.index64tpe, proxyin.id, flathead) + return proxyout._getvalue() + + else: + nextcarry = util.newindex64(context, builder, numba.int64, leng) + nextadvanced = util.newindex64(context, builder, numba.int64, leng) + + util.call(context, builder, cpu.kernels.awkward_regulararray_getitem_next_array_advanced_64, + (util.arrayptr(context, builder, util.index64tpe, nextcarry), + util.arrayptr(context, builder, util.index64tpe, nextadvanced), + util.arrayptr(context, builder, util.index64tpe, advanced), + util.arrayptr(context, builder, util.index64tpe, regular_flathead), + leng, + lenflathead, + proxyin.size), + "in {}, indexing error".format(arraytpe.shortname)) + + nexttpe = arraytpe.contenttpe.carry() + nextval = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry) + + outtpe = nexttpe.getitem_next(tailtpe, True) + return nexttpe.lower_getitem_next(context, builder, nexttpe, tailtpe, nextval, tailval, nextadvanced) + + else: + raise AssertionError(headtpe) + +def lower_carry(context, builder, arraytpe, carrytpe, arrayval, carryval): + import awkward1._numba.identity + + proxyin = numba.cgutils.create_struct_proxy(arraytpe)(context, builder, value=arrayval) + + lencarry = util.arraylen(context, builder, carrytpe, carryval, totpe=numba.int64) + nextcarry = util.newindex64(context, builder, numba.int64, builder.mul(lencarry, proxyin.size)) + util.call(context, builder, cpu.kernels.awkward_regulararray_getitem_carry_64, + (util.arrayptr(context, builder, util.index64tpe, nextcarry), + util.arrayptr(context, builder, carrytpe, carryval), + lencarry, + proxyin.size), + "in {}, indexing error".format(arraytpe.shortname)) + nextcontent = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry) + + rettpe = arraytpe.carry() + proxyout = numba.cgutils.create_struct_proxy(rettpe)(context, builder) + proxyout.content = nextcontent + proxyout.size = proxyin.size + if rettpe.idtpe != numba.none: + proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, rettpe.idtpe, carrytpe, proxyin.id, carryval) + + return proxyout._getvalue() + +@numba.typing.templates.infer_getattr +class type_methods(numba.typing.templates.AttributeTemplate): + key = RegularArrayType + + def generic_resolve(self, tpe, attr): + if attr == "content": + return tpe.contenttpe + + elif attr == "size": + return numba.int64 + + elif attr == "id": + if tpe.idtpe == numba.none: + return numba.optional(identity.IdentityType(numba.int32[:, :])) + else: + return tpe.idtpe + +@numba.extending.lower_getattr(RegularArrayType, "content") +def lower_content(context, builder, tpe, val): + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + if context.enable_nrt: + context.nrt.incref(builder, tpe.contenttpe, proxyin.content) + return proxyin.content + +@numba.extending.lower_getattr(RegularArrayType, "size") +def lower_size(context, builder, tpe, val): + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + return proxyin.size + +@numba.extending.lower_getattr(RegularArrayType, "id") +def lower_id(context, builder, tpe, val): + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + if tpe.idtpe == numba.none: + return context.make_optional_none(builder, identity.IdentityType(numba.int32[:, :])) + else: + if context.enable_nrt: + context.nrt.incref(builder, tpe.idtpe, proxyin.id) + return proxyin.id diff --git a/awkward1/operations/describe.py b/awkward1/operations/describe.py index 010b0efbbe..831a9286a2 100644 --- a/awkward1/operations/describe.py +++ b/awkward1/operations/describe.py @@ -28,10 +28,11 @@ def typeof(array): elif isinstance(array, numpy.ndarray): if len(array.shape) == 0: return typeof(array.reshape((1,))[0]) - elif len(array.shape) == 1: - return awkward1.layout.ArrayType(array.shape[0], awkward1.layout.PrimitiveType(typeof.dtype2primitive[array.dtype.type])) else: - return awkward1.layout.ArrayType(array.shape[0], awkward1.layout.RegularType(array.shape[1:], awkward1.layout.PrimitiveType(typeof.dtype2primitive[array.dtype.type]))) + out = awkward1.layout.PrimitiveType(typeof.dtype2primitive[array.dtype.type]) + for x in array.shape[-1:0:-1]: + out = awkward1.layout.RegularType(out, x) + return awkward1.layout.ArrayType(out, array.shape[0]) elif isinstance(array, awkward1.layout.FillableArray): return array.type diff --git a/awkward1/signatures/Content_8cpp.xml b/awkward1/signatures/Content_8cpp.xml index 6df78d0b02..e77ead2724 100644 --- a/awkward1/signatures/Content_8cpp.xml +++ b/awkward1/signatures/Content_8cpp.xml @@ -3,6 +3,7 @@ Content.cpp awkward/array/ListArray.h + awkward/type/ArrayType.h awkward/Content.h @@ -12,8 +13,13 @@ + + + + + diff --git a/awkward1/signatures/EmptyArray_8cpp.xml b/awkward1/signatures/EmptyArray_8cpp.xml new file mode 100644 index 0000000000..01fc92f289 --- /dev/null +++ b/awkward1/signatures/EmptyArray_8cpp.xml @@ -0,0 +1,48 @@ + + + + EmptyArray.cpp + iomanip + sstream + stdexcept + awkward/type/UnknownType.h + awkward/array/EmptyArray.h + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + awkward + + + + + + + diff --git a/awkward1/signatures/ListArray_8cpp.xml b/awkward1/signatures/ListArray_8cpp.xml index de9e0bec3e..c6fe1cf6e0 100644 --- a/awkward1/signatures/ListArray_8cpp.xml +++ b/awkward1/signatures/ListArray_8cpp.xml @@ -6,10 +6,14 @@ type_traits awkward/cpu-kernels/identity.h awkward/cpu-kernels/getitem.h + awkward/type/ListType.h awkward/Slice.h awkward/array/ListOffsetArray.h awkward/array/ListArray.h + + + @@ -30,20 +34,22 @@ + + - + - + - + diff --git a/awkward1/signatures/ListOffsetArray_8cpp.xml b/awkward1/signatures/ListOffsetArray_8cpp.xml index ddf87c5c4b..e006922065 100644 --- a/awkward1/signatures/ListOffsetArray_8cpp.xml +++ b/awkward1/signatures/ListOffsetArray_8cpp.xml @@ -6,10 +6,14 @@ type_traits awkward/cpu-kernels/identity.h awkward/cpu-kernels/getitem.h + awkward/type/ListType.h awkward/Slice.h awkward/array/ListArray.h awkward/array/ListOffsetArray.h + + + @@ -33,17 +37,19 @@ + + - + - + - + @@ -72,7 +78,7 @@ - + @@ -94,7 +100,7 @@ - + diff --git a/awkward1/signatures/NumpyArray_8cpp.xml b/awkward1/signatures/NumpyArray_8cpp.xml index 308a9755f2..70eec7eb4a 100644 --- a/awkward1/signatures/NumpyArray_8cpp.xml +++ b/awkward1/signatures/NumpyArray_8cpp.xml @@ -7,6 +7,8 @@ stdexcept awkward/cpu-kernels/identity.h awkward/cpu-kernels/getitem.h + awkward/type/PrimitiveType.h + awkward/type/RegularType.h awkward/array/NumpyArray.h @@ -27,6 +29,10 @@ + + + + @@ -37,9 +43,15 @@ - + + + + + + + @@ -74,7 +86,7 @@ - + void @@ -99,7 +111,7 @@ - + @@ -129,7 +141,7 @@ - + @@ -159,7 +171,7 @@ - + const std::vector< ssize_t > @@ -176,7 +188,7 @@ - + const std::vector< ssize_t > @@ -193,7 +205,7 @@ - + diff --git a/awkward1/signatures/RegularArray_8cpp.xml b/awkward1/signatures/RegularArray_8cpp.xml new file mode 100644 index 0000000000..042d4140e9 --- /dev/null +++ b/awkward1/signatures/RegularArray_8cpp.xml @@ -0,0 +1,60 @@ + + + + RegularArray.cpp + iomanip + sstream + stdexcept + awkward/cpu-kernels/identity.h + awkward/cpu-kernels/getitem.h + awkward/type/RegularType.h + awkward/array/RegularArray.h + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + awkward + + + + + + + diff --git a/awkward1/signatures/cpu-kernels_2util_8cpp.xml b/awkward1/signatures/cpu-kernels_2util_8cpp.xml index ee1767ce62..24651eabda 100644 --- a/awkward1/signatures/cpu-kernels_2util_8cpp.xml +++ b/awkward1/signatures/cpu-kernels_2util_8cpp.xml @@ -32,7 +32,7 @@ - + struct Error @@ -57,7 +57,7 @@ - + diff --git a/awkward1/signatures/getitem_8cpp.xml b/awkward1/signatures/getitem_8cpp.xml index 1932861a39..2e5cdc8a84 100644 --- a/awkward1/signatures/getitem_8cpp.xml +++ b/awkward1/signatures/getitem_8cpp.xml @@ -2197,7 +2197,7 @@ - + ERROR @@ -2254,7 +2254,7 @@ - + ERROR @@ -2311,7 +2311,7 @@ - + ERROR @@ -2368,7 +2368,7 @@ - + @@ -2425,7 +2425,7 @@ - + ERROR @@ -2474,7 +2474,7 @@ - + ERROR @@ -2523,7 +2523,7 @@ - + ERROR @@ -2572,7 +2572,504 @@ - + + + + + + typename T + + + ERROR + ERROR awkward_regulararray_getitem_next_at + (T *tocarry, int64_t at, int64_t len, int64_t size) + awkward_regulararray_getitem_next_at + + T * + tocarry + + + int64_t + at + + + int64_t + len + + + int64_t + size + + + + + + + + + + + ERROR + ERROR awkward_regulararray_getitem_next_at_64 + (int64_t *tocarry, int64_t at, int64_t len, int64_t size) + awkward_regulararray_getitem_next_at_64 + + int64_t * + tocarry + + + int64_t + at + + + int64_t + len + + + int64_t + size + + + + + + + + + + + + + typename T + + + ERROR + ERROR awkward_regulararray_getitem_next_range + (T *tocarry, int64_t regular_start, int64_t step, int64_t len, int64_t size, int64_t nextsize) + awkward_regulararray_getitem_next_range + + T * + tocarry + + + int64_t + regular_start + + + int64_t + step + + + int64_t + len + + + int64_t + size + + + int64_t + nextsize + + + + + + + + + + + ERROR + ERROR awkward_regulararray_getitem_next_range_64 + (int64_t *tocarry, int64_t regular_start, int64_t step, int64_t len, int64_t size, int64_t nextsize) + awkward_regulararray_getitem_next_range_64 + + int64_t * + tocarry + + + int64_t + regular_start + + + int64_t + step + + + int64_t + len + + + int64_t + size + + + int64_t + nextsize + + + + + + + + + + + + + typename T + + + ERROR + ERROR awkward_regulararray_getitem_next_range_spreadadvanced + (T *toadvanced, const T *fromadvanced, int64_t len, int64_t nextsize) + awkward_regulararray_getitem_next_range_spreadadvanced + + T * + toadvanced + + + const T * + fromadvanced + + + int64_t + len + + + int64_t + nextsize + + + + + + + + + + + ERROR + ERROR awkward_regulararray_getitem_next_range_spreadadvanced_64 + (int64_t *toadvanced, const int64_t *fromadvanced, int64_t len, int64_t nextsize) + awkward_regulararray_getitem_next_range_spreadadvanced_64 + + int64_t * + toadvanced + + + const int64_t * + fromadvanced + + + int64_t + len + + + int64_t + nextsize + + + + + + + + + + + + + typename T + + + ERROR + ERROR awkward_regulararray_getitem_next_array_regularize + (T *toarray, const T *fromarray, int64_t lenarray, int64_t size) + awkward_regulararray_getitem_next_array_regularize + + T * + toarray + + + const T * + fromarray + + + int64_t + lenarray + + + int64_t + size + + + + + + + + + + + ERROR + ERROR awkward_regulararray_getitem_next_array_regularize_64 + (int64_t *toarray, const int64_t *fromarray, int64_t lenarray, int64_t size) + awkward_regulararray_getitem_next_array_regularize_64 + + int64_t * + toarray + + + const int64_t * + fromarray + + + int64_t + lenarray + + + int64_t + size + + + + + + + + + + + + + typename T + + + ERROR + ERROR awkward_regulararray_getitem_next_array + (T *tocarry, T *toadvanced, const T *fromarray, int64_t len, int64_t lenarray, int64_t size) + awkward_regulararray_getitem_next_array + + T * + tocarry + + + T * + toadvanced + + + const T * + fromarray + + + int64_t + len + + + int64_t + lenarray + + + int64_t + size + + + + + + + + + + + ERROR + ERROR awkward_regulararray_getitem_next_array_64 + (int64_t *tocarry, int64_t *toadvanced, const int64_t *fromarray, int64_t len, int64_t lenarray, int64_t size) + awkward_regulararray_getitem_next_array_64 + + int64_t * + tocarry + + + int64_t * + toadvanced + + + const int64_t * + fromarray + + + int64_t + len + + + int64_t + lenarray + + + int64_t + size + + + + + + + + + + + + + typename T + + + ERROR + ERROR awkward_regulararray_getitem_next_array_advanced + (T *tocarry, T *toadvanced, const T *fromadvanced, const T *fromarray, int64_t len, int64_t lenarray, int64_t size) + awkward_regulararray_getitem_next_array_advanced + + T * + tocarry + + + T * + toadvanced + + + const T * + fromadvanced + + + const T * + fromarray + + + int64_t + len + + + int64_t + lenarray + + + int64_t + size + + + + + + + + + + + ERROR + ERROR awkward_regulararray_getitem_next_array_advanced_64 + (int64_t *tocarry, int64_t *toadvanced, const int64_t *fromadvanced, const int64_t *fromarray, int64_t len, int64_t lenarray, int64_t size) + awkward_regulararray_getitem_next_array_advanced_64 + + int64_t * + tocarry + + + int64_t * + toadvanced + + + const int64_t * + fromadvanced + + + const int64_t * + fromarray + + + int64_t + len + + + int64_t + lenarray + + + int64_t + size + + + + + + + + + + + + + typename T + + + ERROR + ERROR awkward_regulararray_getitem_carry + (T *tocarry, const T *fromcarry, int64_t lencarry, int64_t size) + awkward_regulararray_getitem_carry + + T * + tocarry + + + const T * + fromcarry + + + int64_t + lencarry + + + int64_t + size + + + + + + + + + + + ERROR + ERROR awkward_regulararray_getitem_carry_64 + (int64_t *tocarry, const int64_t *fromcarry, int64_t lencarry, int64_t size) + awkward_regulararray_getitem_carry_64 + + int64_t * + tocarry + + + const int64_t * + fromcarry + + + int64_t + lencarry + + + int64_t + size + + + + + + + + diff --git a/awkward1/signatures/identity_8cpp.xml b/awkward1/signatures/identity_8cpp.xml index 9b068899b2..147c54ec7c 100644 --- a/awkward1/signatures/identity_8cpp.xml +++ b/awkward1/signatures/identity_8cpp.xml @@ -83,10 +83,10 @@ - + ERROR ERROR awkward_identity32_to_identity64 - (int64_t *toptr, const int32_t *fromptr, int64_t length) + (int64_t *toptr, const int32_t *fromptr, int64_t length, int64_t width) awkward_identity32_to_identity64 int64_t * @@ -100,6 +100,10 @@ int64_t length + + int64_t + width + @@ -381,6 +385,134 @@ + + + + typename ID + + + ERROR + ERROR awkward_identity_from_regulararray + (ID *toptr, const ID *fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth) + awkward_identity_from_regulararray + + ID * + toptr + + + const ID * + fromptr + + + int64_t + fromptroffset + + + int64_t + size + + + int64_t + tolength + + + int64_t + fromlength + + + int64_t + fromwidth + + + + + + + + + + + ERROR + ERROR awkward_identity32_from_regulararray + (int32_t *toptr, const int32_t *fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth) + awkward_identity32_from_regulararray + + int32_t * + toptr + + + const int32_t * + fromptr + + + int64_t + fromptroffset + + + int64_t + size + + + int64_t + tolength + + + int64_t + fromlength + + + int64_t + fromwidth + + + + + + + + + + + ERROR + ERROR awkward_identity64_from_regulararray + (int64_t *toptr, const int64_t *fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth) + awkward_identity64_from_regulararray + + int64_t * + toptr + + + const int64_t * + fromptr + + + int64_t + fromptroffset + + + int64_t + size + + + int64_t + tolength + + + int64_t + fromlength + + + int64_t + fromwidth + + + + + + + + + diff --git a/include/awkward/Content.h b/include/awkward/Content.h index 21ee38b99b..1c4e8edc7d 100644 --- a/include/awkward/Content.h +++ b/include/awkward/Content.h @@ -22,16 +22,16 @@ namespace awkward { virtual void setid(const std::shared_ptr id) = 0; virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const = 0; virtual void tojson_part(ToJson& builder) const = 0; - virtual std::shared_ptr type_part() const = 0; + virtual const std::shared_ptr type_part() const = 0; virtual int64_t length() const = 0; virtual const std::shared_ptr shallow_copy() const = 0; - virtual void checksafe() const = 0; + virtual void check_for_iteration() const = 0; virtual const std::shared_ptr getitem_at(int64_t at) const = 0; - virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const = 0; + virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const = 0; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const = 0; - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const = 0; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const = 0; virtual const std::shared_ptr getitem(const Slice& where) const; - virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const = 0; + virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; virtual const std::shared_ptr carry(const Index64& carry) const = 0; virtual const std::pair minmax_depth() const = 0; @@ -39,8 +39,14 @@ namespace awkward { const std::string tostring() const; const std::string tojson(bool pretty, int64_t maxdecimals) const; void tojson(FILE* destination, bool pretty, int64_t maxdecimals, int64_t buffersize) const; - const std::shared_ptr getitem_ellipsis(const Slice& tail, const Index64& advanced) const; - const std::shared_ptr getitem_newaxis(const Slice& tail, const Index64& advanced) const; + + protected: + virtual const std::shared_ptr getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const = 0; + virtual const std::shared_ptr getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const = 0; + virtual const std::shared_ptr getitem_next(const SliceEllipsis& ellipsis, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceNewAxis& newaxis, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const = 0; + }; } diff --git a/include/awkward/Identity.h b/include/awkward/Identity.h index c2c2046bf3..f751f84b04 100644 --- a/include/awkward/Identity.h +++ b/include/awkward/Identity.h @@ -36,7 +36,7 @@ namespace awkward { virtual const std::string location(int64_t where) const = 0; virtual const std::shared_ptr to64() const = 0; virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const = 0; - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const = 0; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const = 0; virtual const std::shared_ptr shallow_copy() const = 0; virtual const std::shared_ptr getitem_carry_64(const Index64& carry) const = 0; @@ -55,7 +55,7 @@ namespace awkward { public: IdentityOf(const Ref ref, const FieldLoc fieldloc, int64_t width, int64_t length) : Identity(ref, fieldloc, 0, width, length) - , ptr_(std::shared_ptr(new T[(size_t)(length*width)])) { } + , ptr_(std::shared_ptr(length*width == 0 ? nullptr : new T[(size_t)(length*width)], awkward::util::array_deleter())) { } IdentityOf(const Ref ref, const FieldLoc fieldloc, int64_t offset, int64_t width, int64_t length, const std::shared_ptr ptr) : Identity(ref, fieldloc, offset, width, length) , ptr_(ptr) { } @@ -66,12 +66,12 @@ namespace awkward { virtual const std::string location(int64_t where) const; virtual const std::shared_ptr to64() const; virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const; virtual const std::shared_ptr shallow_copy() const; virtual const std::shared_ptr getitem_carry_64(const Index64& carry) const; const std::vector getitem_at(int64_t at) const; - const std::vector getitem_at_unsafe(int64_t at) const; + const std::vector getitem_at_nowrap(int64_t at) const; const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; private: diff --git a/include/awkward/Index.h b/include/awkward/Index.h index 9c69e79937..40f7a59c4f 100644 --- a/include/awkward/Index.h +++ b/include/awkward/Index.h @@ -34,10 +34,10 @@ namespace awkward { const std::string tostring() const; const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; T getitem_at(int64_t at) const; - T getitem_at_unsafe(int64_t at) const; - void setitem_at_unsafe(int64_t at, T value) const; + T getitem_at_nowrap(int64_t at) const; + void setitem_at_nowrap(int64_t at, T value) const; IndexOf getitem_range(int64_t start, int64_t stop) const; - IndexOf getitem_range_unsafe(int64_t start, int64_t stop) const; + IndexOf getitem_range_nowrap(int64_t start, int64_t stop) const; virtual const std::shared_ptr shallow_copy() const; private: diff --git a/include/awkward/Iterator.h b/include/awkward/Iterator.h index 800b8cc11a..235aaac19c 100644 --- a/include/awkward/Iterator.h +++ b/include/awkward/Iterator.h @@ -12,7 +12,7 @@ namespace awkward { Iterator(const std::shared_ptr content) : content_(content) , where_(0) { - content.get()->checksafe(); + content.get()->check_for_iteration(); } const std::shared_ptr content() const { return content_; } diff --git a/include/awkward/array/EmptyArray.h b/include/awkward/array/EmptyArray.h index 186781fd6d..99f8ba0584 100644 --- a/include/awkward/array/EmptyArray.h +++ b/include/awkward/array/EmptyArray.h @@ -23,18 +23,22 @@ namespace awkward { virtual void setid(const std::shared_ptr id); virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual void tojson_part(ToJson& builder) const; - virtual std::shared_ptr type_part() const; + virtual const std::shared_ptr type_part() const; virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; - virtual void checksafe() const; + virtual void check_for_iteration() const; virtual const std::shared_ptr getitem_at(int64_t at) const; - virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const; + virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; - virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const; virtual const std::shared_ptr carry(const Index64& carry) const; virtual const std::pair minmax_depth() const; + protected: + virtual const std::shared_ptr getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const; + private: std::shared_ptr id_; }; diff --git a/include/awkward/array/ListArray.h b/include/awkward/array/ListArray.h index b7ec1b2c79..86330d2391 100644 --- a/include/awkward/array/ListArray.h +++ b/include/awkward/array/ListArray.h @@ -30,18 +30,22 @@ namespace awkward { virtual void setid(const std::shared_ptr id); virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual void tojson_part(ToJson& builder) const; - virtual std::shared_ptr type_part() const; + virtual const std::shared_ptr type_part() const; virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; - virtual void checksafe() const; + virtual void check_for_iteration() const; virtual const std::shared_ptr getitem_at(int64_t at) const; - virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const; + virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; - virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const; virtual const std::shared_ptr carry(const Index64& carry) const; virtual const std::pair minmax_depth() const; + protected: + virtual const std::shared_ptr getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const; + private: std::shared_ptr id_; const IndexOf starts_; diff --git a/include/awkward/array/ListOffsetArray.h b/include/awkward/array/ListOffsetArray.h index 187f68d8dc..08f9f5a4d0 100644 --- a/include/awkward/array/ListOffsetArray.h +++ b/include/awkward/array/ListOffsetArray.h @@ -28,18 +28,22 @@ namespace awkward { virtual void setid(const std::shared_ptr id); virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual void tojson_part(ToJson& builder) const; - virtual std::shared_ptr type_part() const; + virtual const std::shared_ptr type_part() const; virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; - virtual void checksafe() const; + virtual void check_for_iteration() const; virtual const std::shared_ptr getitem_at(int64_t at) const; - virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const; + virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; - virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const; virtual const std::shared_ptr carry(const Index64& carry) const; virtual const std::pair minmax_depth() const; + protected: + virtual const std::shared_ptr getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const; + private: std::shared_ptr id_; const IndexOf offsets_; diff --git a/include/awkward/array/NumpyArray.h b/include/awkward/array/NumpyArray.h index 214dd57e6c..5dd2586c60 100644 --- a/include/awkward/array/NumpyArray.h +++ b/include/awkward/array/NumpyArray.h @@ -47,14 +47,14 @@ namespace awkward { virtual void setid(const std::shared_ptr id); virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual void tojson_part(ToJson& builder) const; - virtual std::shared_ptr type_part() const; + virtual const std::shared_ptr type_part() const; virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; - virtual void checksafe() const; + virtual void check_for_iteration() const; virtual const std::shared_ptr getitem_at(int64_t at) const; - virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const; + virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const; virtual const std::shared_ptr getitem(const Slice& where) const; virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; virtual const std::shared_ptr carry(const Index64& carry) const; @@ -63,9 +63,30 @@ namespace awkward { bool iscontiguous() const; void become_contiguous(); const NumpyArray contiguous() const; + + protected: + virtual const std::shared_ptr getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const { + throw std::runtime_error("NumpyArray has its own getitem_next system"); + } + virtual const std::shared_ptr getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const { + throw std::runtime_error("NumpyArray has its own getitem_next system"); + } + virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const { + throw std::runtime_error("NumpyArray has its own getitem_next system"); + } + const NumpyArray contiguous_next(Index64 bytepos) const; const NumpyArray getitem_bystrides(const std::shared_ptr& head, const Slice& tail, int64_t length) const; + const NumpyArray getitem_bystrides(const SliceAt& at, const Slice& tail, int64_t length) const; + const NumpyArray getitem_bystrides(const SliceRange& range, const Slice& tail, int64_t length) const; + const NumpyArray getitem_bystrides(const SliceEllipsis& ellipsis, const Slice& tail, int64_t length) const; + const NumpyArray getitem_bystrides(const SliceNewAxis& newaxis, const Slice& tail, int64_t length) const; const NumpyArray getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const; + const NumpyArray getitem_next(const SliceAt& at, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const; + const NumpyArray getitem_next(const SliceRange& range, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const; + const NumpyArray getitem_next(const SliceEllipsis& ellipsis, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const; + const NumpyArray getitem_next(const SliceNewAxis& newaxis, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const; + const NumpyArray getitem_next(const SliceArray64& array, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const; private: std::shared_ptr id_; diff --git a/include/awkward/array/RawArray.h b/include/awkward/array/RawArray.h index ebe5994aee..76deb72230 100644 --- a/include/awkward/array/RawArray.h +++ b/include/awkward/array/RawArray.h @@ -180,7 +180,7 @@ namespace awkward { } } - virtual std::shared_ptr type_part() const { + virtual const std::shared_ptr type_part() const { if (std::is_same::value) { return std::shared_ptr(new PrimitiveType(PrimitiveType::float64)); } @@ -220,7 +220,7 @@ namespace awkward { virtual const std::shared_ptr shallow_copy() const { return std::shared_ptr(new RawArrayOf(id_, ptr_, offset_, length_, itemsize_)); } - virtual void checksafe() const { + virtual void check_for_iteration() const { if (id_.get() != nullptr && id_.get()->length() < length_) { util::handle_error(failure("len(id) < len(array)", kSliceNone, kSliceNone), id_.get()->classname(), nullptr); } @@ -234,11 +234,11 @@ namespace awkward { if (!(0 <= regular_at && regular_at < length_)) { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); } - return getitem_at_unsafe(regular_at); + return getitem_at_nowrap(regular_at); } - virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const { - return getitem_range_unsafe(at, at + 1); + virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const { + return getitem_range_nowrap(at, at + 1); } virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const { @@ -248,13 +248,13 @@ namespace awkward { if (id_.get() != nullptr && regular_stop > id_.get()->length()) { util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } - return getitem_range_unsafe(regular_start, regular_stop); + return getitem_range_nowrap(regular_start, regular_stop); } - virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const { + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const { std::shared_ptr id(nullptr); if (id_.get() != nullptr) { - id = id_.get()->getitem_range_unsafe(start, stop); + id = id_.get()->getitem_range_nowrap(start, stop); } return std::shared_ptr(new RawArrayOf(id, ptr_, offset_ + start, stop - start, itemsize_)); } @@ -266,76 +266,11 @@ namespace awkward { return getitem_next(nexthead, nexttail, nextadvanced); } - const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { + virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { if (tail.length() != 0) { throw std::invalid_argument("too many indexes for array"); } - - if (head.get() == nullptr) { - return shallow_copy(); - } - - else if (SliceAt* at = dynamic_cast(head.get())) { - return getitem_at(at->at()); - } - - else if (SliceRange* range = dynamic_cast(head.get())) { - if (range->step() == Slice::none() || range->step() == 1) { - return getitem_range(range->start(), range->stop()); - } - else { - int64_t start = range->start(); - int64_t stop = range->stop(); - int64_t step = range->step(); - if (step == Slice::none()) { - step = 1; - } - else if (step == 0) { - throw std::invalid_argument("slice step must not be 0"); - } - awkward_regularize_rangeslice(&start, &stop, step > 0, range->hasstart(), range->hasstop(), length_); - - int64_t numer = abs(start - stop); - int64_t denom = abs(step); - int64_t d = numer / denom; - int64_t m = numer % denom; - int64_t lenhead = d + (m != 0 ? 1 : 0); - - Index64 nextcarry(lenhead); - int64_t* nextcarryptr = nextcarry.ptr().get(); - for (int64_t i = 0; i < lenhead; i++) { - nextcarryptr[i] = start + step*i; - } - - return carry(nextcarry); - } - } - - else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { - throw std::runtime_error("ellipsis"); - } - - else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { - throw std::runtime_error("newaxis"); - } - - else if (SliceArray64* array = dynamic_cast(head.get())) { - assert(advanced.length() == 0); - if (array->shape().size() != 1) { - throw std::runtime_error("array.ndim != 1"); - } - Index64 flathead = array->ravel(); - struct Error err = awkward_regularize_arrayslice_64( - flathead.ptr().get(), - flathead.length(), - length_); - util::handle_error(err, classname(), id_.get()); - return carry(flathead); - } - - else { - throw std::runtime_error("unrecognized slice item type"); - } + return Content::getitem_next(head, tail, advanced); } virtual const std::shared_ptr carry(const Index64& carry) const { @@ -359,6 +294,57 @@ namespace awkward { virtual const std::pair minmax_depth() const { return std::pair(1, 1); } + protected: + virtual const std::shared_ptr getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const { + return getitem_at(at.at()); + } + + virtual const std::shared_ptr getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const { + if (range.step() == Slice::none() || range.step() == 1) { + return getitem_range(range.start(), range.stop()); + } + else { + int64_t start = range.start(); + int64_t stop = range.stop(); + int64_t step = range.step(); + if (step == Slice::none()) { + step = 1; + } + else if (step == 0) { + throw std::invalid_argument("slice step must not be 0"); + } + awkward_regularize_rangeslice(&start, &stop, step > 0, range.hasstart(), range.hasstop(), length_); + + int64_t numer = abs(start - stop); + int64_t denom = abs(step); + int64_t d = numer / denom; + int64_t m = numer % denom; + int64_t lenhead = d + (m != 0 ? 1 : 0); + + Index64 nextcarry(lenhead); + int64_t* nextcarryptr = nextcarry.ptr().get(); + for (int64_t i = 0; i < lenhead; i++) { + nextcarryptr[i] = start + step*i; + } + + return carry(nextcarry); + } + } + + virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const { + assert(advanced.length() == 0); + if (array.shape().size() != 1) { + throw std::runtime_error("array.ndim != 1"); + } + Index64 flathead = array.ravel(); + struct Error err = awkward_regularize_arrayslice_64( + flathead.ptr().get(), + flathead.length(), + length_); + util::handle_error(err, classname(), id_.get()); + return carry(flathead); + } + private: std::shared_ptr id_; const std::shared_ptr ptr_; diff --git a/include/awkward/array/RegularArray.h b/include/awkward/array/RegularArray.h new file mode 100644 index 0000000000..8b54004bfd --- /dev/null +++ b/include/awkward/array/RegularArray.h @@ -0,0 +1,55 @@ +// BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +#ifndef AWKWARD_REGULARARRAY_H_ +#define AWKWARD_REGULARARRAY_H_ + +#include +#include +#include +#include + +#include "awkward/cpu-kernels/util.h" +#include "awkward/Slice.h" +#include "awkward/Content.h" + +namespace awkward { + class RegularArray: public Content { + public: + RegularArray(const std::shared_ptr id, const std::shared_ptr content, int64_t size) + : id_(id) + , content_(content) + , size_(size) { } + + const std::shared_ptr content() const { return content_; } + int64_t size() const { return size_; } + + virtual const std::string classname() const; + virtual const std::shared_ptr id() const { return id_; } + virtual void setid(); + virtual void setid(const std::shared_ptr id); + virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; + virtual void tojson_part(ToJson& builder) const; + virtual const std::shared_ptr type_part() const; + virtual int64_t length() const; + virtual const std::shared_ptr shallow_copy() const; + virtual void check_for_iteration() const; + virtual const std::shared_ptr getitem_at(int64_t at) const; + virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; + virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; + virtual const std::shared_ptr getitem_range_nowrap(int64_t start, int64_t stop) const; + virtual const std::shared_ptr carry(const Index64& carry) const; + virtual const std::pair minmax_depth() const; + + protected: + virtual const std::shared_ptr getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const; + virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const; + + private: + std::shared_ptr id_; + const std::shared_ptr content_; + int64_t size_; + }; +} + +#endif // AWKWARD_REGULARARRAY_H_ diff --git a/include/awkward/cpu-kernels/getitem.h b/include/awkward/cpu-kernels/getitem.h index 95ed22f232..092440a42b 100644 --- a/include/awkward/cpu-kernels/getitem.h +++ b/include/awkward/cpu-kernels/getitem.h @@ -58,6 +58,14 @@ extern "C" { struct Error awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry); struct Error awkward_listarrayU32_getitem_carry_64(uint32_t* tostarts, uint32_t* tostops, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry); struct Error awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry); + + struct Error awkward_regulararray_getitem_next_at_64(int64_t* tocarry, int64_t at, int64_t len, int64_t size); + struct Error awkward_regulararray_getitem_next_range_64(int64_t* tocarry, int64_t regular_start, int64_t step, int64_t len, int64_t size, int64_t nextsize); + struct Error awkward_regulararray_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, int64_t len, int64_t nextsize); + struct Error awkward_regulararray_getitem_next_array_regularize_64(int64_t* toarray, const int64_t* fromarray, int64_t lenarray, int64_t size); + struct Error awkward_regulararray_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromarray, int64_t len, int64_t lenarray, int64_t size); + struct Error awkward_regulararray_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromarray, int64_t len, int64_t lenarray, int64_t size); + struct Error awkward_regulararray_getitem_carry_64(int64_t* tocarry, const int64_t* fromcarry, int64_t lencarry, int64_t size); } #endif // AWKWARDCPU_GETITEM_H_ diff --git a/include/awkward/cpu-kernels/identity.h b/include/awkward/cpu-kernels/identity.h index 25d9ed0276..4066dff4c1 100644 --- a/include/awkward/cpu-kernels/identity.h +++ b/include/awkward/cpu-kernels/identity.h @@ -9,12 +9,15 @@ extern "C" { struct Error awkward_new_identity32(int32_t* toptr, int64_t length); struct Error awkward_new_identity64(int64_t* toptr, int64_t length); - struct Error awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length); + struct Error awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length, int64_t width); struct Error awkward_identity32_from_listarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); struct Error awkward_identity64_from_listarray32(int64_t* toptr, const int64_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); struct Error awkward_identity64_from_listarrayU32(int64_t* toptr, const int64_t* fromptr, const uint32_t* fromstarts, const uint32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); struct Error awkward_identity64_from_listarray64(int64_t* toptr, const int64_t* fromptr, const int64_t* fromstarts, const int64_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); + + struct Error awkward_identity32_from_regulararray(int32_t* toptr, const int32_t* fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth); + struct Error awkward_identity64_from_regulararray(int64_t* toptr, const int64_t* fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth); } #endif // AWKWARDCPU_IDENTITY_H_ diff --git a/include/awkward/fillable/GrowableBuffer.h b/include/awkward/fillable/GrowableBuffer.h index 5faecaa475..ff4d539736 100644 --- a/include/awkward/fillable/GrowableBuffer.h +++ b/include/awkward/fillable/GrowableBuffer.h @@ -88,7 +88,7 @@ namespace awkward { length_++; } - T getitem_at_unsafe(int64_t at) const { + T getitem_at_nowrap(int64_t at) const { return ptr_.get()[at]; } diff --git a/include/awkward/type/ArrayType.h b/include/awkward/type/ArrayType.h index 09117acafb..531dfb70b1 100644 --- a/include/awkward/type/ArrayType.h +++ b/include/awkward/type/ArrayType.h @@ -8,15 +8,15 @@ namespace awkward { class ArrayType: public Type { public: - ArrayType(int64_t length, const std::shared_ptr type): length_(length), type_(type) { } + ArrayType(const std::shared_ptr type, int64_t length): type_(type), length_(length) { } virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; - int64_t length() const; const std::shared_ptr type() const; + int64_t length() const; private: std::shared_ptr type_; diff --git a/include/awkward/type/RegularType.h b/include/awkward/type/RegularType.h index 8eb82778ab..ca4710b5d4 100644 --- a/include/awkward/type/RegularType.h +++ b/include/awkward/type/RegularType.h @@ -10,19 +10,19 @@ namespace awkward { class RegularType: public Type { public: - RegularType(const std::vector shape, const std::shared_ptr type): shape_(shape), type_(type) { } + RegularType(const std::shared_ptr type, int64_t size): type_(type), size_(size) { } virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; - const std::vector shape() const; const std::shared_ptr type() const; + int64_t size() const; private: - const std::vector shape_; const std::shared_ptr type_; + const int64_t size_; }; } diff --git a/test-requirements.txt b/requirements-test.txt similarity index 100% rename from test-requirements.txt rename to requirements-test.txt diff --git a/setup.py b/setup.py index 696ac7fa95..2e32e8a9e3 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ def build_extension(self, ext): test_suite = "tests", python_requires = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", install_requires = open("requirements.txt").read().strip().split(), - tests_require = open("test-requirements.txt").read().strip().split(), + tests_require = open("requirements-test.txt").read().strip().split(), zip_safe = False, classifiers = [ # "Development Status :: 1 - Planning", diff --git a/src/cpu-kernels/getitem.cpp b/src/cpu-kernels/getitem.cpp index 0141cb17e2..4282c253e0 100644 --- a/src/cpu-kernels/getitem.cpp +++ b/src/cpu-kernels/getitem.cpp @@ -274,7 +274,7 @@ ERROR awkward_listarray_getitem_next_range(C* tooffsets, T* tocarry, const C* fr int64_t regular_start = start; int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, step > 0, start != kSliceNone, stop != kSliceNone, length); - if (step > 0) { + if (step > 0) { // FIXME: put this test outside the for loop for (int64_t j = regular_start; j < regular_stop; j += step) { tocarry[k] = fromstarts[startsoffset + i] + j; k++; @@ -385,6 +385,9 @@ ERROR awkward_listarray_getitem_next_array_advanced(T* tocarry, T* toadvanced, c } int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; if (fromadvanced[i] >= lenarray) { + // FIXME: this might be weaker than it should be: the length of each advanced array should + // be exactly the same, and I think it was already checked when creating the Slice. + // If so, this check would be redundant. If not, it's not strong enough. return failure("lengths of advanced indexes must match", i, kSliceNone); } int64_t regular_at = fromarray[fromadvanced[i]]; @@ -429,3 +432,103 @@ ERROR awkward_listarrayU32_getitem_carry_64(uint32_t* tostarts, uint32_t* tostop ERROR awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry) { return awkward_listarray_getitem_carry(tostarts, tostops, fromstarts, fromstops, fromcarry, startsoffset, stopsoffset, lenstarts, lencarry); } + +template +ERROR awkward_regulararray_getitem_next_at(T* tocarry, int64_t at, int64_t len, int64_t size) { + int64_t regular_at = at; + if (regular_at < 0) { + regular_at += size; + } + if (!(0 <= regular_at && regular_at < size)) { + return failure("index out of range", kSliceNone, at); + } + for (int64_t i = 0; i < len; i++) { + tocarry[i] = i*size + regular_at; + } + return success(); +} +ERROR awkward_regulararray_getitem_next_at_64(int64_t* tocarry, int64_t at, int64_t len, int64_t size) { + return awkward_regulararray_getitem_next_at(tocarry, at, len, size); +} + +template +ERROR awkward_regulararray_getitem_next_range(T* tocarry, int64_t regular_start, int64_t step, int64_t len, int64_t size, int64_t nextsize) { + for (int64_t i = 0; i < len; i++) { + for (int64_t j = 0; j < nextsize; j++) { + tocarry[i*nextsize + j] = i*size + regular_start + j*step; + } + } + return success(); +} +ERROR awkward_regulararray_getitem_next_range_64(int64_t* tocarry, int64_t regular_start, int64_t step, int64_t len, int64_t size, int64_t nextsize) { + return awkward_regulararray_getitem_next_range(tocarry, regular_start, step, len, size, nextsize); +} + +template +ERROR awkward_regulararray_getitem_next_range_spreadadvanced(T* toadvanced, const T* fromadvanced, int64_t len, int64_t nextsize) { + for (int64_t i = 0; i < len; i++) { + for (int64_t j = 0; j < nextsize; j++) { + toadvanced[i*nextsize + j] = fromadvanced[i]; + } + } + return success(); +} +ERROR awkward_regulararray_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, int64_t len, int64_t nextsize) { + return awkward_regulararray_getitem_next_range_spreadadvanced(toadvanced, fromadvanced, len, nextsize); +} + +template +ERROR awkward_regulararray_getitem_next_array_regularize(T* toarray, const T* fromarray, int64_t lenarray, int64_t size) { + for (int64_t j = 0; j < lenarray; j++) { + toarray[j] = fromarray[j]; + if (toarray[j] < 0) { + toarray[j] += size; + } + if (!(0 <= toarray[j] && toarray[j] < size)) { + return failure("index out of range", kSliceNone, fromarray[j]); + } + } + return success(); +} +ERROR awkward_regulararray_getitem_next_array_regularize_64(int64_t* toarray, const int64_t* fromarray, int64_t lenarray, int64_t size) { + return awkward_regulararray_getitem_next_array_regularize(toarray, fromarray, lenarray, size); +} + +template +ERROR awkward_regulararray_getitem_next_array(T* tocarry, T* toadvanced, const T* fromarray, int64_t len, int64_t lenarray, int64_t size) { + for (int64_t i = 0; i < len; i++) { + for (int64_t j = 0; j < lenarray; j++) { + tocarry[i*lenarray + j] = i*size + fromarray[j]; + toadvanced[i*lenarray + j] = j; + } + } + return success(); +} +ERROR awkward_regulararray_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromarray, int64_t len, int64_t lenarray, int64_t size) { + return awkward_regulararray_getitem_next_array(tocarry, toadvanced, fromarray, len, lenarray, size); +} + +template +ERROR awkward_regulararray_getitem_next_array_advanced(T* tocarry, T* toadvanced, const T* fromadvanced, const T* fromarray, int64_t len, int64_t lenarray, int64_t size) { + for (int64_t i = 0; i < len; i++) { + tocarry[i] = i*size + fromarray[fromadvanced[i]]; + toadvanced[i] = i; + } + return success(); +} +ERROR awkward_regulararray_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromarray, int64_t len, int64_t lenarray, int64_t size) { + return awkward_regulararray_getitem_next_array_advanced(tocarry, toadvanced, fromadvanced, fromarray, len, lenarray, size); +} + +template +ERROR awkward_regulararray_getitem_carry(T* tocarry, const T* fromcarry, int64_t lencarry, int64_t size) { + for (int64_t i = 0; i < lencarry; i++) { + for (int64_t j = 0; j < size; j++) { + tocarry[i*size + j] = fromcarry[i]*size + j; + } + } + return success(); +} +ERROR awkward_regulararray_getitem_carry_64(int64_t* tocarry, const int64_t* fromcarry, int64_t lencarry, int64_t size) { + return awkward_regulararray_getitem_carry(tocarry, fromcarry, lencarry, size); +} diff --git a/src/cpu-kernels/identity.cpp b/src/cpu-kernels/identity.cpp index 4292442da1..4a3cd7e6f1 100644 --- a/src/cpu-kernels/identity.cpp +++ b/src/cpu-kernels/identity.cpp @@ -16,8 +16,8 @@ ERROR awkward_new_identity64(int64_t* toptr, int64_t length) { return awkward_new_identity(toptr, length); } -ERROR awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length) { - for (int64_t i = 0; i < length; i++) { +ERROR awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length, int64_t width) { + for (int64_t i = 0; i < length*width; i++) { toptr[i]= (int64_t)fromptr[i]; } return success(); @@ -55,3 +55,25 @@ ERROR awkward_identity64_from_listarrayU32(int64_t* toptr, const int64_t* frompt ERROR awkward_identity64_from_listarray64(int64_t* toptr, const int64_t* fromptr, const int64_t* fromstarts, const int64_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { return awkward_identity_from_listarray(toptr, fromptr, fromstarts, fromstops, fromptroffset, startsoffset, stopsoffset, tolength, fromlength, fromwidth); } + +template +ERROR awkward_identity_from_regulararray(ID* toptr, const ID* fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + for (int64_t i = 0; i < fromlength; i++) { + for (int64_t j = 0; j < size; j++) { + for (int64_t k = 0; k < fromwidth; k++) { + toptr[(i*size + j)*(fromwidth + 1) + k] = fromptr[fromptroffset + i*fromwidth + k]; + } + toptr[(i*size + j)*(fromwidth + 1) + fromwidth] = ID(j); + } + } + for (int64_t k = (fromlength + 1)*size*(fromwidth + 1); k < tolength*(fromwidth + 1); k++) { + toptr[k] = -1; + } + return success(); +} +ERROR awkward_identity32_from_regulararray(int32_t* toptr, const int32_t* fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity_from_regulararray(toptr, fromptr, fromptroffset, size, tolength, fromlength, fromwidth); +} +ERROR awkward_identity64_from_regulararray(int64_t* toptr, const int64_t* fromptr, int64_t fromptroffset, int64_t size, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity_from_regulararray(toptr, fromptr, fromptroffset, size, tolength, fromlength, fromwidth); +} diff --git a/src/libawkward/Content.cpp b/src/libawkward/Content.cpp index 15c7a4670d..a6149ce0de 100644 --- a/src/libawkward/Content.cpp +++ b/src/libawkward/Content.cpp @@ -7,7 +7,7 @@ namespace awkward { const ArrayType Content::type() const { - return ArrayType(length(), type_part()); + return ArrayType(type_part(), length()); } const std::string Content::tostring() const { @@ -51,16 +51,40 @@ namespace awkward { Index64 nextstops(1); *nextstarts.ptr().get() = 0; *nextstops.ptr().get() = length(); - ListArrayOf next(std::shared_ptr(nullptr), nextstarts, nextstops, shallow_copy()); + std::shared_ptr next(new ListArrayOf(std::shared_ptr(nullptr), nextstarts, nextstops, shallow_copy())); std::shared_ptr nexthead = where.head(); Slice nexttail = where.tail(); Index64 nextadvanced(0); - std::shared_ptr out = next.getitem_next(nexthead, nexttail, nextadvanced); + std::shared_ptr out = next.get()->getitem_next(nexthead, nexttail, nextadvanced); return out.get()->getitem_at(0); } - const std::shared_ptr Content::getitem_ellipsis(const Slice& tail, const Index64& advanced) const { + const std::shared_ptr Content::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { + if (head.get() == nullptr) { + return shallow_copy(); + } + else if (SliceAt* at = dynamic_cast(head.get())) { + return getitem_next(*at, tail, advanced); + } + else if (SliceRange* range = dynamic_cast(head.get())) { + return getitem_next(*range, tail, advanced); + } + else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { + return getitem_next(*ellipsis, tail, advanced); + } + else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { + return getitem_next(*newaxis, tail, advanced); + } + else if (SliceArray64* array = dynamic_cast(head.get())) { + return getitem_next(*array, tail, advanced); + } + else { + throw std::runtime_error("unrecognized slice type"); + } + } + + const std::shared_ptr Content::getitem_next(const SliceEllipsis& ellipsis, const Slice& tail, const Index64& advanced) const { std::pair minmax = minmax_depth(); int64_t mindepth = minmax.first; int64_t maxdepth = minmax.second; @@ -83,7 +107,7 @@ namespace awkward { } } - const std::shared_ptr Content::getitem_newaxis(const Slice& tail, const Index64& advanced) const { + const std::shared_ptr Content::getitem_next(const SliceNewAxis& newaxis, const Slice& tail, const Index64& advanced) const { throw std::runtime_error("FIXME: insert a RegularArray of 1 here"); } } diff --git a/src/libawkward/Identity.cpp b/src/libawkward/Identity.cpp index ce21dd315c..3167fe0686 100644 --- a/src/libawkward/Identity.cpp +++ b/src/libawkward/Identity.cpp @@ -61,7 +61,7 @@ namespace awkward { else if (std::is_same::value) { Identity64* raw = new Identity64(ref_, fieldloc_, width_, length_); std::shared_ptr out(raw); - awkward_identity32_to_identity64(raw->ptr().get(), reinterpret_cast(ptr_.get()), length_); + awkward_identity32_to_identity64(raw->ptr().get(), reinterpret_cast(ptr_.get()), length_, width_); return out; } } @@ -89,7 +89,7 @@ namespace awkward { } template - const std::shared_ptr IdentityOf::getitem_range_unsafe(int64_t start, int64_t stop) const { + const std::shared_ptr IdentityOf::getitem_range_nowrap(int64_t start, int64_t stop) const { assert(0 <= start && start < length_ && 0 <= stop && stop < length_); return std::shared_ptr(new IdentityOf(ref_, fieldloc_, offset_ + width_*start*(start != stop), width_, (stop - start), ptr_)); } @@ -146,11 +146,11 @@ namespace awkward { if (!(0 <= regular_at && regular_at < length_)) { util::handle_error(failure("index out of range", kSliceNone, at), classname(), nullptr); } - return getitem_at_unsafe(regular_at); + return getitem_at_nowrap(regular_at); } template - const std::vector IdentityOf::getitem_at_unsafe(int64_t at) const { + const std::vector IdentityOf::getitem_at_nowrap(int64_t at) const { assert(0 <= at && at < length_); std::vector out; for (size_t i = (size_t)(offset_ + at); i < (size_t)(offset_ + at + width_); i++) { @@ -164,7 +164,7 @@ namespace awkward { int64_t regular_start = start; int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), length_); - return getitem_range_unsafe(regular_start, regular_stop); + return getitem_range_nowrap(regular_start, regular_stop); } template class IdentityOf; diff --git a/src/libawkward/Index.cpp b/src/libawkward/Index.cpp index 4cc76051b8..46350fa1c5 100644 --- a/src/libawkward/Index.cpp +++ b/src/libawkward/Index.cpp @@ -45,7 +45,7 @@ namespace awkward { if (i != 0) { out << " "; } - out << (int64_t)getitem_at_unsafe(i); + out << (int64_t)getitem_at_nowrap(i); } } else { @@ -53,14 +53,14 @@ namespace awkward { if (i != 0) { out << " "; } - out << (int64_t)getitem_at_unsafe(i); + out << (int64_t)getitem_at_nowrap(i); } out << " ... "; for (int64_t i = length_ - 5; i < length_; i++) { if (i != length_ - 5) { out << " "; } - out << (int64_t)getitem_at_unsafe(i); + out << (int64_t)getitem_at_nowrap(i); } } out << "]\" offset=\"" << offset_ << "\" at=\"0x"; @@ -77,17 +77,17 @@ namespace awkward { if (!(0 <= regular_at && regular_at < length_)) { util::handle_error(failure("index out of range", kSliceNone, at), classname(), nullptr); } - return getitem_at_unsafe(regular_at); + return getitem_at_nowrap(regular_at); } template - T IndexOf::getitem_at_unsafe(int64_t at) const { + T IndexOf::getitem_at_nowrap(int64_t at) const { assert(0 <= at && at < length_); return ptr_.get()[(size_t)(offset_ + at)]; } template - void IndexOf::setitem_at_unsafe(int64_t at, T value) const { + void IndexOf::setitem_at_nowrap(int64_t at, T value) const { assert(0 <= at && at < length_); ptr_.get()[(size_t)(offset_ + at)] = value; } @@ -97,11 +97,11 @@ namespace awkward { int64_t regular_start = start; int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), length_); - return getitem_range_unsafe(regular_start, regular_stop); + return getitem_range_nowrap(regular_start, regular_stop); } template - IndexOf IndexOf::getitem_range_unsafe(int64_t start, int64_t stop) const { + IndexOf IndexOf::getitem_range_nowrap(int64_t start, int64_t stop) const { assert(0 <= start && start < length_ && start <= stop && stop <= length_); return IndexOf(ptr_, offset_ + start*(start != stop), stop - start); } diff --git a/src/libawkward/Iterator.cpp b/src/libawkward/Iterator.cpp index dc1064bc7f..d3d7194782 100644 --- a/src/libawkward/Iterator.cpp +++ b/src/libawkward/Iterator.cpp @@ -10,7 +10,7 @@ namespace awkward { } const std::shared_ptr Iterator::next() { - return content_.get()->getitem_at_unsafe(where_++); + return content_.get()->getitem_at_nowrap(where_++); } const std::string Iterator::tostring_part(const std::string indent, const std::string pre, const std::string post) const { diff --git a/src/libawkward/Slice.cpp b/src/libawkward/Slice.cpp index 9754b34e31..4062225662 100644 --- a/src/libawkward/Slice.cpp +++ b/src/libawkward/Slice.cpp @@ -40,7 +40,7 @@ namespace awkward { if (i != 0) { out << ", "; } - out << (T)index_.getitem_at_unsafe(i*strides_[0]); + out << (T)index_.getitem_at_nowrap(i*strides_[0]); } } else { @@ -48,14 +48,14 @@ namespace awkward { if (i != 0) { out << ", "; } - out << (T)index_.getitem_at_unsafe(i*strides_[0]); + out << (T)index_.getitem_at_nowrap(i*strides_[0]); } out << ", ..., "; for (int64_t i = shape_[0] - 3; i < shape_[0]; i++) { if (i != shape_[0] - 3) { out << ", "; } - out << (T)index_.getitem_at_unsafe(i*strides_[0]); + out << (T)index_.getitem_at_nowrap(i*strides_[0]); } } } diff --git a/src/libawkward/array/EmptyArray.cpp b/src/libawkward/array/EmptyArray.cpp index de4d054d0f..754c2a1575 100644 --- a/src/libawkward/array/EmptyArray.cpp +++ b/src/libawkward/array/EmptyArray.cpp @@ -39,7 +39,7 @@ namespace awkward { builder.endlist(); } - std::shared_ptr EmptyArray::type_part() const { + const std::shared_ptr EmptyArray::type_part() const { return std::shared_ptr(new UnknownType()); } @@ -51,14 +51,14 @@ namespace awkward { return std::shared_ptr(new EmptyArray(id_)); } - void EmptyArray::checksafe() const { } + void EmptyArray::check_for_iteration() const { } const std::shared_ptr EmptyArray::getitem_at(int64_t at) const { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); return std::shared_ptr(nullptr); // make Windows compiler happy } - const std::shared_ptr EmptyArray::getitem_at_unsafe(int64_t at) const { + const std::shared_ptr EmptyArray::getitem_at_nowrap(int64_t at) const { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); return std::shared_ptr(nullptr); // make Windows compiler happy } @@ -67,48 +67,32 @@ namespace awkward { return shallow_copy(); } - const std::shared_ptr EmptyArray::getitem_range_unsafe(int64_t start, int64_t stop) const { + const std::shared_ptr EmptyArray::getitem_range_nowrap(int64_t start, int64_t stop) const { return shallow_copy(); } - const std::shared_ptr EmptyArray::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { - if (head.get() == nullptr) { - return shallow_copy(); - } - - else if (SliceAt* at = dynamic_cast(head.get())) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - return std::shared_ptr(nullptr); // make Windows compiler happy - } - else if (SliceRange* range = dynamic_cast(head.get())) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - return std::shared_ptr(nullptr); // make Windows compiler happy - } - - else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { - return getitem_ellipsis(tail, advanced); - } - - else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { - return getitem_newaxis(tail, advanced); - } + const std::shared_ptr EmptyArray::carry(const Index64& carry) const { + return shallow_copy(); + } - else if (SliceArray64* array = dynamic_cast(head.get())) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - return std::shared_ptr(nullptr); // make Windows compiler happy - } + const std::pair EmptyArray::minmax_depth() const { + return std::pair(1, 1); + } - else { - throw std::runtime_error("unrecognized slice item type"); - } + const std::shared_ptr EmptyArray::getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); + return std::shared_ptr(nullptr); // make Windows compiler happy + } - throw std::runtime_error("unreachable because of EmptyArray::carry"); + const std::shared_ptr EmptyArray::getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); + return std::shared_ptr(nullptr); // make Windows compiler happy } - const std::shared_ptr EmptyArray::carry(const Index64& carry) const { - return shallow_copy(); + const std::shared_ptr EmptyArray::getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); + return std::shared_ptr(nullptr); // make Windows compiler happy } - const std::pair EmptyArray::minmax_depth() const { return std::pair(1, 1); } } diff --git a/src/libawkward/array/ListArray.cpp b/src/libawkward/array/ListArray.cpp index 945afdd2d6..5c03cb7806 100644 --- a/src/libawkward/array/ListArray.cpp +++ b/src/libawkward/array/ListArray.cpp @@ -152,13 +152,13 @@ namespace awkward { void ListArrayOf::tojson_part(ToJson& builder) const { for (int64_t i = 0; i < length(); i++) { builder.beginlist(); - getitem_at_unsafe(i).get()->tojson_part(builder); + getitem_at_nowrap(i).get()->tojson_part(builder); builder.endlist(); } } template - std::shared_ptr ListArrayOf::type_part() const { + const std::shared_ptr ListArrayOf::type_part() const { return std::shared_ptr(new ListType(content_.get()->type_part())); } @@ -173,7 +173,7 @@ namespace awkward { } template - void ListArrayOf::checksafe() const { + void ListArrayOf::check_for_iteration() const { if (stops_.length() < starts_.length()) { util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } @@ -194,13 +194,13 @@ namespace awkward { if (regular_at >= stops_.length()) { util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } - return getitem_at_unsafe(regular_at); + return getitem_at_nowrap(regular_at); } template - const std::shared_ptr ListArrayOf::getitem_at_unsafe(int64_t at) const { - int64_t start = (int64_t)starts_.getitem_at_unsafe(at); - int64_t stop = (int64_t)stops_.getitem_at_unsafe(at); + const std::shared_ptr ListArrayOf::getitem_at_nowrap(int64_t at) const { + int64_t start = (int64_t)starts_.getitem_at_nowrap(at); + int64_t stop = (int64_t)stops_.getitem_at_nowrap(at); int64_t lencontent = content_.get()->length(); if (start == stop) { start = stop = 0; @@ -214,7 +214,7 @@ namespace awkward { if (stop > lencontent) { util::handle_error(failure("starts[i] != stops[i] and stops[i] > len(content)", kSliceNone, at), classname(), id_.get()); } - return content_.get()->getitem_range_unsafe(start, stop); + return content_.get()->getitem_range_nowrap(start, stop); } template @@ -228,164 +228,16 @@ namespace awkward { if (id_.get() != nullptr && regular_stop > id_.get()->length()) { util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } - return getitem_range_unsafe(regular_start, regular_stop); + return getitem_range_nowrap(regular_start, regular_stop); } template - const std::shared_ptr ListArrayOf::getitem_range_unsafe(int64_t start, int64_t stop) const { + const std::shared_ptr ListArrayOf::getitem_range_nowrap(int64_t start, int64_t stop) const { std::shared_ptr id(nullptr); if (id_.get() != nullptr) { - id = id_.get()->getitem_range_unsafe(start, stop); - } - return std::shared_ptr(new ListArrayOf(id, starts_.getitem_range_unsafe(start, stop), stops_.getitem_range_unsafe(start, stop), content_)); - } - - template - const std::shared_ptr ListArrayOf::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { - int64_t lenstarts = starts_.length(); - if (stops_.length() < lenstarts) { - util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); - } - - if (head.get() == nullptr) { - return shallow_copy(); - } - - else if (SliceAt* at = dynamic_cast(head.get())) { - assert(advanced.length() == 0); - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - Index64 nextcarry(lenstarts); - struct Error err = util::awkward_listarray_getitem_next_at_64( - nextcarry.ptr().get(), - starts_.ptr().get(), - stops_.ptr().get(), - lenstarts, - starts_.offset(), - stops_.offset(), - at->at()); - util::handle_error(err, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); - } - - else if (SliceRange* range = dynamic_cast(head.get())) { - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - int64_t start = range->start(); - int64_t stop = range->stop(); - int64_t step = range->step(); - if (step == Slice::none()) { - step = 1; - } - int64_t carrylength; - struct Error err1 = util::awkward_listarray_getitem_next_range_carrylength( - &carrylength, - starts_.ptr().get(), - stops_.ptr().get(), - lenstarts, - starts_.offset(), - stops_.offset(), - start, - stop, - step); - util::handle_error(err1, classname(), id_.get()); - - IndexOf nextoffsets(lenstarts + 1); - Index64 nextcarry(carrylength); - - struct Error err2 = util::awkward_listarray_getitem_next_range_64( - nextoffsets.ptr().get(), - nextcarry.ptr().get(), - starts_.ptr().get(), - stops_.ptr().get(), - lenstarts, - starts_.offset(), - stops_.offset(), - start, - stop, - step); - util::handle_error(err2, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - - if (advanced.length() == 0) { - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, advanced))); - } - else { - int64_t total; - struct Error err1 = util::awkward_listarray_getitem_next_range_counts_64( - &total, - nextoffsets.ptr().get(), - lenstarts); - util::handle_error(err1, classname(), id_.get()); - Index64 nextadvanced(total); - struct Error err2 = util::awkward_listarray_getitem_next_range_spreadadvanced_64( - nextadvanced.ptr().get(), - advanced.ptr().get(), - nextoffsets.ptr().get(), - lenstarts); - util::handle_error(err2, classname(), id_.get()); - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); - } - } - - else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { - return getitem_ellipsis(tail, advanced); - } - - else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { - return getitem_newaxis(tail, advanced); - } - - else if (SliceArray64* array = dynamic_cast(head.get())) { - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - Index64 flathead = array->ravel(); - if (advanced.length() == 0) { - Index64 nextcarry(lenstarts*flathead.length()); - Index64 nextadvanced(lenstarts*flathead.length()); - IndexOf nextoffsets(lenstarts + 1); - struct Error err = util::awkward_listarray_getitem_next_array_64( - nextoffsets.ptr().get(), - nextcarry.ptr().get(), - nextadvanced.ptr().get(), - starts_.ptr().get(), - stops_.ptr().get(), - flathead.ptr().get(), - starts_.offset(), - stops_.offset(), - lenstarts, - flathead.length(), - content_.get()->length()); - util::handle_error(err, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); - } - else { - Index64 nextcarry(lenstarts); - Index64 nextadvanced(lenstarts); - struct Error err = util::awkward_listarray_getitem_next_array_advanced_64( - nextcarry.ptr().get(), - nextadvanced.ptr().get(), - starts_.ptr().get(), - stops_.ptr().get(), - flathead.ptr().get(), - advanced.ptr().get(), - starts_.offset(), - stops_.offset(), - lenstarts, - flathead.length(), - content_.get()->length()); - util::handle_error(err, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); - } - } - - else { - throw std::runtime_error("unrecognized slice item type"); + id = id_.get()->getitem_range_nowrap(start, stop); } + return std::shared_ptr(new ListArrayOf(id, starts_.getitem_range_nowrap(start, stop), stops_.getitem_range_nowrap(start, stop), content_)); } template @@ -420,6 +272,148 @@ namespace awkward { return std::pair(content_depth.first + 1, content_depth.second + 1); } + template + const std::shared_ptr ListArrayOf::getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const { + int64_t lenstarts = starts_.length(); + if (stops_.length() < lenstarts) { + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); + } + + assert(advanced.length() == 0); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + Index64 nextcarry(lenstarts); + struct Error err = util::awkward_listarray_getitem_next_at_64( + nextcarry.ptr().get(), + starts_.ptr().get(), + stops_.ptr().get(), + lenstarts, + starts_.offset(), + stops_.offset(), + at.at()); + util::handle_error(err, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); + } + + template + const std::shared_ptr ListArrayOf::getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const { + int64_t lenstarts = starts_.length(); + if (stops_.length() < lenstarts) { + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); + } + + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + int64_t start = range.start(); + int64_t stop = range.stop(); + int64_t step = range.step(); + if (step == Slice::none()) { + step = 1; + } + int64_t carrylength; + struct Error err1 = util::awkward_listarray_getitem_next_range_carrylength( + &carrylength, + starts_.ptr().get(), + stops_.ptr().get(), + lenstarts, + starts_.offset(), + stops_.offset(), + start, + stop, + step); + util::handle_error(err1, classname(), id_.get()); + + IndexOf nextoffsets(lenstarts + 1); + Index64 nextcarry(carrylength); + + struct Error err2 = util::awkward_listarray_getitem_next_range_64( + nextoffsets.ptr().get(), + nextcarry.ptr().get(), + starts_.ptr().get(), + stops_.ptr().get(), + lenstarts, + starts_.offset(), + stops_.offset(), + start, + stop, + step); + util::handle_error(err2, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + + if (advanced.length() == 0) { + return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, advanced))); + } + else { + int64_t total; + struct Error err1 = util::awkward_listarray_getitem_next_range_counts_64( + &total, + nextoffsets.ptr().get(), + lenstarts); + util::handle_error(err1, classname(), id_.get()); + Index64 nextadvanced(total); + struct Error err2 = util::awkward_listarray_getitem_next_range_spreadadvanced_64( + nextadvanced.ptr().get(), + advanced.ptr().get(), + nextoffsets.ptr().get(), + lenstarts); + util::handle_error(err2, classname(), id_.get()); + return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); + } + } + + template + const std::shared_ptr ListArrayOf::getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const { + int64_t lenstarts = starts_.length(); + if (stops_.length() < lenstarts) { + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); + } + + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + Index64 flathead = array.ravel(); + if (advanced.length() == 0) { + Index64 nextcarry(lenstarts*flathead.length()); + Index64 nextadvanced(lenstarts*flathead.length()); + IndexOf nextoffsets(lenstarts + 1); // FIXME: offsets are regular; don't generate them and replace ListOffsetArray output with a RegularArray + struct Error err = util::awkward_listarray_getitem_next_array_64( + nextoffsets.ptr().get(), + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + starts_.ptr().get(), + stops_.ptr().get(), + flathead.ptr().get(), + starts_.offset(), + stops_.offset(), + lenstarts, + flathead.length(), + content_.get()->length()); + util::handle_error(err, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray + return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); + } + else { + Index64 nextcarry(lenstarts); + Index64 nextadvanced(lenstarts); + struct Error err = util::awkward_listarray_getitem_next_array_advanced_64( + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + starts_.ptr().get(), + stops_.ptr().get(), + flathead.ptr().get(), + advanced.ptr().get(), + starts_.offset(), + stops_.offset(), + lenstarts, + flathead.length(), + content_.get()->length()); + util::handle_error(err, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); + } + } + template class ListArrayOf; template class ListArrayOf; template class ListArrayOf; diff --git a/src/libawkward/array/ListOffsetArray.cpp b/src/libawkward/array/ListOffsetArray.cpp index 1523ba60eb..67fb263e37 100644 --- a/src/libawkward/array/ListOffsetArray.cpp +++ b/src/libawkward/array/ListOffsetArray.cpp @@ -165,13 +165,13 @@ namespace awkward { void ListOffsetArrayOf::tojson_part(ToJson& builder) const { for (int64_t i = 0; i < length(); i++) { builder.beginlist(); - getitem_at_unsafe(i).get()->tojson_part(builder); + getitem_at_nowrap(i).get()->tojson_part(builder); builder.endlist(); } } template - std::shared_ptr ListOffsetArrayOf::type_part() const { + const std::shared_ptr ListOffsetArrayOf::type_part() const { return std::shared_ptr(new ListType(content_.get()->type_part())); } @@ -186,7 +186,7 @@ namespace awkward { } template - void ListOffsetArrayOf::checksafe() const { + void ListOffsetArrayOf::check_for_iteration() const { if (id_.get() != nullptr && id_.get()->length() < offsets_.length() - 1) { util::handle_error(failure("len(id) < len(array)", kSliceNone, kSliceNone), id_.get()->classname(), nullptr); } @@ -201,13 +201,13 @@ namespace awkward { if (!(0 <= regular_at && regular_at < offsets_.length() - 1)) { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); } - return getitem_at_unsafe(regular_at); + return getitem_at_nowrap(regular_at); } template - const std::shared_ptr ListOffsetArrayOf::getitem_at_unsafe(int64_t at) const { - int64_t start = (int64_t)offsets_.getitem_at_unsafe(at); - int64_t stop = (int64_t)offsets_.getitem_at_unsafe(at + 1); + const std::shared_ptr ListOffsetArrayOf::getitem_at_nowrap(int64_t at) const { + int64_t start = (int64_t)offsets_.getitem_at_nowrap(at); + int64_t stop = (int64_t)offsets_.getitem_at_nowrap(at + 1); int64_t lencontent = content_.get()->length(); if (start == stop) { start = stop = 0; @@ -221,7 +221,7 @@ namespace awkward { if (stop > lencontent) { util::handle_error(failure("offsets[i] != offsets[i + 1] and offsets[i + 1] > len(content)", kSliceNone, at), classname(), id_.get()); } - return content_.get()->getitem_range_unsafe(start, stop); + return content_.get()->getitem_range_nowrap(start, stop); } template @@ -232,168 +232,16 @@ namespace awkward { if (id_.get() != nullptr && regular_stop > id_.get()->length()) { util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } - return getitem_range_unsafe(regular_start, regular_stop); + return getitem_range_nowrap(regular_start, regular_stop); } template - const std::shared_ptr ListOffsetArrayOf::getitem_range_unsafe(int64_t start, int64_t stop) const { + const std::shared_ptr ListOffsetArrayOf::getitem_range_nowrap(int64_t start, int64_t stop) const { std::shared_ptr id(nullptr); if (id_.get() != nullptr) { - id = id_.get()->getitem_range_unsafe(start, stop); - } - return std::shared_ptr(new ListOffsetArrayOf(id, offsets_.getitem_range_unsafe(start, stop + 1), content_)); - } - - template - const std::shared_ptr ListOffsetArrayOf::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { - int64_t lenstarts = offsets_.length() - 1; - - if (head.get() == nullptr) { - return shallow_copy(); - } - - else if (SliceAt* at = dynamic_cast(head.get())) { - assert(advanced.length() == 0); - IndexOf starts = make_starts(offsets_); - IndexOf stops = make_stops(offsets_); - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - Index64 nextcarry(lenstarts); - struct Error err = util::awkward_listarray_getitem_next_at_64( - nextcarry.ptr().get(), - starts.ptr().get(), - stops.ptr().get(), - lenstarts, - starts.offset(), - stops.offset(), - at->at()); - util::handle_error(err, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); - } - - else if (SliceRange* range = dynamic_cast(head.get())) { - IndexOf starts = make_starts(offsets_); - IndexOf stops = make_stops(offsets_); - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - int64_t start = range->start(); - int64_t stop = range->stop(); - int64_t step = range->step(); - if (step == Slice::none()) { - step = 1; - } - int64_t carrylength; - struct Error err1 = util::awkward_listarray_getitem_next_range_carrylength( - &carrylength, - starts.ptr().get(), - stops.ptr().get(), - lenstarts, - starts.offset(), - stops.offset(), - start, - stop, - step); - util::handle_error(err1, classname(), id_.get()); - - IndexOf nextoffsets(lenstarts + 1); - Index64 nextcarry(carrylength); - - struct Error err2 = util::awkward_listarray_getitem_next_range_64( - nextoffsets.ptr().get(), - nextcarry.ptr().get(), - starts.ptr().get(), - stops.ptr().get(), - lenstarts, - starts.offset(), - stops.offset(), - start, - stop, - step); - util::handle_error(err2, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - - if (advanced.length() == 0) { - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, advanced))); - } - else { - int64_t total; - struct Error err1 = util::awkward_listarray_getitem_next_range_counts_64( - &total, - nextoffsets.ptr().get(), - lenstarts); - util::handle_error(err1, classname(), id_.get()); - Index64 nextadvanced(total); - struct Error err2 = util::awkward_listarray_getitem_next_range_spreadadvanced_64( - nextadvanced.ptr().get(), - advanced.ptr().get(), - nextoffsets.ptr().get(), - lenstarts); - util::handle_error(err2, classname(), id_.get()); - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); - } - } - - else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { - return getitem_ellipsis(tail, advanced); - } - - else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { - return getitem_newaxis(tail, advanced); - } - - else if (SliceArray64* array = dynamic_cast(head.get())) { - IndexOf starts = make_starts(offsets_); - IndexOf stops = make_stops(offsets_); - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - Index64 flathead = array->ravel(); - if (advanced.length() == 0) { - Index64 nextcarry(lenstarts*flathead.length()); - Index64 nextadvanced(lenstarts*flathead.length()); - IndexOf nextoffsets(lenstarts + 1); - IndexOf nextstops(lenstarts); - struct Error err = util::awkward_listarray_getitem_next_array_64( - nextoffsets.ptr().get(), - nextcarry.ptr().get(), - nextadvanced.ptr().get(), - starts.ptr().get(), - stops.ptr().get(), - flathead.ptr().get(), - starts.offset(), - stops.offset(), - lenstarts, - flathead.length(), - content_.get()->length()); - util::handle_error(err, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); - } - else { - Index64 nextcarry(lenstarts); - Index64 nextadvanced(lenstarts); - struct Error err = util::awkward_listarray_getitem_next_array_advanced_64( - nextcarry.ptr().get(), - nextadvanced.ptr().get(), - starts.ptr().get(), - stops.ptr().get(), - flathead.ptr().get(), - advanced.ptr().get(), - starts.offset(), - stops.offset(), - lenstarts, - flathead.length(), - content_.get()->length()); - util::handle_error(err, classname(), id_.get()); - std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); - } - } - - else { - throw std::runtime_error("unrecognized slice item type"); + id = id_.get()->getitem_range_nowrap(start, stop); } + return std::shared_ptr(new ListOffsetArrayOf(id, offsets_.getitem_range_nowrap(start, stop + 1), content_)); } template @@ -426,6 +274,143 @@ namespace awkward { return std::pair(content_depth.first + 1, content_depth.second + 1); } + template + const std::shared_ptr ListOffsetArrayOf::getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const { + assert(advanced.length() == 0); + int64_t lenstarts = offsets_.length() - 1; + IndexOf starts = make_starts(offsets_); + IndexOf stops = make_stops(offsets_); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + Index64 nextcarry(lenstarts); + struct Error err = util::awkward_listarray_getitem_next_at_64( + nextcarry.ptr().get(), + starts.ptr().get(), + stops.ptr().get(), + lenstarts, + starts.offset(), + stops.offset(), + at.at()); + util::handle_error(err, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); + } + + template + const std::shared_ptr ListOffsetArrayOf::getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const { + int64_t lenstarts = offsets_.length() - 1; + IndexOf starts = make_starts(offsets_); + IndexOf stops = make_stops(offsets_); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + int64_t start = range.start(); + int64_t stop = range.stop(); + int64_t step = range.step(); + if (step == Slice::none()) { + step = 1; + } + int64_t carrylength; + struct Error err1 = util::awkward_listarray_getitem_next_range_carrylength( + &carrylength, + starts.ptr().get(), + stops.ptr().get(), + lenstarts, + starts.offset(), + stops.offset(), + start, + stop, + step); + util::handle_error(err1, classname(), id_.get()); + + IndexOf nextoffsets(lenstarts + 1); + Index64 nextcarry(carrylength); + + struct Error err2 = util::awkward_listarray_getitem_next_range_64( + nextoffsets.ptr().get(), + nextcarry.ptr().get(), + starts.ptr().get(), + stops.ptr().get(), + lenstarts, + starts.offset(), + stops.offset(), + start, + stop, + step); + util::handle_error(err2, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + + if (advanced.length() == 0) { + return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, advanced))); + } + else { + int64_t total; + struct Error err1 = util::awkward_listarray_getitem_next_range_counts_64( + &total, + nextoffsets.ptr().get(), + lenstarts); + util::handle_error(err1, classname(), id_.get()); + Index64 nextadvanced(total); + struct Error err2 = util::awkward_listarray_getitem_next_range_spreadadvanced_64( + nextadvanced.ptr().get(), + advanced.ptr().get(), + nextoffsets.ptr().get(), + lenstarts); + util::handle_error(err2, classname(), id_.get()); + return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); + } + } + + template + const std::shared_ptr ListOffsetArrayOf::getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const { + int64_t lenstarts = offsets_.length() - 1; + IndexOf starts = make_starts(offsets_); + IndexOf stops = make_stops(offsets_); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + Index64 flathead = array.ravel(); + if (advanced.length() == 0) { + Index64 nextcarry(lenstarts*flathead.length()); + Index64 nextadvanced(lenstarts*flathead.length()); + IndexOf nextoffsets(lenstarts + 1); // FIXME: offsets are regular; don't generate them and replace ListOffsetArray output with a RegularArray + IndexOf nextstops(lenstarts); + struct Error err = util::awkward_listarray_getitem_next_array_64( + nextoffsets.ptr().get(), + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + starts.ptr().get(), + stops.ptr().get(), + flathead.ptr().get(), + starts.offset(), + stops.offset(), + lenstarts, + flathead.length(), + content_.get()->length()); + util::handle_error(err, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray + return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); + } + else { + Index64 nextcarry(lenstarts); + Index64 nextadvanced(lenstarts); + struct Error err = util::awkward_listarray_getitem_next_array_advanced_64( + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + starts.ptr().get(), + stops.ptr().get(), + flathead.ptr().get(), + advanced.ptr().get(), + starts.offset(), + stops.offset(), + lenstarts, + flathead.length(), + content_.get()->length()); + util::handle_error(err, classname(), id_.get()); + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); + } + } + template class ListOffsetArrayOf; template class ListOffsetArrayOf; template class ListOffsetArrayOf; diff --git a/src/libawkward/array/NumpyArray.cpp b/src/libawkward/array/NumpyArray.cpp index 988f30dcc9..275c13b2e7 100644 --- a/src/libawkward/array/NumpyArray.cpp +++ b/src/libawkward/array/NumpyArray.cpp @@ -22,7 +22,9 @@ namespace awkward { bool NumpyArray::isempty() const { for (auto x : shape_) { - if (x == 0) return true; + if (x == 0) { + return true; + } } return false; // false for isscalar(), too } @@ -48,7 +50,9 @@ namespace awkward { return *reinterpret_cast(reinterpret_cast(ptr_.get()) + byteoffset_ + at); } - const std::string NumpyArray::classname() const { return "NumpyArray"; } + const std::string NumpyArray::classname() const { + return "NumpyArray"; + } void NumpyArray::setid(const std::shared_ptr id) { if (id.get() != nullptr && length() != id.get()->length()) { @@ -259,13 +263,13 @@ namespace awkward { else { for (int64_t i = 0; i < length(); i++) { builder.beginlist(); - getitem_at_unsafe(i).get()->tojson_part(builder); + getitem_at_nowrap(i).get()->tojson_part(builder); builder.endlist(); } } } - std::shared_ptr NumpyArray::type_part() const { + const std::shared_ptr NumpyArray::type_part() const { if (ndim() == 1) { if (format_.compare("d") == 0) { return std::shared_ptr(new PrimitiveType(PrimitiveType::float64)); @@ -319,8 +323,11 @@ namespace awkward { } else { NumpyArray tmp(id_, ptr_, std::vector({ 1 }), std::vector({ itemsize_ }), byteoffset_, itemsize_, format_); - std::vector shape(shape_.begin() + 1, shape_.end()); - return std::shared_ptr(new RegularType(shape, tmp.type_part())); + std::shared_ptr out = tmp.type_part(); + for (ssize_t i = shape_.size() - 1; i > 0; i--) { + out = std::shared_ptr(new RegularType(out, (int64_t)shape_[i])); + } + return out; } } @@ -337,7 +344,7 @@ namespace awkward { return std::shared_ptr(new NumpyArray(id_, ptr_, shape_, strides_, byteoffset_, itemsize_, format_)); } - void NumpyArray::checksafe() const { + void NumpyArray::check_for_iteration() const { if (id_.get() != nullptr && id_.get()->length() < shape_[0]) { util::handle_error(failure("len(id) < len(array)", kSliceNone, kSliceNone), id_.get()->classname(), nullptr); } @@ -352,10 +359,10 @@ namespace awkward { if (regular_at < 0 || regular_at >= shape_[0]) { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); } - return getitem_at_unsafe(regular_at); + return getitem_at_nowrap(regular_at); } - const std::shared_ptr NumpyArray::getitem_at_unsafe(int64_t at) const { + const std::shared_ptr NumpyArray::getitem_at_nowrap(int64_t at) const { ssize_t byteoffset = byteoffset_ + strides_[0]*((ssize_t)at); const std::vector shape(shape_.begin() + 1, shape_.end()); const std::vector strides(strides_.begin() + 1, strides_.end()); @@ -364,7 +371,7 @@ namespace awkward { if (at >= id_.get()->length()) { util::handle_error(failure("index out of range", kSliceNone, at), id_.get()->classname(), nullptr); } - id = id_.get()->getitem_range_unsafe(at, at + 1); + id = id_.get()->getitem_range_nowrap(at, at + 1); } return std::shared_ptr(new NumpyArray(id, ptr_, shape, strides, byteoffset, itemsize_, format_)); } @@ -374,10 +381,10 @@ namespace awkward { int64_t regular_start = start; int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), shape_[0]); - return getitem_range_unsafe(regular_start, regular_stop); + return getitem_range_nowrap(regular_start, regular_stop); } - const std::shared_ptr NumpyArray::getitem_range_unsafe(int64_t start, int64_t stop) const { + const std::shared_ptr NumpyArray::getitem_range_nowrap(int64_t start, int64_t stop) const { ssize_t byteoffset = byteoffset_ + strides_[0]*((ssize_t)start); std::vector shape; shape.push_back((ssize_t)(stop - start)); @@ -387,7 +394,7 @@ namespace awkward { if (stop > id_.get()->length()) { util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } - id = id_.get()->getitem_range_unsafe(start, stop); + id = id_.get()->getitem_range_nowrap(start, stop); } return std::shared_ptr(new NumpyArray(id, ptr_, shape, strides_, byteoffset, itemsize_, format_)); } @@ -571,101 +578,112 @@ namespace awkward { if (head.get() == nullptr) { return NumpyArray(id_, ptr_, shape_, strides_, byteoffset_, itemsize_, format_); } - else if (SliceAt* at = dynamic_cast(head.get())) { - if (ndim() < 2) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - } - - int64_t i = at->at(); - if (i < 0) i += shape_[1]; - if (i < 0 || i >= shape_[1]) { - util::handle_error(failure("index out of range", kSliceNone, at->at()), classname(), id_.get()); - } - - ssize_t nextbyteoffset = byteoffset_ + ((ssize_t)i)*strides_[1]; - NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), nextbyteoffset, itemsize_, format_); - - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - NumpyArray out = next.getitem_bystrides(nexthead, nexttail, length); + return getitem_bystrides(*at, tail, length); + } + else if (SliceRange* range = dynamic_cast(head.get())) { + return getitem_bystrides(*range, tail, length); + } + else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { + return getitem_bystrides(*ellipsis, tail, length); + } + else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { + return getitem_bystrides(*newaxis, tail, length); + } + else { + throw std::runtime_error("unrecognized slice item type"); + } + } - std::vector outshape = { (ssize_t)length }; - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, out.strides_, out.byteoffset_, itemsize_, format_); + const NumpyArray NumpyArray::getitem_bystrides(const SliceAt& at, const Slice& tail, int64_t length) const { + if (ndim() < 2) { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } - else if (SliceRange* range = dynamic_cast(head.get())) { - if (ndim() < 2) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - } + int64_t i = at.at(); + if (i < 0) i += shape_[1]; + if (i < 0 || i >= shape_[1]) { + util::handle_error(failure("index out of range", kSliceNone, at.at()), classname(), id_.get()); + } - int64_t start = range->start(); - int64_t stop = range->stop(); - int64_t step = range->step(); - if (step == Slice::none()) { - step = 1; - } - awkward_regularize_rangeslice(&start, &stop, step > 0, range->hasstart(), range->hasstop(), (int64_t)shape_[1]); + ssize_t nextbyteoffset = byteoffset_ + ((ssize_t)i)*strides_[1]; + NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), nextbyteoffset, itemsize_, format_); - int64_t numer = abs(start - stop); - int64_t denom = abs(step); - int64_t d = numer / denom; - int64_t m = numer % denom; - int64_t lenhead = d + (m != 0 ? 1 : 0); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + NumpyArray out = next.getitem_bystrides(nexthead, nexttail, length); - ssize_t nextbyteoffset = byteoffset_ + ((ssize_t)start)*strides_[1]; - NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), nextbyteoffset, itemsize_, format_); + std::vector outshape = { (ssize_t)length }; + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, out.strides_, out.byteoffset_, itemsize_, format_); + } - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - NumpyArray out = next.getitem_bystrides(nexthead, nexttail, length*lenhead); + const NumpyArray NumpyArray::getitem_bystrides(const SliceRange& range, const Slice& tail, int64_t length) const { + if (ndim() < 2) { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); + } - std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - std::vector outstrides = { strides_[0], strides_[1]*((ssize_t)step) }; - outstrides.insert(outstrides.end(), out.strides_.begin() + 1, out.strides_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + int64_t start = range.start(); + int64_t stop = range.stop(); + int64_t step = range.step(); + if (step == Slice::none()) { + step = 1; } + awkward_regularize_rangeslice(&start, &stop, step > 0, range.hasstart(), range.hasstop(), (int64_t)shape_[1]); - else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { - std::pair minmax = minmax_depth(); - assert(minmax.first == minmax.second); - int64_t mindepth = minmax.first; + int64_t numer = abs(start - stop); + int64_t denom = abs(step); + int64_t d = numer / denom; + int64_t m = numer % denom; + int64_t lenhead = d + (m != 0 ? 1 : 0); - if (tail.length() == 0 || mindepth - 1 == tail.dimlength()) { - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - return getitem_bystrides(nexthead, nexttail, length); - } - else { - std::vector> tailitems = tail.items(); - std::vector> items = { std::shared_ptr(new SliceEllipsis()) }; - items.insert(items.end(), tailitems.begin(), tailitems.end()); + ssize_t nextbyteoffset = byteoffset_ + ((ssize_t)start)*strides_[1]; + NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), nextbyteoffset, itemsize_, format_); - std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); - Slice nexttail(items); - return getitem_bystrides(nexthead, nexttail, length); - } - } + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + NumpyArray out = next.getitem_bystrides(nexthead, nexttail, length*lenhead); - else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { + std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + std::vector outstrides = { strides_[0], strides_[1]*((ssize_t)step) }; + outstrides.insert(outstrides.end(), out.strides_.begin() + 1, out.strides_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + } + + const NumpyArray NumpyArray::getitem_bystrides(const SliceEllipsis& ellipsis, const Slice& tail, int64_t length) const { + std::pair minmax = minmax_depth(); + assert(minmax.first == minmax.second); + int64_t mindepth = minmax.first; + + if (tail.length() == 0 || mindepth - 1 == tail.dimlength()) { std::shared_ptr nexthead = tail.head(); Slice nexttail = tail.tail(); - NumpyArray out = getitem_bystrides(nexthead, nexttail, length); - - std::vector outshape = { (ssize_t)length, 1 }; - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - std::vector outstrides = { out.strides_[0] }; - outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + return getitem_bystrides(nexthead, nexttail, length); } - else { - throw std::runtime_error("unrecognized slice item type"); + std::vector> tailitems = tail.items(); + std::vector> items = { std::shared_ptr(new SliceEllipsis()) }; + items.insert(items.end(), tailitems.begin(), tailitems.end()); + + std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); + Slice nexttail(items); + return getitem_bystrides(nexthead, nexttail, length); } } + const NumpyArray NumpyArray::getitem_bystrides(const SliceNewAxis& newaxis, const Slice& tail, int64_t length) const { + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + NumpyArray out = getitem_bystrides(nexthead, nexttail, length); + + std::vector outshape = { (ssize_t)length, 1 }; + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + std::vector outstrides = { out.strides_[0] }; + outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + } + const NumpyArray NumpyArray::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const { if (head.get() == nullptr) { std::shared_ptr ptr(new uint8_t[(size_t)(carry.length()*stride)], awkward::util::array_deleter()); @@ -691,206 +709,222 @@ namespace awkward { } else if (SliceAt* at = dynamic_cast(head.get())) { - if (ndim() < 2) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - } + return getitem_next(*at, tail, carry, advanced, length, stride, first); + } + else if (SliceRange* range = dynamic_cast(head.get())) { + return getitem_next(*range, tail, carry, advanced, length, stride, first); + } + else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { + return getitem_next(*ellipsis, tail, carry, advanced, length, stride, first); + } + else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { + return getitem_next(*newaxis, tail, carry, advanced, length, stride, first); + } + else if (SliceArray64* array = dynamic_cast(head.get())) { + return getitem_next(*array, tail, carry, advanced, length, stride, first); + } + else { + throw std::runtime_error("unrecognized slice item type"); + } + } - NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); + const NumpyArray NumpyArray::getitem_next(const SliceAt& at, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const { + if (ndim() < 2) { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); + } - // if we had any array slices, this int would become an array - assert(advanced.length() == 0); + NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); - int64_t regular_at = at->at(); - if (regular_at < 0) { - regular_at += shape_[1]; - } - if (!(0 <= regular_at && regular_at < shape_[1])) { - util::handle_error(failure("index out of range", kSliceNone, at->at()), classname(), id_.get()); - } + // if we had any array slices, this int would become an array + assert(advanced.length() == 0); - Index64 nextcarry(carry.length()); - struct Error err = awkward_numpyarray_getitem_next_at_64( + int64_t regular_at = at.at(); + if (regular_at < 0) { + regular_at += shape_[1]; + } + if (!(0 <= regular_at && regular_at < shape_[1])) { + util::handle_error(failure("index out of range", kSliceNone, at.at()), classname(), id_.get()); + } + + Index64 nextcarry(carry.length()); + struct Error err = awkward_numpyarray_getitem_next_at_64( + nextcarry.ptr().get(), + carry.ptr().get(), + carry.length(), + shape_[1], // because this is contiguous + regular_at); + util::handle_error(err, classname(), id_.get()); + + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length, next.strides_[0], false); + + std::vector outshape = { (ssize_t)length }; + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, out.strides_, out.byteoffset_, itemsize_, format_); + } + + const NumpyArray NumpyArray::getitem_next(const SliceRange& range, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const { + if (ndim() < 2) { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); + } + + int64_t start = range.start(); + int64_t stop = range.stop(); + int64_t step = range.step(); + if (step == Slice::none()) { + step = 1; + } + awkward_regularize_rangeslice(&start, &stop, step > 0, range.hasstart(), range.hasstop(), (int64_t)shape_[1]); + + int64_t numer = abs(start - stop); + int64_t denom = abs(step); + int64_t d = numer / denom; + int64_t m = numer % denom; + int64_t lenhead = d + (m != 0 ? 1 : 0); + + NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + + if (advanced.length() == 0) { + Index64 nextcarry(carry.length()*lenhead); + struct Error err = awkward_numpyarray_getitem_next_range_64( nextcarry.ptr().get(), carry.ptr().get(), carry.length(), + lenhead, shape_[1], // because this is contiguous - regular_at); + start, + step); util::handle_error(err, classname(), id_.get()); - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length, next.strides_[0], false); - - std::vector outshape = { (ssize_t)length }; + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*lenhead, next.strides_[0], false); + std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, out.strides_, out.byteoffset_, itemsize_, format_); + std::vector outstrides = { (ssize_t)lenhead*out.strides_[0] }; + outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); } - else if (SliceRange* range = dynamic_cast(head.get())) { - if (ndim() < 2) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - } + else { + Index64 nextcarry(carry.length()*lenhead); + Index64 nextadvanced(carry.length()*lenhead); + struct Error err = awkward_numpyarray_getitem_next_range_advanced_64( + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + carry.ptr().get(), + advanced.ptr().get(), + carry.length(), + lenhead, + shape_[1], // because this is contiguous + start, + step); + util::handle_error(err, classname(), id_.get()); - int64_t start = range->start(); - int64_t stop = range->stop(); - int64_t step = range->step(); - if (step == Slice::none()) { - step = 1; - } - awkward_regularize_rangeslice(&start, &stop, step > 0, range->hasstart(), range->hasstop(), (int64_t)shape_[1]); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*lenhead, next.strides_[0], false); + std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + std::vector outstrides = { (ssize_t)lenhead*out.strides_[0] }; + outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + } + } - int64_t numer = abs(start - stop); - int64_t denom = abs(step); - int64_t d = numer / denom; - int64_t m = numer % denom; - int64_t lenhead = d + (m != 0 ? 1 : 0); + const NumpyArray NumpyArray::getitem_next(const SliceEllipsis& ellipsis, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const { + std::pair minmax = minmax_depth(); + assert(minmax.first == minmax.second); + int64_t mindepth = minmax.first; - NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); + if (tail.length() == 0 || mindepth - 1 == tail.dimlength()) { std::shared_ptr nexthead = tail.head(); Slice nexttail = tail.tail(); - - if (advanced.length() == 0) { - Index64 nextcarry(carry.length()*lenhead); - struct Error err = awkward_numpyarray_getitem_next_range_64( - nextcarry.ptr().get(), - carry.ptr().get(), - carry.length(), - lenhead, - shape_[1], // because this is contiguous - start, - step); - util::handle_error(err, classname(), id_.get()); - - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*lenhead, next.strides_[0], false); - std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - std::vector outstrides = { (ssize_t)lenhead*out.strides_[0] }; - outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); - } - - else { - Index64 nextcarry(carry.length()*lenhead); - Index64 nextadvanced(carry.length()*lenhead); - struct Error err = awkward_numpyarray_getitem_next_range_advanced_64( - nextcarry.ptr().get(), - nextadvanced.ptr().get(), - carry.ptr().get(), - advanced.ptr().get(), - carry.length(), - lenhead, - shape_[1], // because this is contiguous - start, - step); - util::handle_error(err, classname(), id_.get()); - - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*lenhead, next.strides_[0], false); - std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - std::vector outstrides = { (ssize_t)lenhead*out.strides_[0] }; - outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); - } + return getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); } - - else if (SliceEllipsis* ellipsis = dynamic_cast(head.get())) { - std::pair minmax = minmax_depth(); - assert(minmax.first == minmax.second); - int64_t mindepth = minmax.first; - - if (tail.length() == 0 || mindepth - 1 == tail.dimlength()) { - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - return getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); - } - else { - std::vector> tailitems = tail.items(); - std::vector> items = { std::shared_ptr(new SliceEllipsis()) }; - items.insert(items.end(), tailitems.begin(), tailitems.end()); - std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); - Slice nexttail(items); - return getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); - } + else { + std::vector> tailitems = tail.items(); + std::vector> items = { std::shared_ptr(new SliceEllipsis()) }; + items.insert(items.end(), tailitems.begin(), tailitems.end()); + std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); + Slice nexttail(items); + return getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); } + } - else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); - NumpyArray out = getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); + const NumpyArray NumpyArray::getitem_next(const SliceNewAxis& newaxis, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const { + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + NumpyArray out = getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); - std::vector outshape = { (ssize_t)length, 1 }; - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - std::vector outstrides = { out.strides_[0] }; - outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + std::vector outshape = { (ssize_t)length, 1 }; + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + std::vector outstrides = { out.strides_[0] }; + outstrides.insert(outstrides.end(), out.strides_.begin(), out.strides_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + } + + const NumpyArray NumpyArray::getitem_next(const SliceArray64& array, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const { + if (ndim() < 2) { + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } - else if (SliceArray64* array = dynamic_cast(head.get())) { - if (ndim() < 2) { - util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); - } + NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); - NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); - std::shared_ptr nexthead = tail.head(); - Slice nexttail = tail.tail(); + Index64 flathead = array.ravel(); + struct Error err = awkward_regularize_arrayslice_64( + flathead.ptr().get(), + flathead.length(), + shape_[1]); + util::handle_error(err, classname(), id_.get()); - Index64 flathead = array->ravel(); - struct Error err = awkward_regularize_arrayslice_64( + if (advanced.length() == 0) { + Index64 nextcarry(carry.length()*flathead.length()); + Index64 nextadvanced(carry.length()*flathead.length()); + struct Error err = awkward_numpyarray_getitem_next_array_64( + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + carry.ptr().get(), flathead.ptr().get(), + carry.length(), flathead.length(), - shape_[1]); + shape_[1]); // because this is contiguous util::handle_error(err, classname(), id_.get()); - if (advanced.length() == 0) { - Index64 nextcarry(carry.length()*flathead.length()); - Index64 nextadvanced(carry.length()*flathead.length()); - struct Error err = awkward_numpyarray_getitem_next_array_64( - nextcarry.ptr().get(), - nextadvanced.ptr().get(), - carry.ptr().get(), - flathead.ptr().get(), - carry.length(), - flathead.length(), - shape_[1]); // because this is contiguous - util::handle_error(err, classname(), id_.get()); - - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*flathead.length(), next.strides_[0], false); - - std::vector outshape = { (ssize_t)length }; - std::vector arrayshape = array->shape(); - for (auto x = arrayshape.begin(); x != arrayshape.end(); ++x) { - outshape.push_back((ssize_t)(*x)); - } - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*flathead.length(), next.strides_[0], false); - std::vector outstrides(out.strides_.begin(), out.strides_.end()); - for (auto x = arrayshape.rbegin(); x != arrayshape.rend(); ++x) { - outstrides.insert(outstrides.begin(), ((ssize_t)(*x))*outstrides[0]); - } - return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + std::vector outshape = { (ssize_t)length }; + std::vector arrayshape = array.shape(); + for (auto x = arrayshape.begin(); x != arrayshape.end(); ++x) { + outshape.push_back((ssize_t)(*x)); } + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - else { - Index64 nextcarry(carry.length()); - struct Error err = awkward_numpyarray_getitem_next_array_advanced_64( - nextcarry.ptr().get(), - carry.ptr().get(), - advanced.ptr().get(), - flathead.ptr().get(), - carry.length(), - shape_[1]); // because this is contiguous - util::handle_error(err, classname(), id_.get()); - - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*array->length(), next.strides_[0], false); - - std::vector outshape = { (ssize_t)length }; - outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); - return NumpyArray(out.id_, out.ptr_, outshape, out.strides_, out.byteoffset_, itemsize_, format_); + std::vector outstrides(out.strides_.begin(), out.strides_.end()); + for (auto x = arrayshape.rbegin(); x != arrayshape.rend(); ++x) { + outstrides.insert(outstrides.begin(), ((ssize_t)(*x))*outstrides[0]); } + return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); } else { - throw std::runtime_error("unrecognized slice item type"); + Index64 nextcarry(carry.length()); + struct Error err = awkward_numpyarray_getitem_next_array_advanced_64( + nextcarry.ptr().get(), + carry.ptr().get(), + advanced.ptr().get(), + flathead.ptr().get(), + carry.length(), + shape_[1]); // because this is contiguous + util::handle_error(err, classname(), id_.get()); + + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*array.length(), next.strides_[0], false); + + std::vector outshape = { (ssize_t)length }; + outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); + return NumpyArray(out.id_, out.ptr_, outshape, out.strides_, out.byteoffset_, itemsize_, format_); } } + } diff --git a/src/libawkward/array/RegularArray.cpp b/src/libawkward/array/RegularArray.cpp new file mode 100644 index 0000000000..0db653683d --- /dev/null +++ b/src/libawkward/array/RegularArray.cpp @@ -0,0 +1,291 @@ +// BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +#include +#include +#include + +#include "awkward/cpu-kernels/identity.h" +#include "awkward/cpu-kernels/getitem.h" +#include "awkward/type/RegularType.h" + +#include "awkward/array/RegularArray.h" + +namespace awkward { + const std::string RegularArray::classname() const { + return "RegularArray"; + } + + void RegularArray::setid(const std::shared_ptr id) { + if (id.get() == nullptr) { + content_.get()->setid(id); + } + else { + if (length() != id.get()->length()) { + util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); + } + std::shared_ptr bigid = id; + if (content_.get()->length() > kMaxInt32) { + bigid = id.get()->to64(); + } + if (Identity32* rawid = dynamic_cast(bigid.get())) { + Identity32* rawsubid = new Identity32(Identity::newref(), rawid->fieldloc(), rawid->width() + 1, content_.get()->length()); + std::shared_ptr subid(rawsubid); + struct Error err = awkward_identity32_from_regulararray( + rawsubid->ptr().get(), + rawid->ptr().get(), + rawid->offset(), + size_, + content_.get()->length(), + length(), + rawid->width()); + util::handle_error(err, classname(), id_.get()); + content_.get()->setid(subid); + } + else if (Identity64* rawid = dynamic_cast(bigid.get())) { + Identity64* rawsubid = new Identity64(Identity::newref(), rawid->fieldloc(), rawid->width() + 1, content_.get()->length()); + std::shared_ptr subid(rawsubid); + struct Error err = awkward_identity64_from_regulararray( + rawsubid->ptr().get(), + rawid->ptr().get(), + rawid->offset(), + size_, + content_.get()->length(), + length(), + rawid->width()); + util::handle_error(err, classname(), id_.get()); + content_.get()->setid(subid); + } + else { + throw std::runtime_error("unrecognized Identity specialization"); + } + } + id_ = id; + } + + void RegularArray::setid() { + if (length() < kMaxInt32) { + Identity32* rawid = new Identity32(Identity::newref(), Identity::FieldLoc(), 1, length()); + std::shared_ptr newid(rawid); + struct Error err = awkward_new_identity32(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); + setid(newid); + } + else { + Identity64* rawid = new Identity64(Identity::newref(), Identity::FieldLoc(), 1, length()); + std::shared_ptr newid(rawid); + struct Error err = awkward_new_identity64(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); + setid(newid); + } + } + + const std::string RegularArray::tostring_part(const std::string indent, const std::string pre, const std::string post) const { + std::stringstream out; + out << indent << pre << "<" << classname() << " size=\"" << size_ << "\">\n"; + if (id_.get() != nullptr) { + out << id_.get()->tostring_part(indent + std::string(" "), "", "\n"); + } + out << content_.get()->tostring_part(indent + std::string(" "), "", "\n"); + out << indent << "" << post; + return out.str(); + } + + void RegularArray::tojson_part(ToJson& builder) const { + throw std::runtime_error("tojson_part"); + } + + const std::shared_ptr RegularArray::type_part() const { + return std::shared_ptr(new RegularType(content_.get()->type_part(), size_)); + } + + int64_t RegularArray::length() const { + return content_.get()->length() / size_; // floor of length / size + } + + const std::shared_ptr RegularArray::shallow_copy() const { + return std::shared_ptr(new RegularArray(id_, content_, size_)); + } + + void RegularArray::check_for_iteration() const { } + + const std::shared_ptr RegularArray::getitem_at(int64_t at) const { + int64_t regular_at = at; + int64_t len = length(); + if (regular_at < 0) { + regular_at += len; + } + if (!(0 <= regular_at && regular_at < len)) { + util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); + } + return getitem_at_nowrap(regular_at); + } + + const std::shared_ptr RegularArray::getitem_at_nowrap(int64_t at) const { + return content_.get()->getitem_range_nowrap(at*size_, (at + 1)*size_); + } + + const std::shared_ptr RegularArray::getitem_range(int64_t start, int64_t stop) const { + int64_t regular_start = start; + int64_t regular_stop = stop; + awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), length()); + if (id_.get() != nullptr && regular_stop > id_.get()->length()) { + util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); + } + return getitem_range_nowrap(regular_start, regular_stop); + } + + const std::shared_ptr RegularArray::getitem_range_nowrap(int64_t start, int64_t stop) const { + std::shared_ptr id(nullptr); + if (id_.get() != nullptr) { + id = id_.get()->getitem_range_nowrap(start, stop); + } + return std::shared_ptr(new RegularArray(id_, content_.get()->getitem_range_nowrap(start*size_, stop*size_), size_)); + } + + const std::shared_ptr RegularArray::carry(const Index64& carry) const { + Index64 nextcarry(carry.length()*size_); + + struct Error err = awkward_regulararray_getitem_carry_64( + nextcarry.ptr().get(), + carry.ptr().get(), + carry.length(), + size_); + util::handle_error(err, classname(), id_.get()); + + std::shared_ptr id(nullptr); + if (id_.get() != nullptr) { + id = id_.get()->getitem_carry_64(carry); + } + return std::shared_ptr(new RegularArray(id, content_.get()->carry(nextcarry), size_)); + } + + const std::pair RegularArray::minmax_depth() const { + std::pair content_depth = content_.get()->minmax_depth(); + return std::pair(content_depth.first + 1, content_depth.second + 1); + } + + const std::shared_ptr RegularArray::getitem_next(const SliceAt& at, const Slice& tail, const Index64& advanced) const { + assert(advanced.length() == 0); + + int64_t len = length(); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + Index64 nextcarry(len); + + struct Error err = awkward_regulararray_getitem_next_at_64( + nextcarry.ptr().get(), + at.at(), + len, + size_); + util::handle_error(err, classname(), id_.get()); + + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); + } + + const std::shared_ptr RegularArray::getitem_next(const SliceRange& range, const Slice& tail, const Index64& advanced) const { + int64_t len = length(); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + + assert(range.step() != 0); + int64_t regular_start = range.start(); + int64_t regular_stop = range.stop(); + int64_t regular_step = abs(range.step()); + awkward_regularize_rangeslice(®ular_start, ®ular_stop, range.step() > 0, range.start() != Slice::none(), range.stop() != Slice::none(), size_); + int64_t nextsize = 0; + if (range.step() > 0 && regular_stop - regular_start > 0) { + int64_t diff = regular_stop - regular_start; + nextsize = diff / regular_step; + if (diff % regular_step != 0) { + nextsize++; + } + } + else if (range.step() < 0 && regular_stop - regular_start < 0) { + int64_t diff = regular_start - regular_stop; + nextsize = diff / regular_step; + if (diff % regular_step != 0) { + nextsize++; + } + } + + Index64 nextcarry(len*nextsize); + + struct Error err = awkward_regulararray_getitem_next_range_64( + nextcarry.ptr().get(), + regular_start, + range.step(), + len, + size_, + nextsize); + util::handle_error(err, classname(), id_.get()); + + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + + if (advanced.length() == 0) { + return std::shared_ptr(new RegularArray(id_, nextcontent.get()->getitem_next(nexthead, nexttail, advanced), nextsize)); + } + else { + Index64 nextadvanced(len*nextsize); + + struct Error err = awkward_regulararray_getitem_next_range_spreadadvanced_64( + nextadvanced.ptr().get(), + advanced.ptr().get(), + len, + nextsize); + util::handle_error(err, classname(), id_.get()); + + return std::shared_ptr(new RegularArray(id_, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced), nextsize)); + } + } + + const std::shared_ptr RegularArray::getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const { + int64_t len = length(); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + Index64 flathead = array.ravel(); + Index64 regular_flathead(flathead.length()); + + struct Error err = awkward_regulararray_getitem_next_array_regularize_64( + regular_flathead.ptr().get(), + flathead.ptr().get(), + flathead.length(), + size_); + util::handle_error(err, classname(), id_.get()); + + if (advanced.length() == 0) { + Index64 nextcarry(len*flathead.length()); + Index64 nextadvanced(len*flathead.length()); + + struct Error err = awkward_regulararray_getitem_next_array_64( + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + regular_flathead.ptr().get(), + len, + regular_flathead.length(), + size_); + util::handle_error(err, classname(), id_.get()); + + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + return std::shared_ptr(new RegularArray(id_, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced), flathead.length())); + } + else { + Index64 nextcarry(len); + Index64 nextadvanced(len); + + struct Error err = awkward_regulararray_getitem_next_array_advanced_64( + nextcarry.ptr().get(), + nextadvanced.ptr().get(), + advanced.ptr().get(), + regular_flathead.ptr().get(), + len, + regular_flathead.length(), + size_); + util::handle_error(err, classname(), id_.get()); + + std::shared_ptr nextcontent = content_.get()->carry(nextcarry); + return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); + } + } + +} diff --git a/src/libawkward/fillable/FillableArray.cpp b/src/libawkward/fillable/FillableArray.cpp index 687e307acc..4c84b5503d 100644 --- a/src/libawkward/fillable/FillableArray.cpp +++ b/src/libawkward/fillable/FillableArray.cpp @@ -22,7 +22,7 @@ namespace awkward { } const std::shared_ptr FillableArray::type() const { - return std::shared_ptr(new ArrayType(fillable_.get()->length(), fillable_.get()->type())); + return std::shared_ptr(new ArrayType(fillable_.get()->type(), fillable_.get()->length())); } const std::shared_ptr FillableArray::snapshot() const { diff --git a/src/libawkward/io/root.cpp b/src/libawkward/io/root.cpp index be9e9902e4..37271b2ef1 100644 --- a/src/libawkward/io/root.cpp +++ b/src/libawkward/io/root.cpp @@ -29,7 +29,7 @@ namespace awkward { for (uint32_t i = 0; i < length; i++) { FromROOT_nestedvector_fill(levels, bytepos_tocopy, bytepos, rawdata, whichlevel + 1, itemsize); } - int64_t previous = levels[(unsigned int)whichlevel].getitem_at_unsafe(levels[(unsigned int)whichlevel].length() - 1); + int64_t previous = levels[(unsigned int)whichlevel].getitem_at_nowrap(levels[(unsigned int)whichlevel].length() - 1); levels[(unsigned int)whichlevel].append(previous + length); } } @@ -39,7 +39,7 @@ namespace awkward { assert(rawdata.ndim() == 1); Index64 level0(byteoffsets.length()); - level0.setitem_at_unsafe(0, 0); + level0.setitem_at_nowrap(0, 0); std::vector> levels; for (int64_t i = 0; i < depth; i++) { @@ -50,9 +50,9 @@ namespace awkward { GrowableBuffer bytepos_tocopy(options); for (int64_t i = 0; i < byteoffsets.length() - 1; i++) { - int64_t bytepos = byteoffsets.getitem_at_unsafe(i); + int64_t bytepos = byteoffsets.getitem_at_nowrap(i); FromROOT_nestedvector_fill(levels, bytepos_tocopy, bytepos, rawdata, 0, itemsize); - level0.setitem_at_unsafe(i + 1, levels[0].length()); + level0.setitem_at_nowrap(i + 1, levels[0].length()); } std::shared_ptr ptr(new uint8_t[(size_t)(bytepos_tocopy.length()*itemsize)], awkward::util::array_deleter()); @@ -60,7 +60,7 @@ namespace awkward { uint8_t* toptr = reinterpret_cast(ptr.get()); uint8_t* fromptr = reinterpret_cast(rawdata.ptr().get()); for (int64_t i = 0; i < bytepos_tocopy.length(); i++) { - ssize_t bytepos = (ssize_t)bytepos_tocopy.getitem_at_unsafe(i); + ssize_t bytepos = (ssize_t)bytepos_tocopy.getitem_at_nowrap(i); std::memcpy(&toptr[(ssize_t)(i*itemsize)], &fromptr[offset + bytepos], (size_t)itemsize); } diff --git a/src/libawkward/type/ArrayType.cpp b/src/libawkward/type/ArrayType.cpp index b420549b9d..0a8b50a040 100644 --- a/src/libawkward/type/ArrayType.cpp +++ b/src/libawkward/type/ArrayType.cpp @@ -10,7 +10,7 @@ namespace awkward { } const std::shared_ptr ArrayType::shallow_copy() const { - return std::shared_ptr(new ArrayType(length_, type_)); + return std::shared_ptr(new ArrayType(type_, length_)); } bool ArrayType::equal(std::shared_ptr other) const { diff --git a/src/libawkward/type/RegularType.cpp b/src/libawkward/type/RegularType.cpp index ec8d66708d..1df184a051 100644 --- a/src/libawkward/type/RegularType.cpp +++ b/src/libawkward/type/RegularType.cpp @@ -1,7 +1,6 @@ // BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE #include -#include #include "awkward/type/UnknownType.h" #include "awkward/type/OptionType.h" @@ -10,22 +9,16 @@ namespace awkward { std::string RegularType::tostring_part(std::string indent, std::string pre, std::string post) const { - std::stringstream out; - out << indent << pre; - for (auto x : shape_) { - out << x << " * "; - } - out << type_.get()->tostring_part(indent, "", "") << post; - return out.str(); + return indent + pre + std::to_string(size_) + std::string(" * ") + type_.get()->tostring_part(indent, "", "") + post; } const std::shared_ptr RegularType::shallow_copy() const { - return std::shared_ptr(new RegularType(shape_, type_)); + return std::shared_ptr(new RegularType(type_, size_)); } bool RegularType::equal(std::shared_ptr other) const { if (RegularType* t = dynamic_cast(other.get())) { - return shape() == t->shape() && type().get()->equal(t->type()); + return size() == t->size() && type().get()->equal(t->type()); } else { return false; @@ -40,18 +33,18 @@ namespace awkward { return compatible(dynamic_cast(other.get())->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); } else if (RegularType* t = dynamic_cast(other.get())) { - return shape_ == t->shape() && type_.get()->compatible(t->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); + return size() == t->size() && type_.get()->compatible(t->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); } else { return false; } } - const std::vector RegularType::shape() const { - return shape_; - } - const std::shared_ptr RegularType::type() const { return type_; } + + int64_t RegularType::size() const { + return size_; + } } diff --git a/src/pyawkward.cpp b/src/pyawkward.cpp index c010c552df..3fd62ae1b2 100644 --- a/src/pyawkward.cpp +++ b/src/pyawkward.cpp @@ -16,6 +16,7 @@ #include "awkward/array/ListArray.h" #include "awkward/array/ListOffsetArray.h" #include "awkward/array/EmptyArray.h" +#include "awkward/array/RegularArray.h" #include "awkward/fillable/FillableOptions.h" #include "awkward/fillable/FillableArray.h" #include "awkward/type/Type.h" @@ -109,6 +110,9 @@ py::object box(std::shared_ptr content) { else if (ak::EmptyArray* raw = dynamic_cast(content.get())) { return py::cast(*raw); } + else if (ak::RegularArray* raw = dynamic_cast(content.get())) { + return py::cast(*raw); + } else { throw std::runtime_error("missing boxer for Content subtype"); } @@ -194,6 +198,10 @@ std::shared_ptr unbox_content(py::object obj) { return obj.cast()->shallow_copy(); } catch (py::cast_error err) { } + try { + return obj.cast()->shallow_copy(); + } + catch (py::cast_error err) { } throw std::invalid_argument("content argument must be a Content subtype"); } @@ -559,9 +567,9 @@ py::class_> make_Type(py::handle m, std::str py::class_, ak::Type> make_ArrayType(py::handle m, std::string name) { return (py::class_, ak::Type>(m, name.c_str()) - .def(py::init>()) - .def("length", &ak::ArrayType::length) + .def(py::init, int64_t>()) .def("type", &ak::ArrayType::type) + .def("length", &ak::ArrayType::length) .def("__repr__", &ak::ArrayType::tostring) .def("__eq__", &ak::ArrayType::equal) .def("compatible", &ak::ArrayType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) @@ -625,9 +633,9 @@ py::class_, ak::Type> make py::class_, ak::Type> make_RegularType(py::handle m, std::string name) { return (py::class_, ak::Type>(m, name.c_str()) - .def(py::init, std::shared_ptr>()) - .def_property_readonly("shape", &ak::RegularType::shape) + .def(py::init, int64_t>()) .def_property_readonly("type", &ak::RegularType::type) + .def_property_readonly("size", &ak::RegularType::size) .def("__repr__", &ak::RegularType::tostring) .def("__eq__", &ak::RegularType::equal) .def("compatible", &ak::RegularType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) @@ -833,6 +841,21 @@ py::class_ make_EmptyArray(py::handle m, std::strin ); } +/////////////////////////////////////////////////////////////// RegularArray + +py::class_ make_RegularArray(py::handle m, std::string name) { + return content(py::class_(m, name.c_str()) + .def(py::init([](py::object content, int64_t size, py::object id) -> ak::RegularArray { + return ak::RegularArray(unbox_id(id), std::shared_ptr(unbox_content(content)), size); + }), py::arg("content"), py::arg("size"), py::arg("id") = py::none()) + + .def_property_readonly("size", &ak::RegularArray::size) + .def_property_readonly("content", [](ak::RegularArray& self) -> py::object { + return box(self.content()); + }) + ); +} + /////////////////////////////////////////////////////////////// module PYBIND11_MODULE(layout, m) { @@ -880,6 +903,8 @@ PYBIND11_MODULE(layout, m) { make_EmptyArray(m, "EmptyArray"); + make_RegularArray(m, "RegularArray"); + m.def("fromjson", [](std::string source, int64_t initial, double resize, int64_t buffersize) -> py::object { bool isarray = false; for (char const &x: source) { diff --git a/tests/test_PR016_finish_getitem_for_rawarray.cpp b/tests/test_PR016_finish_getitem_for_rawarray.cpp index d572bd0767..ef44b6c4a3 100644 --- a/tests/test_PR016_finish_getitem_for_rawarray.cpp +++ b/tests/test_PR016_finish_getitem_for_rawarray.cpp @@ -26,7 +26,7 @@ std::string tostring(std::shared_ptr array) { out << "["; for (int i = 0; i < listoffset->length(); i++) { if (i != 0) out << ", "; - out << tostring(listoffset->getitem_at_unsafe(i)); + out << tostring(listoffset->getitem_at_nowrap(i)); } out << "]"; return out.str(); @@ -36,7 +36,7 @@ std::string tostring(std::shared_ptr array) { out << "["; for (int i = 0; i < list->length(); i++) { if (i != 0) out << ", "; - out << tostring(list->getitem_at_unsafe(i)); + out << tostring(list->getitem_at_nowrap(i)); } out << "]"; return out.str(); diff --git a/tests/test_PR021_emptyarray.py b/tests/test_PR021_emptyarray.py index 58e0bbd360..357e84fab3 100644 --- a/tests/test_PR021_emptyarray.py +++ b/tests/test_PR021_emptyarray.py @@ -13,14 +13,14 @@ def test_unknown(): a = awkward1.fromjson("[[], [], []]") assert awkward1.tolist(a) == [[], [], []] assert str(awkward1.typeof(a)) == "3 * var * ???" - assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType()))) - assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")))) - assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64"))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.ListType(awkward1.layout.UnknownType()), 3)) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")), 3)) + assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.PrimitiveType("float64"), 3)) a = awkward1.fromjson("[[], [[], []], [[], [], []]]") assert awkward1.tolist(a) == [[], [[], []], [[], [], []]] assert str(awkward1.typeof(a)) == "3 * var * var * ???" - assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.ListType(awkward1.layout.UnknownType())))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.ListType(awkward1.layout.ListType(awkward1.layout.UnknownType())), 3)) a = awkward1.layout.FillableArray() a.beginlist() @@ -31,16 +31,16 @@ def test_unknown(): a.endlist() assert awkward1.tolist(a) == [[], [], []] assert str(awkward1.typeof(a)) == "3 * var * ???" - assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType()))) - assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")))) - assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64"))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.ListType(awkward1.layout.UnknownType()), 3)) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")), 3)) + assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.PrimitiveType("float64"), 3)) a = a.snapshot() assert awkward1.tolist(a) == [[], [], []] assert str(awkward1.typeof(a)) == "3 * var * ???" - assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType()))) - assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")))) - assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64"))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.ListType(awkward1.layout.UnknownType()), 3)) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")), 3)) + assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(awkward1.layout.PrimitiveType("float64"), 3)) def test_getitem(): a = awkward1.fromjson("[[], [[], []], [[], [], []]]") diff --git a/tests/test_PR023_regular_array.py b/tests/test_PR023_regular_array.py new file mode 100644 index 0000000000..b40a146698 --- /dev/null +++ b/tests/test_PR023_regular_array.py @@ -0,0 +1,291 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import sys +import itertools + +import pytest +import numpy + +import awkward1 + +content = awkward1.layout.NumpyArray(numpy.array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9])); +offsets = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6, 10, 10])) +listoffsetarray = awkward1.layout.ListOffsetArray64(offsets, content) +regulararray = awkward1.layout.RegularArray(listoffsetarray, 2) +starts = awkward1.layout.Index64(numpy.array([0, 1])) +stops = awkward1.layout.Index64(numpy.array([2, 3])) +listarray = awkward1.layout.ListArray64(starts, stops, regulararray) + +def test_type(): + assert str(awkward1.typeof(regulararray)) == "3 * 2 * var * float64" + +def test_iteration(): + assert awkward1.tolist(regulararray) == [[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + +def test_getitem_at(): + assert awkward1.tolist(regulararray[0]) == [[0.0, 1.1, 2.2], []] + assert awkward1.tolist(regulararray[1]) == [[3.3, 4.4], [5.5]] + assert awkward1.tolist(regulararray[2]) == [[6.6, 7.7, 8.8, 9.9], []] + +def test_getitem_range(): + assert awkward1.tolist(regulararray[1:]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(regulararray[:-1]) == [[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]] + +def test_getitem(): + assert awkward1.tolist(regulararray[(0,)]) == [[0.0, 1.1, 2.2], []] + assert awkward1.tolist(regulararray[(1,)]) == [[3.3, 4.4], [5.5]] + assert awkward1.tolist(regulararray[(2,)]) == [[6.6, 7.7, 8.8, 9.9], []] + assert awkward1.tolist(regulararray[(slice(1, None, None),)]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(regulararray[(slice(None, -1, None),)]) == [[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]] + +def test_getitem_deeper(): + assert awkward1.tolist(listarray) == [[[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]], [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]]] + + assert awkward1.tolist(listarray[0, 0, 0]) == [0.0, 1.1, 2.2] + assert awkward1.tolist(listarray[0, 0, 1]) == [] + assert awkward1.tolist(listarray[0, 1, 0]) == [3.3, 4.4] + assert awkward1.tolist(listarray[0, 1, 1]) == [5.5] + assert awkward1.tolist(listarray[1, 0, 0]) == [3.3, 4.4] + assert awkward1.tolist(listarray[1, 0, 1]) == [5.5] + assert awkward1.tolist(listarray[1, 1, 0]) == [6.6, 7.7, 8.8, 9.9] + assert awkward1.tolist(listarray[1, 1, 1]) == [] + + assert awkward1.tolist(listarray[0, 0, 0:]) == [[0.0, 1.1, 2.2], []] + assert awkward1.tolist(listarray[0, 0, 1:]) == [[]] + assert awkward1.tolist(listarray[0, 1, 0:]) == [[3.3, 4.4], [5.5]] + assert awkward1.tolist(listarray[0, 1, 1:]) == [[5.5]] + assert awkward1.tolist(listarray[1, 0, 0:]) == [[3.3, 4.4], [5.5]] + assert awkward1.tolist(listarray[1, 0, 1:]) == [[5.5]] + assert awkward1.tolist(listarray[1, 1, 0:]) == [[6.6, 7.7, 8.8, 9.9], []] + assert awkward1.tolist(listarray[1, 1, 1:]) == [[]] + + assert awkward1.tolist(listarray[[1], 0, 0:]) == [[[3.3, 4.4], [5.5]]] + assert awkward1.tolist(listarray[[1, 0], 0, 0:]) == [[[3.3, 4.4], [5.5]], [[0.0, 1.1, 2.2], []]] + + assert awkward1.tolist(listarray[:, :, [0, 1]]) == [[[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]], [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]]] + assert awkward1.tolist(listarray[:, :, [1, 0]]) == [[[[], [0.0, 1.1, 2.2]], [[5.5], [3.3, 4.4]]], [[[5.5], [3.3, 4.4]], [[], [6.6, 7.7, 8.8, 9.9]]]] + assert awkward1.tolist(listarray[:, :, [1, 0, 1]]) == [[[[], [0.0, 1.1, 2.2], []], [[5.5], [3.3, 4.4], [5.5]]], [[[5.5], [3.3, 4.4], [5.5]], [[], [6.6, 7.7, 8.8, 9.9], []]]] + assert awkward1.tolist(listarray[:, :2, [0, 1]]) == [[[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]], [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]]] + + assert awkward1.tolist(listarray[:1, [0, 0, 1, 1], [0, 1, 0, 1]]) == [[[0.0, 1.1, 2.2], [], [3.3, 4.4], [5.5]]] + assert awkward1.tolist(listarray[:1, [1, 1, 0, 0], [1, 0, 1, 0]]) == [[[5.5], [3.3, 4.4], [], [0.0, 1.1, 2.2]]] + +content2 = awkward1.layout.NumpyArray(numpy.arange(2*3*5*7).reshape(-1, 7)) +regulararrayA = awkward1.layout.RegularArray(content2, 5) +regulararrayB = awkward1.layout.RegularArray(regulararrayA, 3) +modelA = numpy.arange(2*3*5*7).reshape(2*3, 5, 7) +modelB = numpy.arange(2*3*5*7).reshape(2, 3, 5, 7) + +def test_numpy(): + assert awkward1.tolist(regulararrayA) == awkward1.tolist(modelA) + assert awkward1.tolist(regulararrayB) == awkward1.tolist(modelB) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((0, 1, 4, -5), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(regulararrayA[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(None), slice(1, None), slice(None, -1), slice(None, None, 2)), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(regulararrayA[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(1, None), slice(None, -1), 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(regulararrayA[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(regulararrayA[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], slice(1, None), slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(regulararrayA[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-2, -1, 0, 1, 1), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(regulararrayB[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, 1, slice(1, None), slice(None, -1)), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(regulararrayB[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, [1, 0, 0, 1], [0, 1, -1, 1], slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(regulararrayB[cuts]) + +def test_setid(): + regulararray.setid() + assert numpy.asarray(regulararray.id).tolist() == [ + [0], + [1], + [2]] + assert numpy.asarray(regulararray.content.id).tolist() == [ + [0, 0], + [0, 1], + [1, 0], + [1, 1], + [2, 0], + [2, 1]] + assert numpy.asarray(regulararray.content.content.id).tolist() == [ + [0, 0, 0], # 0.0 + [0, 0, 1], # 1.1 + [0, 0, 2], # 2.2 + # [0, 1, ], # (empty list) + [1, 0, 0], # 3.3 + [1, 0, 1], # 4.4 + [1, 1, 0], # 5.5 + [2, 0, 0], # 6.6 + [2, 0, 1], # 7.7 + [2, 0, 2], # 8.8 + [2, 0, 3]] # 9.9 + # [2, 1, ], # (empty list) + + regulararrayB.setid() + assert numpy.asarray(regulararrayB.id).tolist() == [ + [0], + [1]] + assert numpy.asarray(regulararrayB.content.id).tolist() == [ + [0, 0], + [0, 1], + [0, 2], + [1, 0], + [1, 1], + [1, 2]] + +numba = pytest.importorskip("numba") + +def test_numba(): + content = awkward1.layout.NumpyArray(numpy.array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9])); + offsets = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6, 10, 10])) + listoffsetarray = awkward1.layout.ListOffsetArray64(offsets, content) + regulararray = awkward1.layout.RegularArray(listoffsetarray, 2) + regulararray_m1 = awkward1.layout.RegularArray(listoffsetarray[:-1], 2) + regulararray_m2 = awkward1.layout.RegularArray(listoffsetarray[:-2], 2) + starts = awkward1.layout.Index64(numpy.array([0, 1])) + stops = awkward1.layout.Index64(numpy.array([2, 3])) + listarray = awkward1.layout.ListArray64(starts, stops, regulararray) + + for z in range(2): + @numba.njit + def f1(q): + return 3.14 + + assert sys.getrefcount(regulararray) == 2 + f1(regulararray) + assert sys.getrefcount(regulararray) == 2 + + @numba.njit + def f2(q): + return q + + assert sys.getrefcount(regulararray) == 2 + assert awkward1.tolist(f2(regulararray)) == awkward1.tolist(regulararray) + assert sys.getrefcount(regulararray) == 2 + + @numba.njit + def f3(q): + return len(q) + + assert f3(regulararray) == 3 + assert f3(regulararray_m1) == 2 + assert f3(regulararray_m2) == 2 + assert len(regulararray) == 3 + assert len(regulararray_m1) == 2 + assert len(regulararray_m2) == 2 + + @numba.njit + def f4(q): + return q[1] + + assert awkward1.tolist(f4(regulararray)) == [[3.3, 4.4], [5.5]] + + @numba.njit + def f5(q, i): + return q[i] + + assert awkward1.tolist(f5(regulararray, 1)) == [[3.3, 4.4], [5.5]] + + @numba.njit + def f6(q, i): + return q[i] + + assert awkward1.tolist(f6(regulararray, slice(1, None))) == awkward1.tolist(regulararray[slice(1, None)]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(f6(regulararray, slice(None, -1))) == awkward1.tolist(regulararray[slice(None, -1)]) == [[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]] + + @numba.njit + def f7(q, i): + return q[(i,)] + + assert awkward1.tolist(f7(regulararray, 1)) == [[3.3, 4.4], [5.5]] + + @numba.njit + def f8(q, i): + return q[:, i] + + assert awkward1.tolist(f8(listarray, 1)) == awkward1.tolist(listarray[:, 1]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + + @numba.njit + def f9(q): + return q[:, 1, 0] + + assert awkward1.tolist(listarray[:, 1, 0]) == awkward1.tolist(f9(listarray)) == [[3.3, 4.4], [6.6, 7.7, 8.8, 9.9]] + + @numba.njit + def f10(q, i): + return q[:, 1, i] + + assert awkward1.tolist(f10(listarray, slice(None, 1))) == awkward1.tolist(listarray[:, 1, slice(None, 1)]) == [[[3.3, 4.4]], [[6.6, 7.7, 8.8, 9.9]]] + assert awkward1.tolist(f10(listarray, slice(None, None))) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(f10(listarray, slice(None, None, -1))) == [[[5.5], [3.3, 4.4]], [[], [6.6, 7.7, 8.8, 9.9]]] + + @numba.njit + def f11(q, i): + return q[[0, 1], 1, i] + + assert awkward1.tolist(f11(listarray, slice(None, 1))) == awkward1.tolist(listarray[[0, 1], 1, slice(None, 1)]) == [[[3.3, 4.4]], [[6.6, 7.7, 8.8, 9.9]]] + + @numba.njit + def f12(q): + return q[:, 1:, [1, 0, 1]] + + assert awkward1.tolist(f12(listarray)) == awkward1.tolist(listarray[:, 1:, [1, 0, 1]]) == [[[[5.5], [3.3, 4.4], [5.5]]], [[[], [6.6, 7.7, 8.8, 9.9], []]]] + + @numba.njit + def f13(q): + return q[:, [1], [1, 0, 1]] + + assert awkward1.tolist(f13(listarray)) == awkward1.tolist(listarray[:, [1], [1, 0, 1]]) == [[[5.5], [3.3, 4.4], [5.5]], [[], [6.6, 7.7, 8.8, 9.9], []]] + + @numba.njit + def f14(q): + return q.content + + assert awkward1.tolist(f14(regulararray)) == awkward1.tolist(regulararray.content) + + @numba.njit + def f15(q): + return q.size + + assert awkward1.tolist(f15(regulararray)) == awkward1.tolist(regulararray.size) == 2 + + listarray.setid() + assert numpy.asarray(listarray.content.id).tolist() == [[0, 0], [1, 0], [1, 1]] + +# TODO: replace Content::getitem's promotion to ListArray with a promotion to RegularArray. +# TODO: ListArray's and ListOffsetArray's non-advanced getitem array should now output a RegularArray. +# TODO: all getitem arrays should handle non-flat SliceArray by wrapping in RegularArrays. +# TODO: check the FIXME in awkward_listarray_getitem_next_array_advanced. +# TODO: setid should not be allowed on data that can be reached by multiple paths (which will break the ListArray ids above, unfortunately).