diff --git a/README.md b/README.md index e7c498d595..0436af5bc1 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit- * [X] Fully implement `__getitem__` for int/slice/intarray/boolarray/tuple (placeholders for newaxis/ellipsis), with perfect agreement with [Numpy basic/advanced indexing](https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html), to all levels of depth. * [ ] Appendable arrays (a distinct phase from readable arrays, when the type is still in flux) to implement `awkward.fromiter` in C++. * [X] Implemented all types but records; tested all primitives and lists. - * [ ] Expose appendable arrays to Numba. + * [X] Expose appendable arrays to Numba. * [ ] Implement appendable records. * [ ] Test all (requires array types for all). * [X] JSON → Awkward via header-only [RapidJSON](https://rapidjson.org) and `awkward.fromiter`. @@ -106,7 +106,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit- * [ ] `awkward.tonumpy`: to force conversion to Numpy, if possible. Neither Layer 1 nor Layer 2 will have an `__array__` method; in the Numpy sense, they are not "array-like" or "array-compatible." * [ ] `awkward.topandas`: flattening jaggedness into `MultiIndex` rows and nested records into `MultiIndex` columns. This is distinct from the arrays' inheritance from Pandas, distinct from the natural ability to use any one of them as DataFrame columns. * [ ] `awkward.flatten`: same as old with an `axis` parameter. - * [ ] Reducers, such as `awkward.sum`, `awkward.max`, etc., supporing an `axis` method. + * [ ] Reducers, such as `awkward.sum`, `awkward.max`, etc., supporting an `axis` method. * [ ] The non-reducers: `awkward.moment`, `awkward.mean`, `awkward.var`, `awkward.std`. * [ ] `awkward.argmin`, `awkward.argmax`, `awkward.argsort`, and `awkward.sort`: same as old. * [ ] `awkward.where`: like `numpy.where`; old doesn't have this yet, but we'll need it. diff --git a/VERSION_INFO b/VERSION_INFO index 790629964e..7e72641b4b 100644 --- a/VERSION_INFO +++ b/VERSION_INFO @@ -1 +1 @@ -0.1.21 +0.1.22 diff --git a/awkward1/_numba/__init__.py b/awkward1/_numba/__init__.py index 73bd78400c..19e7293d00 100644 --- a/awkward1/_numba/__init__.py +++ b/awkward1/_numba/__init__.py @@ -7,10 +7,12 @@ else: installed = True import awkward1._numba.cpu + import awkward1._numba.libawkward import awkward1._numba.util import awkward1._numba.identity import awkward1._numba.content import awkward1._numba.iterator + import awkward1._numba.fillable import awkward1._numba.array.numpyarray import awkward1._numba.array.listarray import awkward1._numba.array.listoffsetarray diff --git a/awkward1/_numba/fillable.py b/awkward1/_numba/fillable.py new file mode 100644 index 0000000000..89e5d965fc --- /dev/null +++ b/awkward1/_numba/fillable.py @@ -0,0 +1,184 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import numba + +import awkward1.layout +from .._numba import libawkward, util + +@numba.extending.typeof_impl.register(awkward1.layout.FillableArray) +def typeof(val, c): + return FillableArrayType() + +class FillableArrayType(numba.types.Type): + def __init__(self): + super(FillableArrayType, self).__init__("FillableArrayType") + +@numba.typing.templates.infer_global(len) +class type_len(numba.typing.templates.AbstractTemplate): + def generic(self, args, kwargs): + if len(args) == 1 and len(kwargs) == 0: + arraytpe, = args + if isinstance(arraytpe, FillableArrayType): + return numba.typing.templates.signature(numba.types.intp, arraytpe) + +@numba.datamodel.registry.register_default(FillableArrayType) +class FillableArrayModel(numba.datamodel.models.StructModel): + def __init__(self, dmm, fe_type): + members = [("rawptr", numba.types.voidptr), + ("pyptr", numba.types.pyobject)] + super(FillableArrayModel, self).__init__(dmm, fe_type, members) + +@numba.extending.unbox(FillableArrayType) +def unbox(tpe, obj, c): + rawptr_obj = c.pyapi.object_getattr_string(obj, "_ptr") + proxyout = numba.cgutils.create_struct_proxy(tpe)(c.context, c.builder) + proxyout.rawptr = c.pyapi.long_as_voidptr(rawptr_obj) + proxyout.pyptr = obj + c.pyapi.decref(rawptr_obj) + is_error = numba.cgutils.is_not_null(c.builder, c.pyapi.err_occurred()) + return numba.extending.NativeValue(proxyout._getvalue(), is_error) + +@numba.extending.box(FillableArrayType) +def box(tpe, val, c): + proxyin = numba.cgutils.create_struct_proxy(tpe)(c.context, c.builder, value=val) + c.pyapi.incref(proxyin.pyptr) + return proxyin.pyptr + +def call(context, builder, fcn, args): + fcntpe = context.get_function_pointer_type(fcn.numbatpe) + fcnval = context.add_dynamic_addr(builder, fcn.numbatpe.get_pointer(fcn), info=fcn.name) + fcnptr = builder.bitcast(fcnval, fcntpe) + err = context.call_function_pointer(builder, fcnptr, args) + with builder.if_then(builder.icmp_unsigned("!=", err, context.get_constant(numba.uint8, 0)), likely=False): + context.call_conv.return_user_exc(builder, ValueError, (fcn.name + " failed",)) + +@numba.extending.lower_builtin(len, FillableArrayType) +def lower_len(context, builder, sig, args): + tpe, = sig.args + val, = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + result = numba.cgutils.alloca_once(builder, context.get_value_type(numba.int64)) + call(context, builder, libawkward.FillableArray_length, (proxyin.rawptr, result)) + return util.cast(context, builder, numba.int64, numba.intp, builder.load(result)) + +@numba.typing.templates.infer_getattr +class type_methods(numba.typing.templates.AttributeTemplate): + key = FillableArrayType + + @numba.typing.templates.bound_function("clear") + def resolve_clear(self, arraytpe, args, kwargs): + if len(args) == 0 and len(kwargs) == 0: + return numba.types.none() + else: + raise TypeError("wrong number of arguments for FillableArray.clear") + + @numba.typing.templates.bound_function("null") + def resolve_null(self, arraytpe, args, kwargs): + if len(args) == 0 and len(kwargs) == 0: + return numba.types.none() + else: + raise TypeError("wrong number of arguments for FillableArray.null") + + @numba.typing.templates.bound_function("boolean") + def resolve_boolean(self, arraytpe, args, kwargs): + if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], numba.types.Boolean): + return numba.types.none(args[0]) + else: + raise TypeError("wrong number or types of arguments for FillableArray.boolean") + + @numba.typing.templates.bound_function("integer") + def resolve_integer(self, arraytpe, args, kwargs): + if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], numba.types.Integer): + return numba.types.none(args[0]) + else: + raise TypeError("wrong number or types of arguments for FillableArray.integer") + + @numba.typing.templates.bound_function("real") + def resolve_real(self, arraytpe, args, kwargs): + if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], (numba.types.Integer, numba.types.Float)): + return numba.types.none(args[0]) + else: + raise TypeError("wrong number or types of arguments for FillableArray.real") + + @numba.typing.templates.bound_function("beginlist") + def resolve_beginlist(self, arraytpe, args, kwargs): + if len(args) == 0 and len(kwargs) == 0: + return numba.types.none() + else: + raise TypeError("wrong number of arguments for FillableArray.beginlist") + + @numba.typing.templates.bound_function("endlist") + def resolve_endlist(self, arraytpe, args, kwargs): + if len(args) == 0 and len(kwargs) == 0: + return numba.types.none() + else: + raise TypeError("wrong number of arguments for FillableArray.endlist") + +@numba.extending.lower_builtin("clear", FillableArrayType) +def lower_clear(context, builder, sig, args): + tpe, = sig.args + val, = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + call(context, builder, libawkward.FillableArray_clear, (proxyin.rawptr,)) + return context.get_dummy_value() + +@numba.extending.lower_builtin("null", FillableArrayType) +def lower_null(context, builder, sig, args): + tpe, = sig.args + val, = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + call(context, builder, libawkward.FillableArray_null, (proxyin.rawptr,)) + return context.get_dummy_value() + +@numba.extending.lower_builtin("boolean", FillableArrayType, numba.types.Boolean) +def lower_integer(context, builder, sig, args): + tpe, xtpe = sig.args + val, xval = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + x = builder.zext(xval, context.get_value_type(numba.uint8)) + call(context, builder, libawkward.FillableArray_boolean, (proxyin.rawptr, x)) + return context.get_dummy_value() + +@numba.extending.lower_builtin("integer", FillableArrayType, numba.types.Integer) +def lower_integer(context, builder, sig, args): + tpe, xtpe = sig.args + val, xval = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + x = util.cast(context, builder, xtpe, numba.int64, xval) + call(context, builder, libawkward.FillableArray_integer, (proxyin.rawptr, x)) + return context.get_dummy_value() + +@numba.extending.lower_builtin("real", FillableArrayType, numba.types.Integer) +@numba.extending.lower_builtin("real", FillableArrayType, numba.types.Float) +def lower_real(context, builder, sig, args): + tpe, xtpe = sig.args + val, xval = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + if isinstance(xtpe, numba.types.Integer) and xtpe.signed: + x = builder.sitofp(xval, context.get_value_type(numba.types.float64)) + elif isinstance(xtpe, numba.types.Integer): + x = builder.uitofp(xval, context.get_value_type(numba.types.float64)) + elif xtpe.bitwidth < 64: + x = builder.fpext(xval, context.get_value_type(numba.types.float64)) + elif xtpe.bitwidth > 64: + x = builder.fptrunc(xval, context.get_value_type(numba.types.float64)) + else: + x = xval + call(context, builder, libawkward.FillableArray_real, (proxyin.rawptr, x)) + return context.get_dummy_value() + +@numba.extending.lower_builtin("beginlist", FillableArrayType) +def lower_beginlist(context, builder, sig, args): + tpe, = sig.args + val, = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + call(context, builder, libawkward.FillableArray_beginlist, (proxyin.rawptr,)) + return context.get_dummy_value() + +@numba.extending.lower_builtin("endlist", FillableArrayType) +def lower_endlist(context, builder, sig, args): + tpe, = sig.args + val, = args + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + call(context, builder, libawkward.FillableArray_endlist, (proxyin.rawptr,)) + return context.get_dummy_value() diff --git a/awkward1/_numba/iterator.py b/awkward1/_numba/iterator.py index a1a0bdc695..b9a4b03f1a 100644 --- a/awkward1/_numba/iterator.py +++ b/awkward1/_numba/iterator.py @@ -1,8 +1,6 @@ # BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE -import numpy import numba -import numba.typing.arraydecl import awkward1.layout from .._numba import cpu, util, content diff --git a/awkward1/_numba/libawkward.py b/awkward1/_numba/libawkward.py new file mode 100644 index 0000000000..2d3723f35d --- /dev/null +++ b/awkward1/_numba/libawkward.py @@ -0,0 +1,75 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import os +import ctypes +import platform + +import numba +import numba.typing.ctypes_utils + +if platform.system() == "Windows": + libname = "awkward.dll" +elif platform.system() == "Darwin": + libname = "libawkward.dylib" +else: + libname = "libawkward.so" + +libpath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), libname) + +lib = ctypes.cdll.LoadLibrary(libpath) + +# bool awkward_FillableArray_length(void* fillablearray, int64_t* result); +FillableArray_length = lib.awkward_FillableArray_length +FillableArray_length.name = "FillableArray.length" +FillableArray_length.argtypes = [ctypes.c_voidp, ctypes.POINTER(ctypes.c_int64)] +FillableArray_length.restype = ctypes.c_uint8 +FillableArray_length.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_length) + +# bool awkward_FillableArray_clear(void* fillablearray); +FillableArray_clear = lib.awkward_FillableArray_clear +FillableArray_clear.name = "FillableArray.clear" +FillableArray_clear.argtypes = [ctypes.c_voidp] +FillableArray_clear.restype = ctypes.c_uint8 +FillableArray_clear.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_clear) + +# bool awkward_FillableArray_null(void* fillablearray); +FillableArray_null = lib.awkward_FillableArray_null +FillableArray_null.name = "FillableArray.null" +FillableArray_null.argtypes = [ctypes.c_voidp] +FillableArray_null.restype = ctypes.c_uint8 +FillableArray_null.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_null) + +# bool awkward_FillableArray_boolean(void* fillablearray, bool x); +FillableArray_boolean = lib.awkward_FillableArray_boolean +FillableArray_boolean.name = "FillableArray.boolean" +FillableArray_boolean.argtypes = [ctypes.c_voidp, ctypes.c_uint8] +FillableArray_boolean.restype = ctypes.c_uint8 +FillableArray_boolean.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_boolean) + +# bool awkward_FillableArray_integer(void* fillablearray, int64_t x); +FillableArray_integer = lib.awkward_FillableArray_integer +FillableArray_integer.name = "FillableArray.integer" +FillableArray_integer.argtypes = [ctypes.c_voidp, ctypes.c_int64] +FillableArray_integer.restype = ctypes.c_uint8 +FillableArray_integer.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_integer) + +# bool awkward_FillableArray_real(void* fillablearray, double x); +FillableArray_real = lib.awkward_FillableArray_real +FillableArray_real.name = "FillableArray.real" +FillableArray_real.argtypes = [ctypes.c_voidp, ctypes.c_double] +FillableArray_real.restype = ctypes.c_uint8 +FillableArray_real.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_real) + +# bool awkward_FillableArray_beginlist(void* fillablearray); +FillableArray_beginlist = lib.awkward_FillableArray_beginlist +FillableArray_beginlist.name = "FillableArray.beginlist" +FillableArray_beginlist.argtypes = [ctypes.c_voidp] +FillableArray_beginlist.restype = ctypes.c_uint8 +FillableArray_beginlist.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_beginlist) + +# bool awkward_FillableArray_endlist(void* fillablearray); +FillableArray_endlist = lib.awkward_FillableArray_endlist +FillableArray_endlist.name = "FillableArray.endlist" +FillableArray_endlist.argtypes = [ctypes.c_voidp] +FillableArray_endlist.restype = ctypes.c_uint8 +FillableArray_endlist.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_endlist) diff --git a/include/awkward/fillable/FillableArray.h b/include/awkward/fillable/FillableArray.h index be9d22c163..9c5d7e6734 100644 --- a/include/awkward/fillable/FillableArray.h +++ b/include/awkward/fillable/FillableArray.h @@ -23,7 +23,7 @@ namespace awkward { const std::shared_ptr getitem_at(int64_t at) const; const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; const std::shared_ptr getitem(const Slice& where) const; - + void null(); void boolean(bool x); void integer(int64_t x); @@ -49,4 +49,16 @@ namespace awkward { }; } +extern "C" { + uint8_t awkward_FillableArray_length(void* fillablearray, int64_t* result); + uint8_t awkward_FillableArray_clear(void* fillablearray); + + uint8_t awkward_FillableArray_null(void* fillablearray); + uint8_t awkward_FillableArray_boolean(void* fillablearray, bool x); + uint8_t awkward_FillableArray_integer(void* fillablearray, int64_t x); + uint8_t awkward_FillableArray_real(void* fillablearray, double x); + uint8_t awkward_FillableArray_beginlist(void* fillablearray); + uint8_t awkward_FillableArray_endlist(void* fillablearray); +} + #endif // AWKWARD_FILLABLE_H_ diff --git a/include/awkward/type/ArrayType.h b/include/awkward/type/ArrayType.h index 80b53870dd..09117acafb 100644 --- a/include/awkward/type/ArrayType.h +++ b/include/awkward/type/ArrayType.h @@ -13,6 +13,7 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; int64_t length() const; const std::shared_ptr type() const; diff --git a/include/awkward/type/ListType.h b/include/awkward/type/ListType.h index 7e08e67f7b..0b50ef8a35 100644 --- a/include/awkward/type/ListType.h +++ b/include/awkward/type/ListType.h @@ -13,6 +13,7 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; const std::shared_ptr type() const; diff --git a/include/awkward/type/OptionType.h b/include/awkward/type/OptionType.h index 6c198bf2f7..a8d28892b6 100644 --- a/include/awkward/type/OptionType.h +++ b/include/awkward/type/OptionType.h @@ -13,6 +13,7 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; const std::shared_ptr type() const; diff --git a/include/awkward/type/PrimitiveType.h b/include/awkward/type/PrimitiveType.h index fe1df954dc..f87698fb5c 100644 --- a/include/awkward/type/PrimitiveType.h +++ b/include/awkward/type/PrimitiveType.h @@ -28,6 +28,9 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; + + const DType dtype() const; private: const DType dtype_; diff --git a/include/awkward/type/RegularType.h b/include/awkward/type/RegularType.h index bcfc946dae..8eb82778ab 100644 --- a/include/awkward/type/RegularType.h +++ b/include/awkward/type/RegularType.h @@ -15,6 +15,7 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; const std::vector shape() const; const std::shared_ptr type() const; diff --git a/include/awkward/type/Type.h b/include/awkward/type/Type.h index 20d66090a5..3ba016c038 100644 --- a/include/awkward/type/Type.h +++ b/include/awkward/type/Type.h @@ -16,6 +16,7 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const = 0; virtual const std::shared_ptr shallow_copy() const = 0; virtual bool equal(std::shared_ptr other) const = 0; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const = 0; }; } diff --git a/include/awkward/type/UnionType.h b/include/awkward/type/UnionType.h index 7807f4b009..7cf13f7d33 100644 --- a/include/awkward/type/UnionType.h +++ b/include/awkward/type/UnionType.h @@ -15,6 +15,7 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; int64_t numtypes() const; const std::vector> types() const; diff --git a/include/awkward/type/UnknownType.h b/include/awkward/type/UnknownType.h index aace4c9cde..026984fad5 100644 --- a/include/awkward/type/UnknownType.h +++ b/include/awkward/type/UnknownType.h @@ -13,6 +13,7 @@ namespace awkward { virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const; virtual const std::shared_ptr shallow_copy() const; virtual bool equal(std::shared_ptr other) const; + virtual bool compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const; private: }; diff --git a/src/libawkward/fillable/FillableArray.cpp b/src/libawkward/fillable/FillableArray.cpp index 12e489bffb..687e307acc 100644 --- a/src/libawkward/fillable/FillableArray.cpp +++ b/src/libawkward/fillable/FillableArray.cpp @@ -75,3 +75,91 @@ namespace awkward { } } } + +uint8_t awkward_FillableArray_length(void* fillablearray, int64_t* result) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + *result = obj->length(); + } + catch (...) { + return 1; + } + return 0; +} + +uint8_t awkward_FillableArray_clear(void* fillablearray) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + obj->clear(); + } + catch (...) { + return 1; + } + return 0; +} + +uint8_t awkward_FillableArray_null(void* fillablearray) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + obj->null(); + } + catch (...) { + return 1; + } + return 0; +} + +uint8_t awkward_FillableArray_boolean(void* fillablearray, bool x) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + obj->boolean(x); + } + catch (...) { + return 1; + } + return 0; +} + +uint8_t awkward_FillableArray_integer(void* fillablearray, int64_t x) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + obj->integer(x); + } + catch (...) { + return 1; + } + return 0; +} + +uint8_t awkward_FillableArray_real(void* fillablearray, double x) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + obj->real(x); + } + catch (...) { + return 1; + } + return 0; +} + +uint8_t awkward_FillableArray_beginlist(void* fillablearray) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + obj->beginlist(); + } + catch (...) { + return 1; + } + return 0; +} + +uint8_t awkward_FillableArray_endlist(void* fillablearray) { + awkward::FillableArray* obj = reinterpret_cast(fillablearray); + try { + obj->endlist(); + } + catch (...) { + return 1; + } + return 0; +} diff --git a/src/libawkward/type/ArrayType.cpp b/src/libawkward/type/ArrayType.cpp index 8ed46d00f3..b420549b9d 100644 --- a/src/libawkward/type/ArrayType.cpp +++ b/src/libawkward/type/ArrayType.cpp @@ -15,7 +15,16 @@ namespace awkward { bool ArrayType::equal(std::shared_ptr other) const { if (ArrayType* t = dynamic_cast(other.get())) { - return type_.get()->equal(t->type_); // lengths DO NOT need to be equal (unlike RegularType) + return length_ == t->length_ && type_.get()->equal(t->type_); + } + else { + return false; + } + } + + bool ArrayType::compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const { + if (ArrayType* t = dynamic_cast(other.get())) { + return type_.get()->compatible(t->type_, bool_is_int, int_is_float, ignore_null, unknown_is_anything); // lengths DO NOT need to be equal (unlike RegularType) } else { return false; diff --git a/src/libawkward/type/ListType.cpp b/src/libawkward/type/ListType.cpp index 37746d1e73..5c2749ebf7 100644 --- a/src/libawkward/type/ListType.cpp +++ b/src/libawkward/type/ListType.cpp @@ -3,6 +3,7 @@ #include #include "awkward/type/UnknownType.h" +#include "awkward/type/OptionType.h" #include "awkward/type/ListType.h" @@ -16,11 +17,23 @@ namespace awkward { } bool ListType::equal(std::shared_ptr other) const { - if (UnknownType* t = dynamic_cast(other.get())) { + if (ListType* t = dynamic_cast(other.get())) { + return type().get()->equal(t->type()); + } + else { + return false; + } + } + + bool ListType::compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const { + if (unknown_is_anything && dynamic_cast(other.get())) { return true; } + else if (ignore_null && dynamic_cast(other.get())) { + return compatible(dynamic_cast(other.get())->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); + } else if (ListType* t = dynamic_cast(other.get())) { - return type().get()->equal(t->type()); + return type_.get()->compatible(t->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); } else { return false; diff --git a/src/libawkward/type/OptionType.cpp b/src/libawkward/type/OptionType.cpp index dea8e2d5e1..fcf8e6aa95 100644 --- a/src/libawkward/type/OptionType.cpp +++ b/src/libawkward/type/OptionType.cpp @@ -22,11 +22,23 @@ namespace awkward { } bool OptionType::equal(std::shared_ptr other) const { - if (UnknownType* t = dynamic_cast(other.get())) { + if (OptionType* t = dynamic_cast(other.get())) { + return type().get()->equal(t->type()); + } + else { + return false; + } + } + + bool OptionType::compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const { + if (unknown_is_anything && dynamic_cast(other.get())) { return true; } + else if (ignore_null) { + return type_.get()->compatible(other, bool_is_int, int_is_float, ignore_null, unknown_is_anything); + } else if (OptionType* t = dynamic_cast(other.get())) { - return type().get()->equal(t->type()); + return type_.get()->compatible(t->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); } else { return false; diff --git a/src/libawkward/type/PrimitiveType.cpp b/src/libawkward/type/PrimitiveType.cpp index e17f1a9767..c16f75437a 100644 --- a/src/libawkward/type/PrimitiveType.cpp +++ b/src/libawkward/type/PrimitiveType.cpp @@ -3,6 +3,7 @@ #include #include "awkward/type/UnknownType.h" +#include "awkward/type/OptionType.h" #include "awkward/type/PrimitiveType.h" @@ -31,14 +32,88 @@ namespace awkward { } bool PrimitiveType::equal(std::shared_ptr other) const { - if (UnknownType* t = dynamic_cast(other.get())) { + if (PrimitiveType* t = dynamic_cast(other.get())) { + return dtype_ == t->dtype_; + } + else { + return false; + } + } + + bool PrimitiveType::compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const { + if (unknown_is_anything && dynamic_cast(other.get())) { return true; } + else if (ignore_null && dynamic_cast(other.get())) { + return compatible(dynamic_cast(other.get())->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); + } else if (PrimitiveType* t = dynamic_cast(other.get())) { - return dtype_ == t->dtype_; + DType me = dtype_; + DType you = t->dtype(); + if (bool_is_int) { + if (me == boolean) { + me = int8; + } + if (you == boolean) { + you = int8; + } + } + if (int_is_float) { + if (me == int8 || me == int16 || me == int32 || me == int64 || me == uint8 || me == uint16 || me == uint32 || me == uint64) { + me = float64; + } + if (you == int8 || you == int16 || you == int32 || you == int64 || you == uint8 || you == uint16 || you == uint32 || you == uint64) { + you = float64; + } + } + switch (me) { + case boolean: + switch (you) { + case boolean: + return true; + default: + return false; + } + case int8: + case int16: + case int32: + case int64: + case uint8: + case uint16: + case uint32: + case uint64: + switch (you) { + case int8: + case int16: + case int32: + case int64: + case uint8: + case uint16: + case uint32: + case uint64: + return true; + default: + return false; + } + case float32: + case float64: + switch (you) { + case float32: + case float64: + return true; + default: + return false; + } + default: + return false; + } } else { return false; } } + + const PrimitiveType::DType PrimitiveType::dtype() const { + return dtype_; + } } diff --git a/src/libawkward/type/RegularType.cpp b/src/libawkward/type/RegularType.cpp index b31deed60b..ec8d66708d 100644 --- a/src/libawkward/type/RegularType.cpp +++ b/src/libawkward/type/RegularType.cpp @@ -4,6 +4,7 @@ #include #include "awkward/type/UnknownType.h" +#include "awkward/type/OptionType.h" #include "awkward/type/RegularType.h" @@ -23,11 +24,23 @@ namespace awkward { } bool RegularType::equal(std::shared_ptr other) const { - if (UnknownType* t = dynamic_cast(other.get())) { + if (RegularType* t = dynamic_cast(other.get())) { + return shape() == t->shape() && type().get()->equal(t->type()); + } + else { + return false; + } + } + + bool RegularType::compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const { + if (unknown_is_anything && dynamic_cast(other.get())) { return true; } + else if (ignore_null && dynamic_cast(other.get())) { + return compatible(dynamic_cast(other.get())->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); + } else if (RegularType* t = dynamic_cast(other.get())) { - return shape() == t->shape() && type().get()->equal(t->type()); + return shape_ == t->shape() && type_.get()->compatible(t->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); } else { return false; diff --git a/src/libawkward/type/UnionType.cpp b/src/libawkward/type/UnionType.cpp index d02e55cc8c..85eebd1f17 100644 --- a/src/libawkward/type/UnionType.cpp +++ b/src/libawkward/type/UnionType.cpp @@ -4,6 +4,7 @@ #include #include "awkward/type/UnknownType.h" +#include "awkward/type/OptionType.h" #include "awkward/type/UnionType.h" @@ -26,22 +27,63 @@ namespace awkward { } bool UnionType::equal(std::shared_ptr other) const { - if (UnknownType* t = dynamic_cast(other.get())) { + if (UnionType* t = dynamic_cast(other.get())) { + if (types_.size() != t->types_.size()) { + return false; + } + for (size_t i = 0; i < types_.size(); i++) { + if (!types_[i].get()->equal(t->types_[i])) { + return false; + } + } + return true; + } + else { + return false; + } + } + + bool UnionType::compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const { + if (unknown_is_anything && dynamic_cast(other.get())) { return true; } + else if (ignore_null && dynamic_cast(other.get())) { + return compatible(dynamic_cast(other.get())->type(), bool_is_int, int_is_float, ignore_null, unknown_is_anything); + } else if (UnionType* t = dynamic_cast(other.get())) { - if (numtypes() != t->numtypes()) { - return false; + for (auto me : types_) { + bool any = false; + for (auto you : t->types()) { + if (me.get()->compatible(you, bool_is_int, int_is_float, ignore_null, unknown_is_anything)) { + any = true; + break; + } + } + if (!any) { + return false; + } } - for (int64_t i = 0; i < numtypes(); i++) { - if (!type(i).get()->equal(t->type(i))) { + for (auto you : t->types()) { + bool any = false; + for (auto me : types_) { + if (you.get()->compatible(me, bool_is_int, int_is_float, ignore_null, unknown_is_anything)) { + any = true; + break; + } + } + if (!any) { return false; } } return true; } else { - return false; + for (auto me : types_) { + if (!me.get()->compatible(other, bool_is_int, int_is_float, ignore_null, unknown_is_anything)) { + return false; + } + } + return true; } } diff --git a/src/libawkward/type/UnknownType.cpp b/src/libawkward/type/UnknownType.cpp index 7bde53c764..1ce844f9b3 100644 --- a/src/libawkward/type/UnknownType.cpp +++ b/src/libawkward/type/UnknownType.cpp @@ -14,6 +14,20 @@ namespace awkward { } bool UnknownType::equal(std::shared_ptr other) const { - return true; + if (UnknownType* t = dynamic_cast(other.get())) { + return true; + } + else { + return false; + } + } + + bool UnknownType::compatible(std::shared_ptr other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const { + if (UnknownType* t = dynamic_cast(other.get())) { + return true; + } + else { + return unknown_is_anything; + } } } diff --git a/src/pyawkward.cpp b/src/pyawkward.cpp index c4a86493cd..c010c552df 100644 --- a/src/pyawkward.cpp +++ b/src/pyawkward.cpp @@ -526,6 +526,7 @@ py::class_ make_FillableArray(py::handle m, std::string name) .def(py::init([](int64_t initial, double resize) -> ak::FillableArray { return ak::FillableArray(ak::FillableOptions(initial, resize)); }), py::arg("initial") = 1024, py::arg("resize") = 2.0) + .def_property_readonly("_ptr", [](ak::FillableArray* self) -> size_t { return reinterpret_cast(self); }) .def("__repr__", &ak::FillableArray::tostring) .def("__len__", &ak::FillableArray::length) .def("clear", &ak::FillableArray::clear) @@ -563,6 +564,7 @@ py::class_, ak::Type> make_ArrayTy .def("type", &ak::ArrayType::type) .def("__repr__", &ak::ArrayType::tostring) .def("__eq__", &ak::ArrayType::equal) + .def("compatible", &ak::ArrayType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) ); } @@ -571,6 +573,7 @@ py::class_, ak::Type> make_Unk .def(py::init<>()) .def("__repr__", &ak::UnknownType::tostring) .def("__eq__", &ak::UnknownType::equal) + .def("compatible", &ak::UnknownType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) ); } @@ -616,6 +619,7 @@ py::class_, ak::Type> make })) .def("__repr__", &ak::PrimitiveType::tostring) .def("__eq__", &ak::PrimitiveType::equal) + .def("compatible", &ak::PrimitiveType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) ); } @@ -626,6 +630,7 @@ py::class_, ak::Type> make_Reg .def_property_readonly("type", &ak::RegularType::type) .def("__repr__", &ak::RegularType::tostring) .def("__eq__", &ak::RegularType::equal) + .def("compatible", &ak::RegularType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) ); } @@ -635,6 +640,7 @@ py::class_, ak::Type> make_ListType( .def_property_readonly("type", &ak::ListType::type) .def("__repr__", &ak::ListType::tostring) .def("__eq__", &ak::ListType::equal) + .def("compatible", &ak::ListType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) ); } @@ -644,6 +650,7 @@ py::class_, ak::Type> make_Optio .def_property_readonly("type", &ak::OptionType::type) .def("__repr__", &ak::OptionType::tostring) .def("__eq__", &ak::OptionType::equal) + .def("compatible", &ak::OptionType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) ); } @@ -667,6 +674,7 @@ py::class_, ak::Type> make_UnionTy .def("type", &ak::UnionType::type) .def("__repr__", &ak::UnionType::tostring) .def("__eq__", &ak::UnionType::equal) + .def("compatible", &ak::UnionType::compatible, py::arg("other"), py::arg("bool_is_int") = false, py::arg("int_is_float") = false, py::arg("ignore_null") = true, py::arg("unknown_is_anything") = true) ); } diff --git a/tests/test_PR018_fromiter_fillable.py b/tests/test_PR018_fromiter_fillable.py index d821d0f02a..9cdf2fecaf 100644 --- a/tests/test_PR018_fromiter_fillable.py +++ b/tests/test_PR018_fromiter_fillable.py @@ -31,20 +31,20 @@ def test_types(): assert [repr(x) for x in t3.types] == ["int32", "float64"] assert repr(t4.type) == "int32" assert repr(t5.type) == "var * int32" - assert t0 == t1 - assert t0 == t2 - assert t0 == t3 - assert t0 == t4 - assert t0 == t5 - assert t0 == t6 - assert t1 == t0 - assert t2 == t0 - assert t3 == t0 - assert t4 == t0 - assert t5 == t0 - assert t6 == t0 - assert t4 == t4b - assert t4 != t5 + assert t0.compatible(t1) + assert t0.compatible(t2) + assert t0.compatible(t3) + assert t0.compatible(t4) + assert t0.compatible(t5) + assert t0.compatible(t6) + assert t1.compatible(t0) + assert t2.compatible(t0) + assert t3.compatible(t0) + assert t4.compatible(t0) + assert t5.compatible(t0) + assert t6.compatible(t0) + assert t4.compatible(t4b) + assert not t4.compatible(t5) def test_boolean(): a = awkward1.layout.FillableArray() diff --git a/tests/test_PR021_emptyarray.py b/tests/test_PR021_emptyarray.py index fce9d19e01..58e0bbd360 100644 --- a/tests/test_PR021_emptyarray.py +++ b/tests/test_PR021_emptyarray.py @@ -13,14 +13,14 @@ def test_unknown(): a = awkward1.fromjson("[[], [], []]") assert awkward1.tolist(a) == [[], [], []] assert str(awkward1.typeof(a)) == "3 * var * ???" - assert awkward1.typeof(a) == awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType())) - assert awkward1.typeof(a) == awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64"))) - assert awkward1.typeof(a) != awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64")) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType()))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")))) + assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64"))) a = awkward1.fromjson("[[], [[], []], [[], [], []]]") assert awkward1.tolist(a) == [[], [[], []], [[], [], []]] assert str(awkward1.typeof(a)) == "3 * var * var * ???" - assert awkward1.typeof(a) == awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.ListType(awkward1.layout.UnknownType()))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.ListType(awkward1.layout.UnknownType())))) a = awkward1.layout.FillableArray() a.beginlist() @@ -31,16 +31,16 @@ def test_unknown(): a.endlist() assert awkward1.tolist(a) == [[], [], []] assert str(awkward1.typeof(a)) == "3 * var * ???" - assert awkward1.typeof(a) == awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType())) - assert awkward1.typeof(a) == awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64"))) - assert awkward1.typeof(a) != awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64")) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType()))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")))) + assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64"))) a = a.snapshot() assert awkward1.tolist(a) == [[], [], []] assert str(awkward1.typeof(a)) == "3 * var * ???" - assert awkward1.typeof(a) == awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType())) - assert awkward1.typeof(a) == awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64"))) - assert awkward1.typeof(a) != awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64")) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.UnknownType()))) + assert awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")))) + assert not awkward1.typeof(a).compatible(awkward1.layout.ArrayType(3, awkward1.layout.PrimitiveType("float64"))) def test_getitem(): a = awkward1.fromjson("[[], [[], []], [[], [], []]]") diff --git a/tests/test_PR022_fillablearray_in_numba.py b/tests/test_PR022_fillablearray_in_numba.py new file mode 100644 index 0000000000..c534f334b9 --- /dev/null +++ b/tests/test_PR022_fillablearray_in_numba.py @@ -0,0 +1,123 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import sys +import os +import json + +import pytest +import numpy + +import awkward1 + +numba = pytest.importorskip("numba") + +def test_boxing(): + @numba.njit + def f1(q): + z = q + return 3.14 + + a = awkward1.layout.FillableArray() + assert sys.getrefcount(a) == 2 + f1(a) + assert sys.getrefcount(a) == 2 + + @numba.njit + def f2(q): + z = q + return q + + a = awkward1.layout.FillableArray() + assert sys.getrefcount(a) == 2 + f2(a) + assert sys.getrefcount(a) == 2 + b = f2(a) + assert sys.getrefcount(a) == 3 + + assert str(b.snapshot()) == "" + +def test_simple(): + @numba.njit + def f1(q): + q.clear() + return 3.14 + + a = awkward1.layout.FillableArray() + f1(a) + +def test_boolean(): + @numba.njit + def f1(q): + q.boolean(True) + q.boolean(False) + q.boolean(False) + return q + + a = awkward1.layout.FillableArray() + b = f1(a) + assert awkward1.tolist(a.snapshot()) == [True, False, False] + assert awkward1.tolist(b.snapshot()) == [True, False, False] + +def test_integer(): + @numba.njit + def f1(q): + q.integer(1) + q.integer(2) + q.integer(3) + return q + + a = awkward1.layout.FillableArray() + b = f1(a) + assert awkward1.tolist(a.snapshot()) == [1, 2, 3] + assert awkward1.tolist(b.snapshot()) == [1, 2, 3] + +def test_real(): + @numba.njit + def f1(q, z): + q.real(1) + q.real(2.2) + q.real(z) + return q + + a = awkward1.layout.FillableArray() + b = f1(a, numpy.array([3.5], dtype=numpy.float32)[0]) + assert awkward1.tolist(a.snapshot()) == [1, 2.2, 3.5] + assert awkward1.tolist(b.snapshot()) == [1, 2.2, 3.5] + +def test_list(): + @numba.njit + def f1(q): + q.beginlist() + q.real(1.1) + q.real(2.2) + q.real(3.3) + q.endlist() + q.beginlist() + q.endlist() + q.beginlist() + q.real(4.4) + q.real(5.5) + q.endlist() + return q + + a = awkward1.layout.FillableArray() + b = f1(a) + assert awkward1.tolist(a.snapshot()) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] + assert awkward1.tolist(b.snapshot()) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] + + @numba.njit + def f2(q): + return len(q) + + assert f2(a) == 3 + assert f2(b) == 3 + + @numba.njit + def f3(q): + q.clear() + return q + + c = f3(b) + assert awkward1.tolist(a.snapshot()) == [] + assert awkward1.tolist(b.snapshot()) == [] + assert awkward1.tolist(c.snapshot()) == []