FillableArrays must be usable in Numba. (#22)

* Start work.

* Pure-C hooks into FillableArray compile.

* And start writing fillable for Numba.

* Change Type::equal to Type::compatible to better specify what it does.

* Added (true) Type::equal and parameterized Type::compatible.

* Add FillableArrayType for Numba.

* Implemented boxing for FillableArray.

* Can call FillableArray.clear.

* [skip ci] Implemented but have not tested FillableArray.integer.

* All FillableArray methods work in Numba.

* FillableArrays in Numba is finished.

* Fix 32-bit error.

README.md

@@ -60,7 +60,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit-
    * [X] Fully implement `__getitem__` for int/slice/intarray/boolarray/tuple (placeholders for newaxis/ellipsis), with perfect agreement with [Numpy basic/advanced indexing](https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html), to all levels of depth.
    * [ ] Appendable arrays (a distinct phase from readable arrays, when the type is still in flux) to implement `awkward.fromiter` in C++.
       * [X] Implemented all types but records; tested all primitives and lists.
-      * [ ] Expose appendable arrays to Numba.
+      * [X] Expose appendable arrays to Numba.
       * [ ] Implement appendable records.
       * [ ] Test all (requires array types for all).
    * [X] JSON → Awkward via header-only [RapidJSON](https://rapidjson.org) and `awkward.fromiter`.
@@ -106,7 +106,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit-
       * [ ] `awkward.tonumpy`: to force conversion to Numpy, if possible. Neither Layer 1 nor Layer 2 will have an `__array__` method; in the Numpy sense, they are not "array-like" or "array-compatible."
       * [ ] `awkward.topandas`: flattening jaggedness into `MultiIndex` rows and nested records into `MultiIndex` columns. This is distinct from the arrays' inheritance from Pandas, distinct from the natural ability to use any one of them as DataFrame columns.
       * [ ] `awkward.flatten`: same as old with an `axis` parameter.
-      * [ ] Reducers, such as `awkward.sum`, `awkward.max`, etc., supporing an `axis` method.
+      * [ ] Reducers, such as `awkward.sum`, `awkward.max`, etc., supporting an `axis` method.
       * [ ] The non-reducers: `awkward.moment`, `awkward.mean`, `awkward.var`, `awkward.std`.
       * [ ] `awkward.argmin`, `awkward.argmax`, `awkward.argsort`, and `awkward.sort`: same as old.
       * [ ] `awkward.where`: like `numpy.where`; old doesn't have this yet, but we'll need it.

VERSION_INFO

@@ -1 +1 @@
-0.1.21
+0.1.22

awkward1/_numba/__init__.py

@@ -7,10 +7,12 @@
 else:
     installed = True
     import awkward1._numba.cpu
+    import awkward1._numba.libawkward
     import awkward1._numba.util
     import awkward1._numba.identity
     import awkward1._numba.content
     import awkward1._numba.iterator
+    import awkward1._numba.fillable
     import awkward1._numba.array.numpyarray
     import awkward1._numba.array.listarray
     import awkward1._numba.array.listoffsetarray

awkward1/_numba/fillable.py

@@ -0,0 +1,184 @@
+# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE
+
+import numba
+
+import awkward1.layout
+from .._numba import libawkward, util
+
+@numba.extending.typeof_impl.register(awkward1.layout.FillableArray)
+def typeof(val, c):
+    return FillableArrayType()
+
+class FillableArrayType(numba.types.Type):
+    def __init__(self):
+        super(FillableArrayType, self).__init__("FillableArrayType")
+
+@numba.typing.templates.infer_global(len)
+class type_len(numba.typing.templates.AbstractTemplate):
+    def generic(self, args, kwargs):
+        if len(args) == 1 and len(kwargs) == 0:
+            arraytpe, = args
+            if isinstance(arraytpe, FillableArrayType):
+                return numba.typing.templates.signature(numba.types.intp, arraytpe)
+
+@numba.datamodel.registry.register_default(FillableArrayType)
+class FillableArrayModel(numba.datamodel.models.StructModel):
+    def __init__(self, dmm, fe_type):
+        members = [("rawptr", numba.types.voidptr),
+                   ("pyptr", numba.types.pyobject)]
+        super(FillableArrayModel, self).__init__(dmm, fe_type, members)
+
+@numba.extending.unbox(FillableArrayType)
+def unbox(tpe, obj, c):
+    rawptr_obj = c.pyapi.object_getattr_string(obj, "_ptr")
+    proxyout = numba.cgutils.create_struct_proxy(tpe)(c.context, c.builder)
+    proxyout.rawptr = c.pyapi.long_as_voidptr(rawptr_obj)
+    proxyout.pyptr = obj
+    c.pyapi.decref(rawptr_obj)
+    is_error = numba.cgutils.is_not_null(c.builder, c.pyapi.err_occurred())
+    return numba.extending.NativeValue(proxyout._getvalue(), is_error)
+
+@numba.extending.box(FillableArrayType)
+def box(tpe, val, c):
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(c.context, c.builder, value=val)
+    c.pyapi.incref(proxyin.pyptr)
+    return proxyin.pyptr
+
+def call(context, builder, fcn, args):
+    fcntpe = context.get_function_pointer_type(fcn.numbatpe)
+    fcnval = context.add_dynamic_addr(builder, fcn.numbatpe.get_pointer(fcn), info=fcn.name)
+    fcnptr = builder.bitcast(fcnval, fcntpe)
+    err = context.call_function_pointer(builder, fcnptr, args)
+    with builder.if_then(builder.icmp_unsigned("!=", err, context.get_constant(numba.uint8, 0)), likely=False):
+        context.call_conv.return_user_exc(builder, ValueError, (fcn.name + " failed",))
+
+@numba.extending.lower_builtin(len, FillableArrayType)
+def lower_len(context, builder, sig, args):
+    tpe, = sig.args
+    val, = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    result = numba.cgutils.alloca_once(builder, context.get_value_type(numba.int64))
+    call(context, builder, libawkward.FillableArray_length, (proxyin.rawptr, result))
+    return util.cast(context, builder, numba.int64, numba.intp, builder.load(result))
+
+@numba.typing.templates.infer_getattr
+class type_methods(numba.typing.templates.AttributeTemplate):
+    key = FillableArrayType
+
+    @numba.typing.templates.bound_function("clear")
+    def resolve_clear(self, arraytpe, args, kwargs):
+        if len(args) == 0 and len(kwargs) == 0:
+            return numba.types.none()
+        else:
+            raise TypeError("wrong number of arguments for FillableArray.clear")
+
+    @numba.typing.templates.bound_function("null")
+    def resolve_null(self, arraytpe, args, kwargs):
+        if len(args) == 0 and len(kwargs) == 0:
+            return numba.types.none()
+        else:
+            raise TypeError("wrong number of arguments for FillableArray.null")
+
+    @numba.typing.templates.bound_function("boolean")
+    def resolve_boolean(self, arraytpe, args, kwargs):
+        if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], numba.types.Boolean):
+            return numba.types.none(args[0])
+        else:
+            raise TypeError("wrong number or types of arguments for FillableArray.boolean")
+
+    @numba.typing.templates.bound_function("integer")
+    def resolve_integer(self, arraytpe, args, kwargs):
+        if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], numba.types.Integer):
+            return numba.types.none(args[0])
+        else:
+            raise TypeError("wrong number or types of arguments for FillableArray.integer")
+
+    @numba.typing.templates.bound_function("real")
+    def resolve_real(self, arraytpe, args, kwargs):
+        if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], (numba.types.Integer, numba.types.Float)):
+            return numba.types.none(args[0])
+        else:
+            raise TypeError("wrong number or types of arguments for FillableArray.real")
+
+    @numba.typing.templates.bound_function("beginlist")
+    def resolve_beginlist(self, arraytpe, args, kwargs):
+        if len(args) == 0 and len(kwargs) == 0:
+            return numba.types.none()
+        else:
+            raise TypeError("wrong number of arguments for FillableArray.beginlist")
+
+    @numba.typing.templates.bound_function("endlist")
+    def resolve_endlist(self, arraytpe, args, kwargs):
+        if len(args) == 0 and len(kwargs) == 0:
+            return numba.types.none()
+        else:
+            raise TypeError("wrong number of arguments for FillableArray.endlist")
+
+@numba.extending.lower_builtin("clear", FillableArrayType)
+def lower_clear(context, builder, sig, args):
+    tpe, = sig.args
+    val, = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    call(context, builder, libawkward.FillableArray_clear, (proxyin.rawptr,))
+    return context.get_dummy_value()
+
+@numba.extending.lower_builtin("null", FillableArrayType)
+def lower_null(context, builder, sig, args):
+    tpe, = sig.args
+    val, = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    call(context, builder, libawkward.FillableArray_null, (proxyin.rawptr,))
+    return context.get_dummy_value()
+
+@numba.extending.lower_builtin("boolean", FillableArrayType, numba.types.Boolean)
+def lower_integer(context, builder, sig, args):
+    tpe, xtpe = sig.args
+    val, xval = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    x = builder.zext(xval, context.get_value_type(numba.uint8))
+    call(context, builder, libawkward.FillableArray_boolean, (proxyin.rawptr, x))
+    return context.get_dummy_value()
+
+@numba.extending.lower_builtin("integer", FillableArrayType, numba.types.Integer)
+def lower_integer(context, builder, sig, args):
+    tpe, xtpe = sig.args
+    val, xval = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    x = util.cast(context, builder, xtpe, numba.int64, xval)
+    call(context, builder, libawkward.FillableArray_integer, (proxyin.rawptr, x))
+    return context.get_dummy_value()
+
+@numba.extending.lower_builtin("real", FillableArrayType, numba.types.Integer)
+@numba.extending.lower_builtin("real", FillableArrayType, numba.types.Float)
+def lower_real(context, builder, sig, args):
+    tpe, xtpe = sig.args
+    val, xval = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    if isinstance(xtpe, numba.types.Integer) and xtpe.signed:
+        x = builder.sitofp(xval, context.get_value_type(numba.types.float64))
+    elif isinstance(xtpe, numba.types.Integer):
+        x = builder.uitofp(xval, context.get_value_type(numba.types.float64))
+    elif xtpe.bitwidth < 64:
+        x = builder.fpext(xval, context.get_value_type(numba.types.float64))
+    elif xtpe.bitwidth > 64:
+        x = builder.fptrunc(xval, context.get_value_type(numba.types.float64))
+    else:
+        x = xval
+    call(context, builder, libawkward.FillableArray_real, (proxyin.rawptr, x))
+    return context.get_dummy_value()
+
+@numba.extending.lower_builtin("beginlist", FillableArrayType)
+def lower_beginlist(context, builder, sig, args):
+    tpe, = sig.args
+    val, = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    call(context, builder, libawkward.FillableArray_beginlist, (proxyin.rawptr,))
+    return context.get_dummy_value()
+
+@numba.extending.lower_builtin("endlist", FillableArrayType)
+def lower_endlist(context, builder, sig, args):
+    tpe, = sig.args
+    val, = args
+    proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+    call(context, builder, libawkward.FillableArray_endlist, (proxyin.rawptr,))
+    return context.get_dummy_value()

awkward1/_numba/iterator.py

@@ -1,8 +1,6 @@
 # BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE
 
-import numpy
 import numba
-import numba.typing.arraydecl
 
 import awkward1.layout
 from .._numba import cpu, util, content

awkward1/_numba/libawkward.py

@@ -0,0 +1,75 @@
+# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE
+
+import os
+import ctypes
+import platform
+
+import numba
+import numba.typing.ctypes_utils
+
+if platform.system() == "Windows":
+    libname = "awkward.dll"
+elif platform.system() == "Darwin":
+    libname = "libawkward.dylib"
+else:
+    libname = "libawkward.so"
+
+libpath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), libname)
+
+lib = ctypes.cdll.LoadLibrary(libpath)
+
+# bool awkward_FillableArray_length(void* fillablearray, int64_t* result);
+FillableArray_length = lib.awkward_FillableArray_length
+FillableArray_length.name = "FillableArray.length"
+FillableArray_length.argtypes = [ctypes.c_voidp, ctypes.POINTER(ctypes.c_int64)]
+FillableArray_length.restype  = ctypes.c_uint8
+FillableArray_length.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_length)
+
+# bool awkward_FillableArray_clear(void* fillablearray);
+FillableArray_clear = lib.awkward_FillableArray_clear
+FillableArray_clear.name = "FillableArray.clear"
+FillableArray_clear.argtypes = [ctypes.c_voidp]
+FillableArray_clear.restype  = ctypes.c_uint8
+FillableArray_clear.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_clear)
+
+# bool awkward_FillableArray_null(void* fillablearray);
+FillableArray_null = lib.awkward_FillableArray_null
+FillableArray_null.name = "FillableArray.null"
+FillableArray_null.argtypes = [ctypes.c_voidp]
+FillableArray_null.restype  = ctypes.c_uint8
+FillableArray_null.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_null)
+
+# bool awkward_FillableArray_boolean(void* fillablearray, bool x);
+FillableArray_boolean = lib.awkward_FillableArray_boolean
+FillableArray_boolean.name = "FillableArray.boolean"
+FillableArray_boolean.argtypes = [ctypes.c_voidp, ctypes.c_uint8]
+FillableArray_boolean.restype  = ctypes.c_uint8
+FillableArray_boolean.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_boolean)
+
+# bool awkward_FillableArray_integer(void* fillablearray, int64_t x);
+FillableArray_integer = lib.awkward_FillableArray_integer
+FillableArray_integer.name = "FillableArray.integer"
+FillableArray_integer.argtypes = [ctypes.c_voidp, ctypes.c_int64]
+FillableArray_integer.restype  = ctypes.c_uint8
+FillableArray_integer.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_integer)
+
+# bool awkward_FillableArray_real(void* fillablearray, double x);
+FillableArray_real = lib.awkward_FillableArray_real
+FillableArray_real.name = "FillableArray.real"
+FillableArray_real.argtypes = [ctypes.c_voidp, ctypes.c_double]
+FillableArray_real.restype  = ctypes.c_uint8
+FillableArray_real.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_real)
+
+# bool awkward_FillableArray_beginlist(void* fillablearray);
+FillableArray_beginlist = lib.awkward_FillableArray_beginlist
+FillableArray_beginlist.name = "FillableArray.beginlist"
+FillableArray_beginlist.argtypes = [ctypes.c_voidp]
+FillableArray_beginlist.restype  = ctypes.c_uint8
+FillableArray_beginlist.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_beginlist)
+
+# bool awkward_FillableArray_endlist(void* fillablearray);
+FillableArray_endlist = lib.awkward_FillableArray_endlist
+FillableArray_endlist.name = "FillableArray.endlist"
+FillableArray_endlist.argtypes = [ctypes.c_voidp]
+FillableArray_endlist.restype  = ctypes.c_uint8
+FillableArray_endlist.numbatpe = numba.typing.ctypes_utils.make_function_type(FillableArray_endlist)

include/awkward/fillable/FillableArray.h

@@ -23,7 +23,7 @@ namespace awkward {
     const std::shared_ptr<Content> getitem_at(int64_t at) const;
     const std::shared_ptr<Content> getitem_range(int64_t start, int64_t stop) const;
     const std::shared_ptr<Content> getitem(const Slice& where) const;
-    
+
     void null();
     void boolean(bool x);
     void integer(int64_t x);
@@ -49,4 +49,16 @@ namespace awkward {
   };
 }
 
+extern "C" {
+  uint8_t awkward_FillableArray_length(void* fillablearray, int64_t* result);
+  uint8_t awkward_FillableArray_clear(void* fillablearray);
+
+  uint8_t awkward_FillableArray_null(void* fillablearray);
+  uint8_t awkward_FillableArray_boolean(void* fillablearray, bool x);
+  uint8_t awkward_FillableArray_integer(void* fillablearray, int64_t x);
+  uint8_t awkward_FillableArray_real(void* fillablearray, double x);
+  uint8_t awkward_FillableArray_beginlist(void* fillablearray);
+  uint8_t awkward_FillableArray_endlist(void* fillablearray);
+}
+
 #endif // AWKWARD_FILLABLE_H_

include/awkward/type/ArrayType.h

@@ -13,6 +13,7 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const;
     virtual const std::shared_ptr<Type> shallow_copy() const;
     virtual bool equal(std::shared_ptr<Type> other) const;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const;
 
     int64_t length() const;
     const std::shared_ptr<Type> type() const;

include/awkward/type/ListType.h

@@ -13,6 +13,7 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const;
     virtual const std::shared_ptr<Type> shallow_copy() const;
     virtual bool equal(std::shared_ptr<Type> other) const;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const;
 
   const std::shared_ptr<Type> type() const;
 

include/awkward/type/OptionType.h

@@ -13,6 +13,7 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const;
     virtual const std::shared_ptr<Type> shallow_copy() const;
     virtual bool equal(std::shared_ptr<Type> other) const;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const;
 
   const std::shared_ptr<Type> type() const;
 

include/awkward/type/PrimitiveType.h

@@ -28,6 +28,9 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const;
     virtual const std::shared_ptr<Type> shallow_copy() const;
     virtual bool equal(std::shared_ptr<Type> other) const;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const;
+
+  const DType dtype() const;
 
   private:
     const DType dtype_;

include/awkward/type/RegularType.h

@@ -15,6 +15,7 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const;
     virtual const std::shared_ptr<Type> shallow_copy() const;
     virtual bool equal(std::shared_ptr<Type> other) const;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const;
 
     const std::vector<int64_t> shape() const;
     const std::shared_ptr<Type> type() const;

include/awkward/type/Type.h

@@ -16,6 +16,7 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const = 0;
     virtual const std::shared_ptr<Type> shallow_copy() const = 0;
     virtual bool equal(std::shared_ptr<Type> other) const = 0;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const = 0;
   };
 }
 

include/awkward/type/UnionType.h

@@ -15,6 +15,7 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const;
     virtual const std::shared_ptr<Type> shallow_copy() const;
     virtual bool equal(std::shared_ptr<Type> other) const;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const;
 
     int64_t numtypes() const;
     const std::vector<std::shared_ptr<Type>> types() const;

include/awkward/type/UnknownType.h

@@ -13,6 +13,7 @@ namespace awkward {
     virtual std::string tostring_part(std::string indent, std::string pre, std::string post) const;
     virtual const std::shared_ptr<Type> shallow_copy() const;
     virtual bool equal(std::shared_ptr<Type> other) const;
+    virtual bool compatible(std::shared_ptr<Type> other, bool bool_is_int, bool int_is_float, bool ignore_null, bool unknown_is_anything) const;
 
   private:
   };