Replace 'lookup' and 'reverselookup' with a single property (that act…

…s like 'reverselookup'). (#37) * [WIP] Replace 'lookup' and 'reverselookup' with a single property (that acts like 'reverselookup'). * [skip ci] Starting transitioning... * [skip ci] Transitioned, but have not compiled yet... * [skip ci] Compiles... * [skip ci] test_PR025 succeeds... * [skip ci] test_PR026 succeeds... * Done.
scikit-hep · Dec 26, 2019 · 5f73d23 · 5f73d23
1 parent a8b4472
commit 5f73d23
Show file tree

Hide file tree

Showing 43 changed files with 313 additions and 721 deletions.
diff --git a/VERSION_INFO b/VERSION_INFO
@@ -1 +1 @@
-0.1.36
+0.1.37
diff --git a/awkward1/_numba/array/recordarray.py b/awkward1/_numba/array/recordarray.py
@@ -11,24 +11,23 @@
 
 @numba.extending.typeof_impl.register(awkward1.layout.RecordArray)
 def typeof(val, c):
-    return RecordArrayType([numba.typeof(x) for x in val.fields()], val.lookup, val.reverselookup, numba.typeof(val.id), numba.none if val.isbare else numba.typeof(val.type))
+    return RecordArrayType([numba.typeof(x) for x in val.fields()], None if val.istuple else tuple(val.keys()), numba.typeof(val.id), numba.none if val.isbare else numba.typeof(val.type))
 
 @numba.extending.typeof_impl.register(awkward1.layout.Record)
 def typeof(val, c):
     return RecordType(numba.typeof(val.array))
 
 class RecordArrayType(content.ContentType):
-    def __init__(self, contenttpes, lookup, reverselookup, idtpe, typetpe):
-        super(RecordArrayType, self).__init__(name="ak::RecordArrayType([{0}], {1}, {2}, id={3}, type={4})".format(", ".join(x.name for x in contenttpes), lookup, reverselookup, idtpe.name, typetpe.name))
+    def __init__(self, contenttpes, keys, idtpe, typetpe):
+        super(RecordArrayType, self).__init__(name="ak::RecordArrayType([{0}], {1}, id={2}, type={3})".format(", ".join(x.name for x in contenttpes), keys, idtpe.name, typetpe.name))
         self.contenttpes = contenttpes
-        self.lookup = lookup
-        self.reverselookup = reverselookup
+        self.keys = keys
         self.idtpe = idtpe
         self.typetpe = typetpe
 
     @property
     def istuple(self):
-        return self.lookup is None
+        return self.keys is None
 
     @property
     def numfields(self):
@@ -45,7 +44,7 @@ def getitem_range(self):
         return self
 
     def getitem_str(self, key):
-        return self.contenttpes[awkward1._util.field2index(self.lookup, self.numfields, key)]
+        return self.contenttpes[awkward1._util.key2index(self.keys, key)]
 
     def getitem_tuple(self, wheretpe):
         import awkward1._numba.array.regulararray
@@ -60,19 +59,19 @@ def getitem_next(self, wheretpe, isadvanced):
         tailtpe = numba.types.Tuple(wheretpe.types[1:])
 
         if isinstance(headtpe, numba.types.StringLiteral):
-            index = awkward1._util.field2index(self.lookup, self.numfields, headtpe.literal_value)
+            index = awkward1._util.key2index(self.keys, headtpe.literal_value)
             nexttpe = self.contenttpes[index]
 
         else:
             contenttpes = []
             for t in self.contenttpes:
                 contenttpes.append(t.getitem_next(numba.types.Tuple((headtpe,)), isadvanced))
-            nexttpe = RecordArrayType(contenttpes, self.lookup, self.reverselookup, numba.none, numba.none)
+            nexttpe = RecordArrayType(contenttpes, self.keys, numba.none, numba.none)
 
         return nexttpe.getitem_next(tailtpe, isadvanced)
 
     def carry(self):
-        return RecordArrayType([x.carry() for x in self.contenttpes], self.lookup, self.reverselookup, self.idtpe, self.typetpe)
+        return RecordArrayType([x.carry() for x in self.contenttpes], self.keys, self.idtpe, self.typetpe)
 
     @property
     def lower_len(self):
@@ -227,36 +226,16 @@ def box(tpe, val, c):
 
     else:
         RecordArray_obj = c.pyapi.unserialize(c.pyapi.serialize_object(awkward1.layout.RecordArray))
-        from_lookup_obj = c.pyapi.object_getattr_string(RecordArray_obj, "from_lookup")
-        if tpe.lookup is None:
-            lookup_obj = c.pyapi.make_none()
-        else:
-            lookup_obj = c.pyapi.dict_new(len(tpe.lookup))
-            for key, fieldindex in tpe.lookup.items():
-                key_obj = c.pyapi.unserialize(c.pyapi.serialize_object(key))
-                fieldindex_obj = c.pyapi.unserialize(c.pyapi.serialize_object(fieldindex))
-                c.pyapi.dict_setitem(lookup_obj, key_obj, fieldindex_obj)
-                c.pyapi.decref(key_obj)
-                c.pyapi.decref(fieldindex_obj)
-        if tpe.reverselookup is None:
-            reverselookup_obj = c.pyapi.make_none()
-        else:
-            reverselookup_obj = c.pyapi.list_new(c.context.get_constant(numba.intp, 0))
-            for key in tpe.reverselookup:
-                key_obj = c.pyapi.unserialize(c.pyapi.serialize_object(key))
-                c.pyapi.list_append(reverselookup_obj, key_obj)
-                c.pyapi.decref(key_obj)
         contents_obj = c.pyapi.list_new(c.context.get_constant(numba.intp, 0))
         for i, t in enumerate(tpe.contenttpes):
             x_obj = c.pyapi.from_native_value(t, getattr(proxyin, field(i)), c.env_manager)
             c.pyapi.list_append(contents_obj, x_obj)
             c.pyapi.decref(x_obj)
-        out = c.pyapi.call_function_objargs(from_lookup_obj, [contents_obj, lookup_obj, reverselookup_obj] + args)
+        keys_obj = c.pyapi.unserialize(c.pyapi.serialize_object(tpe.keys))
+        out = c.pyapi.call_function_objargs(RecordArray_obj, [contents_obj, keys_obj] + args)
         c.pyapi.decref(RecordArray_obj)
-        c.pyapi.decref(from_lookup_obj)
-        c.pyapi.decref(lookup_obj)
-        c.pyapi.decref(reverselookup_obj)
         c.pyapi.decref(contents_obj)
+        c.pyapi.decref(keys_obj)
 
     for x in args:
         c.pyapi.decref(x)
@@ -337,7 +316,7 @@ def lower_getitem_range(context, builder, sig, args):
 def lower_getitem_str(context, builder, sig, args):
     rettpe, (tpe, wheretpe) = sig.return_type, sig.args
     val, whereval = args
-    index = awkward1._util.field2index(tpe.lookup, tpe.numfields, wheretpe.literal_value)
+    index = awkward1._util.key2index(tpe.keys, wheretpe.literal_value)
 
     proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
 
@@ -393,12 +372,12 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval,
     proxyin = numba.cgutils.create_struct_proxy(arraytpe)(context, builder, value=arrayval)
 
     if isinstance(headtpe, numba.types.StringLiteral):
-        index = awkward1._util.field2index(arraytpe.lookup, arraytpe.numfields, headtpe.literal_value)
+        index = awkward1._util.key2index(arraytpe.keys, headtpe.literal_value)
         nexttpe = arraytpe.contenttpes[index]
         nextval = getattr(proxyin, field(index))
 
     else:
-        nexttpe = RecordArrayType([t.getitem_next(numba.types.Tuple((headtpe,)), advanced is not None) for t in arraytpe.contenttpes], arraytpe.lookup, arraytpe.reverselookup, numba.none, numba.none)   # FIXME: Type::none()   # arraytpe.typetpe if util.preserves_type(headtpe, advanced is not None) else 
+        nexttpe = RecordArrayType([t.getitem_next(numba.types.Tuple((headtpe,)), advanced is not None) for t in arraytpe.contenttpes], arraytpe.keys, numba.none, numba.none)   # FIXME: Type::none()   # arraytpe.typetpe if util.preserves_type(headtpe, advanced is not None) else
         proxyout = numba.cgutils.create_struct_proxy(nexttpe)(context, builder)
         proxyout.length = proxyin.length
         wrappedheadtpe = numba.types.Tuple((headtpe,))

diff --git a/awkward1/_numba/types.py b/awkward1/_numba/types.py
@@ -56,7 +56,7 @@ def typeof_UnionType(val, c):
 
 @numba.extending.typeof_impl.register(awkward1.layout.RecordType)
 def typeof_RecordType(val, c):
-    return RecordTypeType([numba.typeof(x) for x in val.types], val.lookup, val.reverselookup, val.parameters)
+    return RecordTypeType([numba.typeof(x) for x in val.types], None if val.istuple else tuple(val.keys()), val.parameters)
 
 class TypeType(numba.types.Type):
     pass
@@ -97,11 +97,10 @@ def __init__(self, typetpes, parameters):
         self.parameters = parameters
 
 class RecordTypeType(TypeType):
-    def __init__(self, typetpes, lookup, reverselookup, parameters):
-        super(RecordTypeType, self).__init__(name="ak::RecordTypeType([{0}], {1}, {2}, parameters={3})".format(", ".join(x.name for x in typetpes), repr(lookup), repr(reverselookup), json.dumps(parameters)))
+    def __init__(self, typetpes, keys, parameters):
+        super(RecordTypeType, self).__init__(name="ak::RecordTypeType([{0}], {1}, parameters={2})".format(", ".join(x.name for x in typetpes), repr(keys), json.dumps(parameters)))
         self.typetpes = typetpes
-        self.lookup = lookup
-        self.reverselookup = reverselookup
+        self.keys = keys
         self.parameters = parameters
 
 @numba.extending.register_model(UnknownTypeType)
@@ -307,35 +306,11 @@ def box_RecordType(tpe, val, c):
         c.pyapi.tuple_setitem(types_obj, i, x_obj)
     parameters_obj = box_parameters(tpe.parameters, c)
 
-    if tpe.lookup is None:
-        out = c.pyapi.call_function_objargs(class_obj, (types_obj, parameters_obj))
-
-    else:
-        from_lookup_obj = c.pyapi.object_getattr_string(class_obj, "from_lookup")
-        if tpe.lookup is None:
-            lookup_obj = c.pyapi.make_none()
-        else:
-            lookup_obj = c.pyapi.dict_new(len(tpe.lookup))
-            for key, fieldindex in tpe.lookup.items():
-                key_obj = c.pyapi.unserialize(c.pyapi.serialize_object(key))
-                fieldindex_obj = c.pyapi.unserialize(c.pyapi.serialize_object(fieldindex))
-                c.pyapi.dict_setitem(lookup_obj, key_obj, fieldindex_obj)
-                c.pyapi.decref(key_obj)
-                c.pyapi.decref(fieldindex_obj)
-        if tpe.reverselookup is None:
-            reverselookup_obj = c.pyapi.make_none()
-        else:
-            reverselookup_obj = c.pyapi.list_new(c.context.get_constant(numba.intp, 0))
-            for key in tpe.reverselookup:
-                key_obj = c.pyapi.unserialize(c.pyapi.serialize_object(key))
-                c.pyapi.list_append(reverselookup_obj, key_obj)
-                c.pyapi.decref(key_obj)
-        out = c.pyapi.call_function_objargs(from_lookup_obj, (types_obj, lookup_obj, reverselookup_obj, parameters_obj))
-        c.pyapi.decref(from_lookup_obj)
-        c.pyapi.decref(lookup_obj)
-        c.pyapi.decref(reverselookup_obj)
+    keys_obj = c.pyapi.unserialize(c.pyapi.serialize_object(tpe.keys))
+    out = c.pyapi.call_function_objargs(class_obj, (types_obj, keys_obj, parameters_obj))
 
     c.pyapi.decref(class_obj)
     c.pyapi.decref(types_obj)
+    c.pyapi.decref(keys_obj)
     c.pyapi.decref(parameters_obj)
     return out
diff --git a/awkward1/_util.py b/awkward1/_util.py
@@ -22,23 +22,26 @@ def wrap(content, namespace):
     else:
         return content
 
-def field2index(lookup, numfields, key):
-    if isinstance(key, (int, numbers.Integral, numpy.integer)):
-        attempt = key
+def key2index(keys, key):
+    if keys is None:
+        attempt = None
     else:
-        attempt = None if lookup is None else lookup.get(key)
+        try:
+            attempt = keys.index(key)
+        except ValueError:
+            attempt = None
 
     if attempt is None:
-        m = field2index._pattern.match(key)
+        m = key2index._pattern.match(key)
         if m is not None:
             attempt = m.group(0)
 
-    if attempt is None or attempt >= numfields:
-        raise ValueError("key {0} not found in Record".format(repr(key)))
+    if attempt is None:
+        raise ValueError("key {0} not found in record".format(repr(key)))
     else:
         return attempt
 
-field2index._pattern = re.compile(r"^[1-9][0-9]*$")
+key2index._pattern = re.compile(r"^[1-9][0-9]*$")
 
 def minimally_touching_string(limit_length, layout, namespace):
     import awkward1.layout

diff --git a/include/awkward/Content.h b/include/awkward/Content.h
@@ -46,8 +46,6 @@ namespace awkward {
     virtual int64_t fieldindex(const std::string& key) const = 0;
     virtual const std::string key(int64_t fieldindex) const = 0;
     virtual bool haskey(const std::string& key) const = 0;
-    virtual const std::vector<std::string> keyaliases(int64_t fieldindex) const = 0;
-    virtual const std::vector<std::string> keyaliases(const std::string& key) const = 0;
     virtual const std::vector<std::string> keys() const = 0;
 
     const std::string tostring() const;

diff --git a/include/awkward/array/EmptyArray.h b/include/awkward/array/EmptyArray.h
@@ -40,8 +40,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
   protected:

diff --git a/include/awkward/array/ListArray.h b/include/awkward/array/ListArray.h
@@ -42,8 +42,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
   protected:

diff --git a/include/awkward/array/ListOffsetArray.h b/include/awkward/array/ListOffsetArray.h
@@ -41,8 +41,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
   protected:

diff --git a/include/awkward/array/NumpyArray.h b/include/awkward/array/NumpyArray.h
@@ -60,8 +60,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
     bool iscontiguous() const;

diff --git a/include/awkward/array/RawArray.h b/include/awkward/array/RawArray.h
@@ -370,14 +370,6 @@ namespace awkward {
       throw std::invalid_argument("array contains no Records");
     }
 
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override {
-      throw std::invalid_argument("array contains no Records");
-    }
-
-    const std::vector<std::string> keyaliases(const std::string& key) const override {
-      throw std::invalid_argument("array contains no Records");
-    }
-
     const std::vector<std::string> keys() const override {
       throw std::invalid_argument("array contains no Records");
     }

diff --git a/include/awkward/array/Record.h b/include/awkward/array/Record.h
@@ -12,8 +12,7 @@ namespace awkward {
     const std::shared_ptr<Content> array() const;
     int64_t at() const;
     const std::vector<std::shared_ptr<Content>> contents() const;
-    const std::shared_ptr<RecordArray::Lookup> lookup() const;
-    const std::shared_ptr<RecordArray::ReverseLookup> reverselookup() const;
+    const std::shared_ptr<util::RecordLookup> recordlookup() const;
     bool istuple() const;
 
     bool isscalar() const override;
@@ -43,8 +42,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
     const std::shared_ptr<Content> field(int64_t fieldindex) const;

diff --git a/include/awkward/array/RecordArray.h b/include/awkward/array/RecordArray.h
@@ -14,16 +14,12 @@
 namespace awkward {
   class RecordArray: public Content {
   public:
-    typedef std::unordered_map<std::string, size_t> Lookup;
-    typedef std::vector<std::string> ReverseLookup;
-
-    RecordArray(const std::shared_ptr<Identity>& id, const std::shared_ptr<Type>& type, const std::vector<std::shared_ptr<Content>>& contents, const std::shared_ptr<Lookup>& lookup, const std::shared_ptr<ReverseLookup>& reverselookup);
+    RecordArray(const std::shared_ptr<Identity>& id, const std::shared_ptr<Type>& type, const std::vector<std::shared_ptr<Content>>& contents, const std::shared_ptr<util::RecordLookup>& recordlookup);
     RecordArray(const std::shared_ptr<Identity>& id, const std::shared_ptr<Type>& type, const std::vector<std::shared_ptr<Content>>& contents);
     RecordArray(const std::shared_ptr<Identity>& id, const std::shared_ptr<Type>& type, int64_t length, bool istuple);
 
     const std::vector<std::shared_ptr<Content>> contents() const;
-    const std::shared_ptr<Lookup> lookup() const;
-    const std::shared_ptr<ReverseLookup> reverselookup() const;
+    const std::shared_ptr<util::RecordLookup> recordlookup() const;
     bool istuple() const;
 
     const std::string classname() const override;
@@ -50,8 +46,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
     const std::shared_ptr<Content> field(int64_t fieldindex) const;
@@ -62,7 +56,6 @@ namespace awkward {
 
     void append(const std::shared_ptr<Content>& content, const std::string& key);
     void append(const std::shared_ptr<Content>& content);
-    void setkey(int64_t fieldindex, const std::string& key);
 
   protected:
     void checktype() const override;
@@ -75,8 +68,7 @@ namespace awkward {
 
   private:
     std::vector<std::shared_ptr<Content>> contents_;
-    std::shared_ptr<Lookup> lookup_;
-    std::shared_ptr<ReverseLookup> reverselookup_;
+    std::shared_ptr<util::RecordLookup> recordlookup_;
     int64_t length_;
   };
 }

diff --git a/include/awkward/array/RegularArray.h b/include/awkward/array/RegularArray.h
@@ -42,8 +42,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
   protected:

diff --git a/include/awkward/type/ArrayType.h b/include/awkward/type/ArrayType.h
@@ -17,8 +17,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
     const std::shared_ptr<Type> type() const;

diff --git a/include/awkward/type/ListType.h b/include/awkward/type/ListType.h
@@ -17,8 +17,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
   const std::shared_ptr<Type> type() const;

diff --git a/include/awkward/type/OptionType.h b/include/awkward/type/OptionType.h
@@ -17,8 +17,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
   const std::shared_ptr<Type> type() const;

diff --git a/include/awkward/type/PrimitiveType.h b/include/awkward/type/PrimitiveType.h
@@ -32,8 +32,6 @@ namespace awkward {
     int64_t fieldindex(const std::string& key) const override;
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
-    const std::vector<std::string> keyaliases(int64_t fieldindex) const override;
-    const std::vector<std::string> keyaliases(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
 
   const DType dtype() const;