Regression: cannot concatenate LEGEND event files

[gipert]

I cannot concatenate evt files anymore with the latest version:

all_data = lh5.read_as(
    "/evt",
    ["/dvs_ro/cfs/cdirs/m2676/data/lngs/l200/public/prodenv/prod-blind/ref-v2.1.4/generated/tier/pet/phy/l200-p03-r000-phy-tier_pet.lh5",
     "/dvs_ro/cfs/cdirs/m2676/data/lngs/l200/public/prodenv/prod-blind/ref-v2.1.4/generated/tier/pet/phy/l200-p03-r001-phy-tier_pet.lh5"],
    library="ak"
)

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File <timed exec>:5

File /pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/core.py:355, in read_as(name, lh5_file, library, **kwargs)
    351         kwargs2[k] = v
    353 # read the LGDO from disk
    354 # NOTE: providing a buffer does not make much sense
--> 355 obj = read(name, lh5_file, **kwargs1)
    357 if isinstance(obj, tuple):
    358     obj = obj[0]

File /pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/core.py:151, in read(name, lh5_file, start_row, n_rows, idx, use_h5idx, field_mask, obj_buf, obj_buf_start, decompress, locking)
    148     idx_i = None
    149 n_rows_i = n_rows - n_rows_read
--> 151 obj_ret = read(
    152     name,
    153     h5f,
    154     start_row,
    155     n_rows_i,
    156     idx_i,
    157     use_h5idx,
    158     field_mask,
    159     obj_buf,
    160     obj_buf_start,
    161     decompress,
    162 )
    163 if isinstance(obj_ret, tuple):
    164     obj_buf, n_rows_read_i = obj_ret

File /pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/core.py:182, in read(name, lh5_file, start_row, n_rows, idx, use_h5idx, field_mask, obj_buf, obj_buf_start, decompress, locking)
    179 if isinstance(idx, np.ndarray) and idx.dtype == np.dtype("?"):
    180     idx = np.where(idx)[0]
--> 182 obj, n_rows_read = _serializers._h5_read_lgdo(
    183     lh5_obj.id,
    184     lh5_obj.file.filename,
    185     lh5_obj.name,
    186     start_row=start_row,
    187     n_rows=n_rows,
    188     idx=idx,
    189     use_h5idx=use_h5idx,
    190     field_mask=field_mask,
    191     obj_buf=obj_buf,
    192     obj_buf_start=obj_buf_start,
    193     decompress=decompress,
    194 )
    196 return obj if obj_buf is None else (obj, n_rows_read)

File /pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/_serializers/read/composite.py:116, in _h5_read_lgdo(h5o, fname, oname, start_row, n_rows, idx, use_h5idx, field_mask, obj_buf, obj_buf_start, decompress)
    113         idx = idxa[:n_rows]  # works even if n_rows > len(idxa)
    115 if lgdotype is Table:
--> 116     return _h5_read_table(
    117         h5o,
    118         fname,
    119         oname,
    120         start_row=start_row,
    121         n_rows=n_rows,
    122         idx=idx,
    123         use_h5idx=use_h5idx,
    124         field_mask=field_mask,
    125         obj_buf=obj_buf,
    126         obj_buf_start=obj_buf_start,
    127         decompress=decompress,
    128     )
    130 if lgdotype is Histogram:
    131     return _h5_read_histogram(
    132         h5o,
    133         fname,
   (...)
    142         decompress=decompress,
    143     )

File [/](https://jupyter-dev.nersc.gov/user/pertoldi/perlmutter-exclusive-node-cpu/lab/tree/global/homes/p/pertoldi/user/new-evt-tier-test.ipynb)pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/_serializers/read/composite.py:322, in _h5_read_table(h5g, fname, oname, start_row, n_rows, idx, use_h5idx, field_mask, obj_buf, obj_buf_start, decompress)
    319     fld_buf = obj_buf[field]
    321 h5o = h5py.h5o.open(h5g, field.encode("utf-8"))
--> 322 col_dict[field], n_rows_read = _h5_read_lgdo(
    323     h5o,
    324     fname,
    325     f"{oname}/{field}",
    326     start_row=start_row,
    327     n_rows=n_rows,
    328     idx=idx,
    329     use_h5idx=use_h5idx,
    330     obj_buf=fld_buf,
    331     obj_buf_start=obj_buf_start,
    332     decompress=decompress,
    333 )
    334 h5o.close()
    336 if obj_buf is not None and obj_buf_start + n_rows_read > len(obj_buf):

File /pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/_serializers/read/composite.py:116, in _h5_read_lgdo(h5o, fname, oname, start_row, n_rows, idx, use_h5idx, field_mask, obj_buf, obj_buf_start, decompress)
    113         idx = idxa[:n_rows]  # works even if n_rows > len(idxa)
    115 if lgdotype is Table:
--> 116     return _h5_read_table(
    117         h5o,
    118         fname,
    119         oname,
    120         start_row=start_row,
    121         n_rows=n_rows,
    122         idx=idx,
    123         use_h5idx=use_h5idx,
    124         field_mask=field_mask,
    125         obj_buf=obj_buf,
    126         obj_buf_start=obj_buf_start,
    127         decompress=decompress,
    128     )
    130 if lgdotype is Histogram:
    131     return _h5_read_histogram(
    132         h5o,
    133         fname,
   (...)
    142         decompress=decompress,
    143     )

File [/](https://jupyter-dev.nersc.gov/user/pertoldi/perlmutter-exclusive-node-cpu/lab/tree/global/homes/p/pertoldi/user/new-evt-tier-test.ipynb)pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/_serializers/read/composite.py:322, in _h5_read_table(h5g, fname, oname, start_row, n_rows, idx, use_h5idx, field_mask, obj_buf, obj_buf_start, decompress)
    319     fld_buf = obj_buf[field]
    321 h5o = h5py.h5o.open(h5g, field.encode("utf-8"))
--> 322 col_dict[field], n_rows_read = _h5_read_lgdo(
    323     h5o,
    324     fname,
    325     f"{oname}/{field}",
    326     start_row=start_row,
    327     n_rows=n_rows,
    328     idx=idx,
    329     use_h5idx=use_h5idx,
    330     obj_buf=fld_buf,
    331     obj_buf_start=obj_buf_start,
    332     decompress=decompress,
    333 )
    334 h5o.close()
    336 if obj_buf is not None and obj_buf_start + n_rows_read > len(obj_buf):

File /pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/_serializers/read/composite.py:174, in _h5_read_lgdo(h5o, fname, oname, start_row, n_rows, idx, use_h5idx, field_mask, obj_buf, obj_buf_start, decompress)
    160     return _h5_read_vector_of_encoded_vectors(
    161         h5o,
    162         fname,
   (...)
    170         decompress=decompress,
    171     )
    173 if lgdotype is VectorOfVectors:
--> 174     return _h5_read_vector_of_vectors(
    175         h5o,
    176         fname,
    177         oname,
    178         start_row=start_row,
    179         n_rows=n_rows,
    180         idx=idx,
    181         use_h5idx=use_h5idx,
    182         obj_buf=obj_buf,
    183         obj_buf_start=obj_buf_start,
    184     )
    186 if lgdotype is FixedSizeArray:
    187     return _h5_read_fixedsize_array(
    188         h5o,
    189         fname,
   (...)
    196         obj_buf_start=obj_buf_start,
    197     )

File /pscratch/sd/p/pertoldi/.conda/legend/lib/python3.11/site-packages/lgdo/lh5/_serializers/read/vector_of_vectors.py:159, in _h5_read_vector_of_vectors(h5g, fname, oname, start_row, n_rows, idx, use_h5idx, obj_buf, obj_buf_start)
    157     fdb_size = fd_buf_start + fd_n_rows
    158     if len(fd_buf) < fdb_size:
--> 159         fd_buf.nda.resize(fdb_size, refcheck=False)
    161 # now read
    162 h5o = h5py.h5o.open(h5g, b"flattened_data")

AttributeError: 'VectorOfVectors' object has no attribute 'nda'

[gipert]

Bug introduced with v1.10

[iguinn]

This the same problem this PR was trying to solve: #115

The problem is with nested VoVs (which we don't have a test for). I will try to have a look at this again to understand why it's failing refcheck hopefully tomorrow.