filegroup_and_database.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FitsFileGroup and Database\n",
    "\n",
    "Currently astropop uses simple [Astropy Table](https://docs.astropy.org/en/stable/table/) to store headers data in `FitsFileGroup`. This can be fast, but have the problem to be in-memory only.\n",
    "\n",
    "So, we will investigate the use of SQL database to store the header table and filtering.\n",
    "\n",
    "Fortunately, Python has a built-in sqlite module called [sqlite3](https://docs.python.org/3/library/sqlite3.html), that we can use here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SQLite3 Wrapper\n",
    "\n",
    "First of all, lest implement a wrapper to do a more object-oriented sql work. This is limited to a single table database and just some functions and interactions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file _db.py\n",
    "import sqlite3 as sql\n",
    "import numpy as np\n",
    "\n",
    "from astropop.logger import logger\n",
    "\n",
    "\n",
    "np_to_sql = {\n",
    "    'i': 'INTEGER',\n",
    "    'f': 'REAL',\n",
    "    'S': 'TEXT',\n",
    "    'U': 'TEXT',\n",
    "    'b': 'BOOLEAN',\n",
    "}\n",
    "\n",
    "\n",
    "def _sanitize_colnames(colnames):\n",
    "    \"\"\"Sanitize the colnames to avoid invalid characteres like '-'.\"\"\"\n",
    "    return [c.replace('-', '_').replace(' ', '_').lower() for c in colnames]\n",
    "\n",
    "\n",
    "class Database:\n",
    "    \"\"\"Database creation and manipulation with SQL.\"\"\"\n",
    "\n",
    "    def __init__(self, db, table='table', dtype=None):\n",
    "        self._db = db\n",
    "        self._con = sql.connect(self._db)\n",
    "        self._cur = self._con.cursor()\n",
    "        self._table = table\n",
    "        self._add_table(dtype)\n",
    "\n",
    "    def execute(self, command):\n",
    "        \"\"\"Execute a SQL command in the database.\"\"\"\n",
    "        logger.debug('executing sql command: \"%s\"',\n",
    "                     str.replace(command, '\\n', ' '))\n",
    "        self._cur.execute(command)\n",
    "        res = self._cur.fetchall()\n",
    "        self._con.commit()\n",
    "        return res\n",
    "\n",
    "    def _add_table(self, dtype):\n",
    "        \"\"\"Create a table in database.\"\"\"\n",
    "        logger.debug('Initializing \"%s\" table.', self._table)\n",
    "        tables = [i[0] for i in self.execute(\"SELECT name FROM sqlite_master \"\n",
    "                                             \"WHERE type='table';\")]\n",
    "        if self._table in tables:\n",
    "            logger.debug('table \"%s\" already exists', self._table)\n",
    "            return\n",
    "        comm = f\"CREATE TABLE {self._table}\"\n",
    "        comm += \" (\\n_id INTEGER PRIMARY KEY AUTOINCREMENT\"\n",
    "        if dtype is not None:\n",
    "            comm += \",\\n\"\n",
    "            for i, name in enumerate(dtype.names):\n",
    "                kind = dtype[i].kind\n",
    "                comm += f\"\\t'{name}' {np_to_sql[kind]}\"\n",
    "                if i != len(dtype) - 1:\n",
    "                    comm += \",\\n\"\n",
    "        comm += \"\\n);\"\n",
    "        self.execute(comm)\n",
    "\n",
    "    def colnames(self):\n",
    "        \"\"\"Get the column names of the current table.\"\"\"\n",
    "        self.execute(f\"SELECT * FROM {self._table} WHERE 1=0\")\n",
    "        return [i[0].lower() for i in self._cur.description]\n",
    "\n",
    "    def add_column(self, column, dtype):\n",
    "        \"\"\"Add a column to a table.\"\"\"\n",
    "        col = _sanitize_colnames([column])[0]\n",
    "        comm = f\"ALTER TABLE {self._table} ADD COLUMN '{col}' \"\n",
    "        logger.debug('adding column \"%s\" \"%s\" \"%s\"', col, dtype, dtype.kind)\n",
    "        comm += f\"{np_to_sql[dtype.kind]};\"\n",
    "        self.execute(comm)\n",
    "\n",
    "    def add_row(self, data, add_columns=False):\n",
    "        \"\"\"Add a dict row to a table.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        data : dict\n",
    "            Dictionary of data to add.\n",
    "        add_columns : bool (optional)\n",
    "            If True, add missing columns to the table.\n",
    "        \"\"\"\n",
    "        data_c = {}\n",
    "        sanitized = _sanitize_colnames(data.keys())\n",
    "        # create a dict copy with sanitized keys\n",
    "        for k, ks in zip(data.keys(), sanitized):\n",
    "            data_c[ks] = data[k]\n",
    "\n",
    "        if add_columns:\n",
    "            # add missing columns\n",
    "            cols = set(self.colnames())\n",
    "            for k in data_c.keys():\n",
    "                if k not in cols:\n",
    "                    self.add_column(k, np.array([data_c[k]]).dtype)\n",
    "\n",
    "        # create the sql command and add the row\n",
    "        cols = self.colnames()\n",
    "        comm = f\"INSERT INTO {self._table} VALUES (\"\n",
    "        for i, name in enumerate(cols):\n",
    "            if name in data_c.keys():\n",
    "                d = data_c[name]\n",
    "                if isinstance(d, str):\n",
    "                    d = f\"'{d}'\"\n",
    "                comm += f\"{d}\"\n",
    "            elif name == '_id':\n",
    "                comm += \"NULL\"\n",
    "            else:\n",
    "                comm += \"NULL\"\n",
    "            if i != len(cols) - 1:\n",
    "                comm += \", \"\n",
    "        comm += \");\"\n",
    "        self.execute(comm)\n",
    "\n",
    "    def select(self, columns=None, where=None):\n",
    "        \"\"\"Select rows from a table.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        columns : list (optional)\n",
    "            List of columns to select. If None, select all columns.\n",
    "        where : dict (optional)\n",
    "            Dictionary of conditions to select rows. Keys are column names,\n",
    "            values are values to compare. All rows equal to the values will\n",
    "            be selected. If None, all rows are selected.\n",
    "        \"\"\"\n",
    "        if columns is None:\n",
    "            columns = '*'\n",
    "        else:\n",
    "            # only use sanitized column names\n",
    "            columns = ', '.join(_sanitize_colnames(columns))\n",
    "\n",
    "        if where is None:\n",
    "            _where = '1=1'\n",
    "        else:\n",
    "            for i, (k, v) in enumerate(where.items()):\n",
    "                if isinstance(v, str):\n",
    "                    # avoid sql errors\n",
    "                    v = f\"'{v}'\"\n",
    "                if i == 0:\n",
    "                    _where = f\"{k}={v}\"\n",
    "                else:\n",
    "                    _where += f\" AND {k}={v}\"\n",
    "        return self.execute(f\"SELECT {columns} FROM {self._table} \"\n",
    "                            f\"WHERE {_where}\")\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Get the number of rows in the current table.\"\"\"\n",
    "        return self.execute(f\"SELECT COUNT(*) FROM {self._table}\")[0][0]\n",
    "\n",
    "    def __del__(self):\n",
    "        \"\"\"Delete the class, closing the db connection.\"\"\"\n",
    "        # ensure connection is closed.\n",
    "        self._con.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.io import fits\n",
    "from astropop.file_collection import list_fits_files\n",
    "from astropop.logger import logger\n",
    "\n",
    "logger.setLevel('INFO')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-07 23:21:46,350 astropop - INFO - number of files 427  [707346715]\n"
     ]
    }
   ],
   "source": [
    "files = list_fits_files('/home/julio/19jan30', fits_extensions=['fits.gz'])\n",
    "logger.info('number of files %i', len(files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Is everything working?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a in-memory database\n",
    "db = Database(':memory:', 'headers')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_files(files):\n",
    "    for i, f in enumerate(files):\n",
    "        logger.debug(\"file %i from %i\", i+1, len(files))\n",
    "        header = dict(fits.getheader(f))\n",
    "        header.pop('COMMENT', None)\n",
    "        header.pop('HISTORY', None)\n",
    "        db.add_row(header, add_columns=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_files(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('HD126593_STD_L0_F2', 'HD126593', 'F2', 'L0', 16, 1e-06),\n",
       " ('HD126593_STD_L1_F2', 'HD126593', 'F2', 'L1', 16, 1e-06),\n",
       " ('HD126593_STD_L2_F2', 'HD126593', 'F2', 'L2', 16, 1e-06),\n",
       " ('HD126593_STD_L3_F2', 'HD126593', 'F2', 'L3', 16, 1e-06),\n",
       " ('HD126593_STD_L4_F2', 'HD126593', 'F2', 'L4', 16, 1e-06),\n",
       " ('HD126593_STD_L5_F2', 'HD126593', 'F2', 'L5', 16, 1e-06),\n",
       " ('HD126593_STD_L6_F2', 'HD126593', 'F2', 'L6', 16, 1e-06),\n",
       " ('HD126593_STD_L7_F2', 'HD126593', 'F2', 'L7', 16, 1e-06),\n",
       " ('HD126593_STD_L8_F2', 'HD126593', 'F2', 'L8', 16, 1e-06),\n",
       " ('HD126593_STD_L9_F2', 'HD126593', 'F2', 'L9', 16, 1e-06),\n",
       " ('HD126593_STD_LA_F2', 'HD126593', 'F2', 'LA', 16, 1e-06),\n",
       " ('HD126593_STD_LB_F2', 'HD126593', 'F2', 'LB', 16, 1e-06),\n",
       " ('HD126593_STD_LC_F2', 'HD126593', 'F2', 'LC', 16, 1e-06),\n",
       " ('HD126593_STD_LD_F2', 'HD126593', 'F2', 'LD', 16, 1e-06),\n",
       " ('HD126593_STD_LE_F2', 'HD126593', 'F2', 'LE', 16, 1e-06),\n",
       " ('HD126593_STD_LF_F2', 'HD126593', 'F2', 'LF', 16, 1e-06)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db.select(columns=['image', 'object', 'filter', 'lamina', 'bitpix', 'readtime'], where={'OBJECT': 'HD126593'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "columns:  ['_id', 'simple', 'bitpix', 'naxis', 'naxis1', 'naxis2', 'extend', 'bzero', 'bscale', 'head', 'acqmode', 'readmode', 'imgrect', 'hbin', 'vbin', 'subrect', 'datatype', 'xtype', 'xunit', 'raywave', 'calbwvnm', 'trigger', 'calib', 'dllver', 'exposure', 'temp', 'readtime', 'operatn', 'gain', 'emrealgn', 'vclkamp', 'vshift', 'outptamp', 'preamp', 'serno', 'unsttemp', 'blclamp', 'precan', 'flipx', 'flipy', 'cntcvtmd', 'cntcvt', 'dtnwlgth', 'sntvty', 'spsnfltr', 'thrshld', 'pcntenld', 'nsethsld', 'ptnthld1', 'ptnthld2', 'ptnthld3', 'ptnthld4', 'avgftrmd', 'avgfctr', 'frmcnt', 'portmode', 'lsheight', 'lsspeed', 'lsaltdir', 'lsctrl', 'lsdir', 'fksmode', 'fktmode', 'usertxt1', 'usertxt2', 'usertxt3', 'usertxt4', 'date', 'frame', 'eshtmode', 'preampgaintext', 'spectrographserial', 'shamrockisactive', 'spectrographname', 'spectrographisactive', 'irigdataavailable', 'irigdata', 'detector', 'exptime', 'camgain', 'date_obs', 'ra', 'dec', 'epoch', 'airmass', 'jd', 'st', 'ha', 'image', 'telescop', 'platescl', 'instrume', 'object', 'observer', 'rdnoise', 'lamina', 'lam_pos', 'filter', 'fil_pos', 'focusval', 'w_bar', 'w_temp', 'w_hum']\n",
      "files: 427\n"
     ]
    }
   ],
   "source": [
    "print('columns: ', db.colnames())\n",
    "print('files:', len(db))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like everything is working properly. All 427 files were added and all columns are present. What is int seems to be returned as int and what is float seems to be float. The filtering is also working too.\n",
    "\n",
    "Notes:\n",
    "\n",
    "- SQL tables don't like `-` character in name. So it is replaced by `_`. Same for spaces. This create possible conflicts, but may be a smaller problem in practice."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FitsFileGroup implementations\n",
    "\n",
    "Lets try our two `FitsFileGroup` implementations. One using `astropy.table.Table`, another using our SQL wrapper."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.table import Table\n",
    "from astropop.file_collection import list_fits_files\n",
    "\n",
    "from astropop.fits_utils import _fits_extensions, \\\n",
    "                                _fits_extensions_with_compress\n",
    "from astropop.framedata import check_framedata\n",
    "from astropop.py_utils import check_iterable\n",
    "from astropop.logger import logger\n",
    "\n",
    "\n",
    "def create_table_summary(headers, n):\n",
    "    \"\"\"Create a table summary of headers.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    headers: iterator\n",
    "        Iterator for a list of header files.\n",
    "    n: int\n",
    "        Number of headers to iterate.\n",
    "    \"\"\"\n",
    "    summary_dict = {}\n",
    "    for i, head in enumerate(headers):\n",
    "        logger.debug('Reading file %i from %i', i, n)\n",
    "        keys = head.keys()\n",
    "        for k in keys:\n",
    "            k_lower = k.lower()\n",
    "            if k_lower in ('history', 'comment'):\n",
    "                logger.debug('%s key ignored', k)\n",
    "                continue\n",
    "            if k_lower not in summary_dict.keys():\n",
    "                summary_dict[k_lower] = [None]*n\n",
    "            summary_dict[k_lower][i] = head.get(k)\n",
    "\n",
    "    return Table(summary_dict)\n",
    "\n",
    "\n",
    "def gen_mask(table, keywords):\n",
    "    \"\"\"Generate a mask to be applyed in the filtering.\"\"\"\n",
    "    if len(table) == 0:\n",
    "        return []\n",
    "\n",
    "    t = Table(table)\n",
    "\n",
    "    mask = np.ones(len(t), dtype=bool)\n",
    "    for k, v in keywords.items():\n",
    "        if not check_iterable(v):\n",
    "            v = [v]\n",
    "        k = k.lower()\n",
    "        if k not in t.colnames:\n",
    "            t[k] = [None]*len(t)\n",
    "        nmask = [t[k][i] in v for i in range(len(t))]\n",
    "        mask &= np.array(nmask)\n",
    "\n",
    "    return mask\n",
    "\n",
    "\n",
    "class FitsFileGroup_Table():\n",
    "    \"\"\"Easy handle groups of fits files.\"\"\"\n",
    "\n",
    "    def __init__(self, location=None, files=None, ext=0,\n",
    "                 compression=False, **kwargs):\n",
    "        self._ext = ext\n",
    "        self._extensions = kwargs.get('fits_ext',\n",
    "                                      _fits_extensions_with_compress\n",
    "                                      if compression else _fits_extensions)\n",
    "\n",
    "        self._include = kwargs.get('glob_include')\n",
    "        self._exclude = kwargs.get('glob_exclude')\n",
    "        self._keywords = kwargs.get('keywords')\n",
    "\n",
    "        if location is None and files is None:\n",
    "            raise ValueError(\"You must specify a 'location'\"\n",
    "                             \"or a list of 'files'\")\n",
    "        if files is None and location is not None:\n",
    "            files = list_fits_files(location, self._extensions,\n",
    "                                    self._include, self._exclude)\n",
    "\n",
    "        self._files = files\n",
    "        self._location = location\n",
    "\n",
    "        self._summary = create_table_summary(self.headers(), len(self))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.files)\n",
    "\n",
    "    @property\n",
    "    def files(self):\n",
    "        return self._files.copy()\n",
    "\n",
    "    @property\n",
    "    def location(self):\n",
    "        return self._location\n",
    "\n",
    "    @property\n",
    "    def keywords(self):\n",
    "        return self._keywords\n",
    "\n",
    "    @property\n",
    "    def summary(self):\n",
    "        return Table(self._summary)\n",
    "\n",
    "    def __copy__(self, files=None, summary=None):\n",
    "        nfg = FitsFileGroup_Table.__new__(FitsFileGroup_Table)\n",
    "        for k, v in self.__dict__.items():\n",
    "            if k == '_summary':\n",
    "                nfg._summary = summary or self._summary\n",
    "            elif k == '_files':\n",
    "                nfg._files = files if files is not None else self._files\n",
    "            else:\n",
    "                nfg.__dict__[k] = v\n",
    "        return nfg\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        if isinstance(item, str):\n",
    "            # string will be interpreted as collumn name\n",
    "            if item.lower() not in self._summary.colnames:\n",
    "                raise KeyError(f'Column {item} not found.')\n",
    "            return self._summary.columns[item.lower()]\n",
    "\n",
    "        # returning FitsFileGroups\n",
    "        if isinstance(item, (int, np.integer)):\n",
    "            # single index will be interpreted as a single file group\n",
    "            return self.__copy__(files=[self._files[item]],\n",
    "                                 summary=self._summary[item])\n",
    "        if (isinstance(item, slice)):\n",
    "            files = self._files[item]\n",
    "            summ = self._summary[item]\n",
    "            return self.__copy__(files=files, summary=summ)\n",
    "        if isinstance(item, (np.ndarray, list, tuple)):\n",
    "            item = np.array(item)\n",
    "            if len(item) == 0:\n",
    "                return self.__copy__(files=[], summary=self._summary[item])\n",
    "            files = list(np.take(self._files, item))\n",
    "            summ = self._summary[item]\n",
    "            return self.__copy__(files=files, summary=summ)\n",
    "\n",
    "        raise KeyError(f'{item}')\n",
    "\n",
    "    def filtered(self, keywords=None):\n",
    "        \"\"\"Create a new FileGroup with only filtered files.\"\"\"\n",
    "        where = np.where(gen_mask(self._summary, keywords))[0]\n",
    "        return self[where]\n",
    "\n",
    "    def values(self, keyword, unique=False):\n",
    "        \"\"\"Return the values of a keyword in the summary.\n",
    "\n",
    "        If unique, only unique values returned.\n",
    "        \"\"\"\n",
    "        if keyword not in self.summary.colnames:\n",
    "            if unique:\n",
    "                n = 1\n",
    "            else:\n",
    "                n = len(self.summary)\n",
    "            return [None]*n\n",
    "        if unique:\n",
    "            return list(set(self.summary[keyword].tolist()))\n",
    "        return self.summary[keyword].tolist()\n",
    "\n",
    "    def add_column(self, name, values, mask=None):\n",
    "        \"\"\"Add a new column to the summary.\"\"\"\n",
    "        if not check_iterable(values):\n",
    "            values = [values]*len(self.summary)\n",
    "        elif len(values) != len(self.summary):\n",
    "            values = [values]*len(self.summary)\n",
    "\n",
    "        self.summary[name] = values\n",
    "        self.summary[name].mask = mask\n",
    "\n",
    "    def _intern_yelder(self, files=None, ext=None, ret_type=None,\n",
    "                       **kwargs):\n",
    "        \"\"\"Iter over files.\"\"\"\n",
    "        ext = ext if ext is not None else self._ext\n",
    "        files = files if files is not None else self._files\n",
    "        for i in files:\n",
    "            if ret_type == 'header':\n",
    "                yield fits.open(i, **kwargs)[ext].header\n",
    "            if ret_type == 'data':\n",
    "                yield fits.open(i, **kwargs)[ext].data\n",
    "            if ret_type == 'hdu':\n",
    "                yield fits.open(i, **kwargs)[ext]\n",
    "            if ret_type == 'framedata':\n",
    "                yield check_framedata(i, hdu=ext, **kwargs)\n",
    "\n",
    "    def hdus(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their HDUs.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='hdu', **kwargs)\n",
    "\n",
    "    def headers(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their headers.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='header', **kwargs)\n",
    "\n",
    "    def data(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='data', **kwargs)\n",
    "\n",
    "    def framedata(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='framedata', **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><i>Table length=16</i>\n",
       "<table id=\"table140451111142544\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>simple</th><th>bitpix</th><th>naxis</th><th>naxis1</th><th>naxis2</th><th>extend</th><th>bzero</th><th>bscale</th><th>head</th><th>acqmode</th><th>readmode</th><th>imgrect</th><th>hbin</th><th>vbin</th><th>subrect</th><th>datatype</th><th>xtype</th><th>xunit</th><th>raywave</th><th>calbwvnm</th><th>trigger</th><th>calib</th><th>dllver</th><th>exposure</th><th>temp</th><th>readtime</th><th>operatn</th><th>gain</th><th>emrealgn</th><th>vclkamp</th><th>vshift</th><th>outptamp</th><th>preamp</th><th>serno</th><th>unsttemp</th><th>blclamp</th><th>precan</th><th>flipx</th><th>flipy</th><th>cntcvtmd</th><th>cntcvt</th><th>dtnwlgth</th><th>sntvty</th><th>spsnfltr</th><th>thrshld</th><th>pcntenld</th><th>nsethsld</th><th>ptnthld1</th><th>ptnthld2</th><th>ptnthld3</th><th>ptnthld4</th><th>avgftrmd</th><th>avgfctr</th><th>frmcnt</th><th>portmode</th><th>lsheight</th><th>lsspeed</th><th>lsaltdir</th><th>lsctrl</th><th>lsdir</th><th>fksmode</th><th>fktmode</th><th>usertxt1</th><th>usertxt2</th><th>usertxt3</th><th>usertxt4</th><th>date</th><th>frame</th><th>eshtmode</th><th>preampgaintext</th><th>spectrographserial</th><th>shamrockisactive</th><th>spectrographname</th><th>spectrographisactive</th><th>irigdataavailable</th><th>irigdata</th><th>detector</th><th>exptime</th><th>camgain</th><th>date-obs</th><th>ra</th><th>dec</th><th>epoch</th><th>airmass</th><th>jd</th><th>st</th><th>ha</th><th>image</th><th>telescop</th><th>platescl</th><th>instrume</th><th>object</th><th>observer</th><th>rdnoise</th><th>lamina</th><th>lam-pos</th><th>filter</th><th>fil-pos</th><th>focusval</th><th>w-bar</th><th>w-temp</th><th>w-hum</th></tr></thead>\n",
       "<thead><tr><th>bool</th><th>int64</th><th>int64</th><th>int64</th><th>int64</th><th>bool</th><th>int64</th><th>int64</th><th>str1</th><th>str6</th><th>str5</th><th>str15</th><th>str1</th><th>str1</th><th>str14</th><th>str6</th><th>str12</th><th>int64</th><th>float64</th><th>int64</th><th>str8</th><th>str11</th><th>str13</th><th>str9</th><th>str5</th><th>float64</th><th>int64</th><th>str3</th><th>int64</th><th>int64</th><th>str8</th><th>str12</th><th>str4</th><th>str4</th><th>str4</th><th>bool</th><th>int64</th><th>int64</th><th>int64</th><th>int64</th><th>int64</th><th>float64</th><th>float64</th><th>int64</th><th>float64</th><th>int64</th><th>int64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>int64</th><th>int64</th><th>int64</th><th>int64</th><th>int64</th><th>float64</th><th>int64</th><th>int64</th><th>int64</th><th>int64</th><th>int64</th><th>str1</th><th>str1</th><th>str1</th><th>str1</th><th>str19</th><th>str23</th><th>int64</th><th>str1</th><th>str1</th><th>int64</th><th>str1</th><th>int64</th><th>int64</th><th>str1</th><th>str1</th><th>str9</th><th>str4</th><th>str23</th><th>str8</th><th>str9</th><th>str6</th><th>str5</th><th>str13</th><th>str8</th><th>str8</th><th>str18</th><th>str9</th><th>str4</th><th>str10</th><th>str12</th><th>str28</th><th>str3</th><th>str2</th><th>str2</th><th>str2</th><th>str1</th><th>str4</th><th>str5</th><th>str4</th><th>str2</th></tr></thead>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:44:29</td><td>2019-01-31T06:44:29.186</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:44:29.186</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.390</td><td>2458514.78087</td><td>12:22:55</td><td>-2:07:16</td><td>HD126593_STD_L0_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L0</td><td>0</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>51</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:44:41</td><td>2019-01-31T06:44:41.496</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:44:41.496</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.389</td><td>2458514.78101</td><td>12:23:07</td><td>-2:07:04</td><td>HD126593_STD_L1_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L1</td><td>1</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>51</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:44:53</td><td>2019-01-31T06:44:53.820</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:44:53.820</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.389</td><td>2458514.78115</td><td>12:23:20</td><td>-2:06:52</td><td>HD126593_STD_L2_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L2</td><td>2</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>51</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:45:06</td><td>2019-01-31T06:45:06.140</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:45:06.140</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.389</td><td>2458514.78129</td><td>12:23:31</td><td>-2:06:40</td><td>HD126593_STD_L3_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L3</td><td>3</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>51</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:45:18</td><td>2019-01-31T06:45:18.461</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:45:18.461</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.388</td><td>2458514.78144</td><td>12:23:44</td><td>-2:06:27</td><td>HD126593_STD_L4_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L4</td><td>4</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>51</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:45:30</td><td>2019-01-31T06:45:30.788</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:45:30.788</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.388</td><td>2458514.78158</td><td>12:23:56</td><td>-2:06:16</td><td>HD126593_STD_L5_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L5</td><td>5</td><td>F2</td><td>2</td><td>2248</td><td>756,0</td><td>18,8</td><td>51</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:45:43</td><td>2019-01-31T06:45:43.112</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:45:43.112</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.387</td><td>2458514.78172</td><td>12:24:08</td><td>-2:06:04</td><td>HD126593_STD_L6_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L6</td><td>6</td><td>F2</td><td>2</td><td>2248</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:45:55</td><td>2019-01-31T06:45:55.442</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:45:55.442</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.387</td><td>2458514.78186</td><td>12:24:21</td><td>-2:05:50</td><td>HD126593_STD_L7_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L7</td><td>7</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:46:07</td><td>2019-01-31T06:46:07.764</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:46:07.764</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.387</td><td>2458514.78201</td><td>12:24:33</td><td>-2:05:38</td><td>HD126593_STD_L8_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L8</td><td>8</td><td>F2</td><td>2</td><td>2248</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:46:20</td><td>2019-01-31T06:46:20.088</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:46:20.088</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.386</td><td>2458514.78215</td><td>12:24:45</td><td>-2:05:26</td><td>HD126593_STD_L9_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>L9</td><td>9</td><td>F2</td><td>2</td><td>2248</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:46:32</td><td>2019-01-31T06:46:32.406</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:46:32.406</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.386</td><td>2458514.78229</td><td>12:24:58</td><td>-2:05:14</td><td>HD126593_STD_LA_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>LA</td><td>10</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:46:44</td><td>2019-01-31T06:46:44.731</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:46:44.731</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.385</td><td>2458514.78244</td><td>12:25:10</td><td>-2:05:01</td><td>HD126593_STD_LB_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>LB</td><td>11</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:46:57</td><td>2019-01-31T06:46:57.054</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:46:57.054</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.385</td><td>2458514.78258</td><td>12:25:22</td><td>-2:04:49</td><td>HD126593_STD_LC_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>LC</td><td>12</td><td>F2</td><td>2</td><td>2248</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:47:09</td><td>2019-01-31T06:47:09.392</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:47:09.392</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.385</td><td>2458514.78272</td><td>12:25:34</td><td>-2:04:37</td><td>HD126593_STD_LD_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>LD</td><td>13</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:47:21</td><td>2019-01-31T06:47:21.705</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:47:21.705</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.384</td><td>2458514.78286</td><td>12:25:48</td><td>-2:04:23</td><td>HD126593_STD_LE_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>LE</td><td>14</td><td>F2</td><td>2</td><td>2247</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "<tr><td>True</td><td>16</td><td>2</td><td>1024</td><td>1024</td><td>True</td><td>32768</td><td>1</td><td></td><td>Single</td><td>Image</td><td>1, 1024,1024, 1</td><td>1</td><td>1</td><td>1, 1024,1024,1</td><td>Counts</td><td>Pixel number</td><td>0</td><td>0.0</td><td>0</td><td>Internal</td><td>0.,1.,0.,0.</td><td>2.100.30039.0</td><td>5,00000</td><td>-74,1</td><td>1e-06</td><td>4</td><td>1.5</td><td>0</td><td>0</td><td>12.9E-06</td><td>Conventional</td><td>2.5x</td><td>4335</td><td>-999</td><td>False</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>500.0</td><td>0.0</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.802597e-45</td><td>0</td><td>1</td><td>1</td><td>-1</td><td>0</td><td>0.0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td></td><td></td><td></td><td></td><td>2019-01-31T06:47:34</td><td>2019-01-31T06:47:34.024</td><td>0</td><td></td><td></td><td>0</td><td></td><td>0</td><td>0</td><td></td><td></td><td>5,00000</td><td>2.5x</td><td>2019-01-31T06:47:34.024</td><td>14:28:50</td><td>-60:32:25</td><td>2000.0</td><td>1.384</td><td>2458514.78301</td><td>12:26:00</td><td>-2:04:11</td><td>HD126593_STD_LF_F2</td><td>0.60m(BC)</td><td>0.34</td><td>Ikon 19002</td><td>HD126593</td><td>Julio Cesar Neves Campagnolo</td><td>6.3</td><td>LF</td><td>15</td><td>F2</td><td>2</td><td>2248</td><td>756,0</td><td>18,8</td><td>52</td></tr>\n",
       "</table></div>"
      ],
      "text/plain": [
       "<Table length=16>\n",
       "simple bitpix naxis naxis1 naxis2 extend bzero bscale head acqmode readmode     imgrect     hbin vbin ... platescl  instrume   object            observer           rdnoise lamina lam-pos filter fil-pos focusval w-bar w-temp w-hum\n",
       " bool  int64  int64 int64  int64   bool  int64 int64  str1   str6    str5        str15      str1 str1 ...   str4     str10     str12              str28               str3   str2    str2   str2    str1    str4    str5  str4   str2\n",
       "------ ------ ----- ------ ------ ------ ----- ------ ---- ------- -------- --------------- ---- ---- ... -------- ---------- -------- ---------------------------- ------- ------ ------- ------ ------- -------- ----- ------ -----\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L0       0     F2       2     2247 756,0   18,8    51\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L1       1     F2       2     2247 756,0   18,8    51\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L2       2     F2       2     2247 756,0   18,8    51\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L3       3     F2       2     2247 756,0   18,8    51\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L4       4     F2       2     2247 756,0   18,8    51\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L5       5     F2       2     2248 756,0   18,8    51\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L6       6     F2       2     2248 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L7       7     F2       2     2247 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L8       8     F2       2     2248 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     L9       9     F2       2     2248 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     LA      10     F2       2     2247 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     LB      11     F2       2     2247 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     LC      12     F2       2     2248 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     LD      13     F2       2     2247 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     LE      14     F2       2     2247 756,0   18,8    52\n",
       "  True     16     2   1024   1024   True 32768      1       Single    Image 1, 1024,1024, 1    1    1 ...     0.34 Ikon 19002 HD126593 Julio Cesar Neves Campagnolo     6.3     LF      15     F2       2     2248 756,0   18,8    52"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = FitsFileGroup_Table(files=files)\n",
    "f.filtered({'object': 'HD126593'}).summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SQL implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FitsFileGroup_Table():\n",
    "    \"\"\"Easy handle groups of fits files.\"\"\"\n",
    "\n",
    "    def __init__(self, location=None, files=None, ext=0,\n",
    "                 compression=False, database=':memory:', **kwargs):\n",
    "        self._ext = ext\n",
    "        self._extensions = kwargs.get('fits_ext',\n",
    "                                      _fits_extensions_with_compress\n",
    "                                      if compression else _fits_extensions)\n",
    "\n",
    "        self._include = kwargs.get('glob_include')\n",
    "        self._exclude = kwargs.get('glob_exclude')\n",
    "        self._keywords = kwargs.get('keywords')\n",
    "        self._db = Database(database, table='headers')\n",
    "\n",
    "        if location is None and files is None:\n",
    "            raise ValueError(\"You must specify a 'location'\"\n",
    "                             \"or a list of 'files'\")\n",
    "        if files is None and location is not None:\n",
    "            files = list_fits_files(location, self._extensions,\n",
    "                                    self._include, self._exclude)\n",
    "\n",
    "        self._files = files\n",
    "        self._location = location\n",
    "        \n",
    "        self._read_db()\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.files)\n",
    "\n",
    "    @property\n",
    "    def files(self):\n",
    "        return self._files.copy()\n",
    "\n",
    "    @property\n",
    "    def location(self):\n",
    "        return self._location\n",
    "\n",
    "    @property\n",
    "    def keywords(self):\n",
    "        return self._keywords\n",
    "\n",
    "    @property\n",
    "    def summary(self):\n",
    "        return Table(names=self._db.colnames(),\n",
    "                     rows=self._db.select())\n",
    "\n",
    "    def __copy__(self, files=None, db=None):\n",
    "        nfg = FitsFileGroup.__new__(FitsFileGroup)\n",
    "        for k, v in self.__dict__.items():\n",
    "            if k == '_db':\n",
    "                nfg._db = db or self._db\n",
    "            elif k == '_files':\n",
    "                nfg._files = files if files is not None else self._files\n",
    "            else:\n",
    "                nfg.__dict__[k] = v\n",
    "        return nfg\n",
    "    \n",
    "    def _read_db(self):\n",
    "        \"\"\"Read the database.\"\"\"\n",
    "\n",
    "    def filtered(self, keywords=None):\n",
    "        \"\"\"Create a new FileGroup with only filtered files.\"\"\"\n",
    "        rows = self._db.select(where=keywords)\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def values(self, keyword, unique=False):\n",
    "        \"\"\"Return the values of a keyword in the summary.\n",
    "\n",
    "        If unique, only unique values returned.\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def add_column(self, name, values, mask=None):\n",
    "        \"\"\"Add a new column to the summary.\"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def _intern_yelder(self, files=None, ext=None, ret_type=None,\n",
    "                       **kwargs):\n",
    "        \"\"\"Iter over files.\"\"\"\n",
    "        ext = ext if ext is not None else self._ext\n",
    "        files = files if files is not None else self._files\n",
    "        for i in files:\n",
    "            if ret_type == 'header':\n",
    "                yield fits.open(i, **kwargs)[ext].header\n",
    "            if ret_type == 'data':\n",
    "                yield fits.open(i, **kwargs)[ext].data\n",
    "            if ret_type == 'hdu':\n",
    "                yield fits.open(i, **kwargs)[ext]\n",
    "            if ret_type == 'framedata':\n",
    "                yield check_framedata(i, hdu=ext, **kwargs)\n",
    "\n",
    "    def hdus(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their HDUs.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='hdu', **kwargs)\n",
    "\n",
    "    def headers(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their headers.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='header', **kwargs)\n",
    "\n",
    "    def data(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='data', **kwargs)\n",
    "\n",
    "    def framedata(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='framedata', **kwargs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}