get_atomic_DREAM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from io import StringIO\n",
    "import requests\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = 'Wavelength Lower_Level Lower_Type Lower_J Upper_Level Upper_Type Upper_J log_gf gA CF'\n",
    "target_url = 'https://agif.umons.ac.be/databases/tables/'\n",
    "ions = {57: 'La', \n",
    "        58: 'Ce', \n",
    "        59: 'Pr', \n",
    "        60: 'Nd', \n",
    "        61: 'Pm', \n",
    "        62: 'Sm', \n",
    "        63: 'Eu', \n",
    "        64: 'Gd', \n",
    "        65: 'Tb', \n",
    "        66: 'Dy', \n",
    "        67: 'Ho', \n",
    "        68: 'Er', \n",
    "        69: 'Tm', \n",
    "        70: 'Yb', \n",
    "        71: 'Lu'\n",
    "       }\n",
    "ion_stages = {0: 'I', 1: 'II', 2: 'III', 3: 'IV'}\n",
    "maximum_ion_stage = 3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ion_data(Z, stage):\n",
    "    ion = ions[Z] + ion_stages[stage] + '.txt'\n",
    "#     print('Now working on:', ion)\n",
    "    response = requests.get(target_url + ion)\n",
    "    if response.status_code == 404:\n",
    "        response = requests.get(target_url + ion[:-4]+'_cor.txt')\n",
    "        if response.status_code == 404:\n",
    "            print(ion[:-4], '\\t not found')\n",
    "            return\n",
    "    \n",
    "    raw_data = response.text.split('\\n', 6)[6]\n",
    "    raw_data = columns + '\\n' + raw_data\n",
    "    raw_data = re.sub(' +', ' ', raw_data)\n",
    "    raw_data = re.sub(' \\n+', '\\n', raw_data)\n",
    "#     raw_data = raw_data.replace('NORM', '-1')\n",
    "    raw_data = raw_data.replace('\\t', ' ')[:-4]\n",
    "#     raw_data = raw_data.replace('*', '')[:-4]\n",
    "#     print(raw_data)\n",
    "    virt_file = StringIO(raw_data)\n",
    "    df = pd.read_csv(virt_file, delimiter=r'\\s+', skipfooter=1, engine='python')\n",
    "    df['Z'] = Z\n",
    "    df['C'] = stage\n",
    "    df = df.set_index(['Z', 'C'])\n",
    "#     print(df)\n",
    "    print(ion[:-4], '\\t loaded')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_ion_between(Z_min, Z_max, stage_min, stage_max):\n",
    "    dataframe = pd.DataFrame(columns = ['Z', 'C'] + columns.split(' '))\n",
    "    dataframe = dataframe.set_index(['Z', 'C'])\n",
    "    for Z in range(Z_min, Z_max+1):\n",
    "        for stage in range(stage_min, stage_max+1):\n",
    "            print(Z, stage)\n",
    "            ion_df = get_ion_data(Z, stage)\n",
    "            dataframe = pd.concat([dataframe, ion_df])\n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57 0\n",
      "LaI \t loaded\n",
      "57 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/b1/mpq0hfxx6z75kyr7qm84czvc0000gn/T/ipykernel_53281/784400984.py:8: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  dataframe = pd.concat([dataframe, ion_df])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LaII \t not found\n",
      "57 2\n",
      "LaIII \t loaded\n",
      "57 3\n",
      "LaIV \t not found\n",
      "58 0\n",
      "CeI \t not found\n",
      "58 1\n",
      "CeII \t loaded\n",
      "58 2\n",
      "CeIII \t loaded\n",
      "58 3\n",
      "CeIV \t not found\n",
      "59 0\n",
      "PrI \t not found\n",
      "59 1\n",
      "PrII \t loaded\n",
      "59 2\n",
      "PrIII \t loaded\n",
      "59 3\n",
      "PrIV \t loaded\n",
      "60 0\n",
      "NdI \t not found\n",
      "60 1\n",
      "NdII \t loaded\n",
      "60 2\n",
      "NdIII \t loaded\n",
      "60 3\n",
      "NdIV \t loaded\n",
      "61 0\n",
      "PmI \t not found\n",
      "61 1\n",
      "PmII \t not found\n",
      "61 2\n",
      "PmIII \t not found\n",
      "61 3\n",
      "PmIV \t not found\n",
      "62 0\n",
      "SmI \t not found\n",
      "62 1\n",
      "SmII \t loaded\n",
      "62 2\n",
      "SmIII \t loaded\n",
      "62 3\n",
      "SmIV \t not found\n",
      "63 0\n",
      "EuI \t not found\n",
      "63 1\n",
      "EuII \t not found\n",
      "63 2\n",
      "EuIII \t loaded\n",
      "63 3\n",
      "EuIV \t not found\n",
      "64 0\n",
      "GdI \t not found\n",
      "64 1\n",
      "GdII \t not found\n",
      "64 2\n",
      "GdIII \t loaded\n",
      "64 3\n",
      "GdIV \t not found\n",
      "65 0\n",
      "TbI \t not found\n",
      "65 1\n",
      "TbII \t not found\n",
      "65 2\n",
      "TbIII \t loaded\n",
      "65 3\n",
      "TbIV \t not found\n",
      "66 0\n",
      "DyI \t not found\n",
      "66 1\n",
      "DyII \t not found\n",
      "66 2\n",
      "DyIII \t loaded\n",
      "66 3\n",
      "DyIV \t not found\n",
      "67 0\n",
      "HoI \t not found\n",
      "67 1\n",
      "HoII \t not found\n",
      "67 2\n",
      "HoIII \t loaded\n",
      "67 3\n",
      "HoIV \t not found\n",
      "68 0\n",
      "ErI \t not found\n",
      "68 1\n",
      "ErII \t loaded\n",
      "68 2\n",
      "ErIII \t loaded\n",
      "68 3\n",
      "ErIV \t not found\n",
      "69 0\n",
      "TmI \t not found\n",
      "69 1\n",
      "TmII \t loaded\n",
      "69 2\n",
      "TmIII \t loaded\n",
      "69 3\n",
      "TmIV \t loaded\n",
      "70 0\n",
      "YbI \t not found\n",
      "70 1\n",
      "YbII \t loaded\n",
      "70 2\n",
      "YbIII \t loaded\n",
      "70 3\n",
      "YbIV \t loaded\n",
      "71 0\n",
      "LuI \t loaded\n",
      "71 1\n",
      "LuII \t loaded\n",
      "71 2\n",
      "LuIII \t loaded\n",
      "71 3\n",
      "LuIV \t not found\n"
     ]
    }
   ],
   "source": [
    "# get_ion_data(60, 1)\n",
    "atomic_data = get_all_ion_between(57, 71, 0, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Wavelength</th>\n",
       "      <th>Lower_Level</th>\n",
       "      <th>Lower_Type</th>\n",
       "      <th>Lower_J</th>\n",
       "      <th>Upper_Level</th>\n",
       "      <th>Upper_Type</th>\n",
       "      <th>Upper_J</th>\n",
       "      <th>log_gf</th>\n",
       "      <th>gA</th>\n",
       "      <th>CF</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z</th>\n",
       "      <th>C</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">57</th>\n",
       "      <th>0</th>\n",
       "      <td>3175.982</td>\n",
       "      <td>0</td>\n",
       "      <td>(e)</td>\n",
       "      <td>1.5</td>\n",
       "      <td>31477</td>\n",
       "      <td>(o)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-0.74</td>\n",
       "      <td>121000000.0</td>\n",
       "      <td>0.047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3215.810</td>\n",
       "      <td>1053</td>\n",
       "      <td>(e)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>32141</td>\n",
       "      <td>(o)</td>\n",
       "      <td>3.5</td>\n",
       "      <td>-0.45</td>\n",
       "      <td>232000000.0</td>\n",
       "      <td>0.052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3235.643</td>\n",
       "      <td>0</td>\n",
       "      <td>(e)</td>\n",
       "      <td>1.5</td>\n",
       "      <td>30897</td>\n",
       "      <td>(o)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-1.94</td>\n",
       "      <td>7420000.0</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3247.034</td>\n",
       "      <td>0</td>\n",
       "      <td>(e)</td>\n",
       "      <td>1.5</td>\n",
       "      <td>30788</td>\n",
       "      <td>(o)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-0.99</td>\n",
       "      <td>63400000.0</td>\n",
       "      <td>0.064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3285.926</td>\n",
       "      <td>1053</td>\n",
       "      <td>(e)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>31477</td>\n",
       "      <td>(o)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-1.74</td>\n",
       "      <td>11400000.0</td>\n",
       "      <td>0.018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">71</th>\n",
       "      <th>2</th>\n",
       "      <td>7310.129</td>\n",
       "      <td>86681</td>\n",
       "      <td>(e)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>100357</td>\n",
       "      <td>(o)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>122000000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7534.288</td>\n",
       "      <td>92322</td>\n",
       "      <td>(e)</td>\n",
       "      <td>1.5</td>\n",
       "      <td>105591</td>\n",
       "      <td>(o)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.44</td>\n",
       "      <td>321000000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7936.530</td>\n",
       "      <td>93108</td>\n",
       "      <td>(e)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>105704</td>\n",
       "      <td>(o)</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.57</td>\n",
       "      <td>393000000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8008.692</td>\n",
       "      <td>93108</td>\n",
       "      <td>(e)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>105591</td>\n",
       "      <td>(o)</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-0.74</td>\n",
       "      <td>19100000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9530.983</td>\n",
       "      <td>92322</td>\n",
       "      <td>(e)</td>\n",
       "      <td>1.5</td>\n",
       "      <td>102811</td>\n",
       "      <td>(o)</td>\n",
       "      <td>1.5</td>\n",
       "      <td>-0.69</td>\n",
       "      <td>14900000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>71444 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Wavelength Lower_Level Lower_Type  Lower_J Upper_Level Upper_Type  \\\n",
       "Z  C                                                                      \n",
       "57 0    3175.982           0        (e)      1.5       31477        (o)   \n",
       "   0    3215.810        1053        (e)      2.5       32141        (o)   \n",
       "   0    3235.643           0        (e)      1.5       30897        (o)   \n",
       "   0    3247.034           0        (e)      1.5       30788        (o)   \n",
       "   0    3285.926        1053        (e)      2.5       31477        (o)   \n",
       "...          ...         ...        ...      ...         ...        ...   \n",
       "71 2    7310.129       86681        (e)      0.5      100357        (o)   \n",
       "   2    7534.288       92322        (e)      1.5      105591        (o)   \n",
       "   2    7936.530       93108        (e)      2.5      105704        (o)   \n",
       "   2    8008.692       93108        (e)      2.5      105591        (o)   \n",
       "   2    9530.983       92322        (e)      1.5      102811        (o)   \n",
       "\n",
       "      Upper_J  log_gf           gA     CF  \n",
       "Z  C                                       \n",
       "57 0      2.5   -0.74  121000000.0  0.047  \n",
       "   0      3.5   -0.45  232000000.0  0.052  \n",
       "   0      2.5   -1.94    7420000.0   0.06  \n",
       "   0      2.5   -0.99   63400000.0  0.064  \n",
       "   0      2.5   -1.74   11400000.0  0.018  \n",
       "...       ...     ...          ...    ...  \n",
       "71 2      0.5   -0.01  122000000.0    1.0  \n",
       "   2      2.5    0.44  321000000.0    1.0  \n",
       "   2      3.5    0.57  393000000.0    1.0  \n",
       "   2      2.5   -0.74   19100000.0    1.0  \n",
       "   2      1.5   -0.69   14900000.0    1.0  \n",
       "\n",
       "[71444 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "atomic_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DREAM_atomic_data_20241106-1325.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/b1/mpq0hfxx6z75kyr7qm84czvc0000gn/T/ipykernel_53281/1789134644.py:5: FutureWarning: Starting with pandas version 3.0 all arguments of to_hdf except for the argument 'path_or_buf' will be keyword-only.\n",
      "  atomic_data.to_hdf(output_name, 'atomic_data')\n",
      "/var/folders/b1/mpq0hfxx6z75kyr7qm84czvc0000gn/T/ipykernel_53281/1789134644.py:5: PerformanceWarning: \n",
      "your performance may suffer as PyTables will pickle object types that it cannot\n",
      "map directly to c-types [inferred_type->integer,key->axis1_level0] [items->None]\n",
      "\n",
      "  atomic_data.to_hdf(output_name, 'atomic_data')\n",
      "/var/folders/b1/mpq0hfxx6z75kyr7qm84czvc0000gn/T/ipykernel_53281/1789134644.py:5: PerformanceWarning: \n",
      "your performance may suffer as PyTables will pickle object types that it cannot\n",
      "map directly to c-types [inferred_type->integer,key->axis1_level1] [items->None]\n",
      "\n",
      "  atomic_data.to_hdf(output_name, 'atomic_data')\n",
      "/var/folders/b1/mpq0hfxx6z75kyr7qm84czvc0000gn/T/ipykernel_53281/1789134644.py:5: PerformanceWarning: \n",
      "your performance may suffer as PyTables will pickle object types that it cannot\n",
      "map directly to c-types [inferred_type->mixed-integer,key->block1_values] [items->Index(['Lower_Level', 'Lower_Type', 'Upper_Level', 'Upper_Type', 'CF'], dtype='object')]\n",
      "\n",
      "  atomic_data.to_hdf(output_name, 'atomic_data')\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "timestr = time.strftime(\"%Y%m%d-%H%M\")\n",
    "output_name = 'DREAM_atomic_data_'+timestr+'.h5'\n",
    "print(output_name)\n",
    "atomic_data.to_hdf(output_name, 'atomic_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}