reproducibility/RQ1/UsersWithManyFIlesGPFS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "88d80ef6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2bbdadb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Inode (file unique ID)</th>\n",
       "      <th>KB Allocated</th>\n",
       "      <th>File Size</th>\n",
       "      <th>Creation Time in days from today</th>\n",
       "      <th>Change Time in days from today</th>\n",
       "      <th>Modification time in days from today</th>\n",
       "      <th>Acces time in days from today</th>\n",
       "      <th>GID numeric ID for the group owner of the file</th>\n",
       "      <th>UID numeric ID for the owner of the file</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100663296</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1447</td>\n",
       "      <td>1447</td>\n",
       "      <td>3131</td>\n",
       "      <td>1447</td>\n",
       "      <td>2035</td>\n",
       "      <td>762231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100663297</td>\n",
       "      <td>0</td>\n",
       "      <td>188</td>\n",
       "      <td>1447</td>\n",
       "      <td>1447</td>\n",
       "      <td>1937</td>\n",
       "      <td>1447</td>\n",
       "      <td>2010</td>\n",
       "      <td>614955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100663301</td>\n",
       "      <td>0</td>\n",
       "      <td>567</td>\n",
       "      <td>1447</td>\n",
       "      <td>1447</td>\n",
       "      <td>3142</td>\n",
       "      <td>1447</td>\n",
       "      <td>2035</td>\n",
       "      <td>762231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100663304</td>\n",
       "      <td>0</td>\n",
       "      <td>87</td>\n",
       "      <td>1447</td>\n",
       "      <td>1447</td>\n",
       "      <td>3142</td>\n",
       "      <td>1447</td>\n",
       "      <td>2035</td>\n",
       "      <td>762231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100663306</td>\n",
       "      <td>0</td>\n",
       "      <td>1689</td>\n",
       "      <td>1447</td>\n",
       "      <td>1447</td>\n",
       "      <td>1937</td>\n",
       "      <td>1447</td>\n",
       "      <td>2010</td>\n",
       "      <td>614955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999995</th>\n",
       "      <td>124522107</td>\n",
       "      <td>64</td>\n",
       "      <td>5739</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>146</td>\n",
       "      <td>2</td>\n",
       "      <td>2005</td>\n",
       "      <td>6052828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999996</th>\n",
       "      <td>124522107</td>\n",
       "      <td>64</td>\n",
       "      <td>5739</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>146</td>\n",
       "      <td>2</td>\n",
       "      <td>2005</td>\n",
       "      <td>6052828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999997</th>\n",
       "      <td>124522108</td>\n",
       "      <td>0</td>\n",
       "      <td>1051</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>161</td>\n",
       "      <td>15</td>\n",
       "      <td>2005</td>\n",
       "      <td>6052828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999998</th>\n",
       "      <td>124522108</td>\n",
       "      <td>0</td>\n",
       "      <td>1051</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>161</td>\n",
       "      <td>15</td>\n",
       "      <td>2005</td>\n",
       "      <td>6052828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999999</th>\n",
       "      <td>124522109</td>\n",
       "      <td>64</td>\n",
       "      <td>3864</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>111</td>\n",
       "      <td>15</td>\n",
       "      <td>2005</td>\n",
       "      <td>6052828</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000000 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Inode (file unique ID)  KB Allocated  File Size  \\\n",
       "0                     100663296             0          8   \n",
       "1                     100663297             0        188   \n",
       "2                     100663301             0        567   \n",
       "3                     100663304             0         87   \n",
       "4                     100663306             0       1689   \n",
       "...                         ...           ...        ...   \n",
       "9999995               124522107            64       5739   \n",
       "9999996               124522107            64       5739   \n",
       "9999997               124522108             0       1051   \n",
       "9999998               124522108             0       1051   \n",
       "9999999               124522109            64       3864   \n",
       "\n",
       "         Creation Time in days from today  Change Time in days from today  \\\n",
       "0                                    1447                            1447   \n",
       "1                                    1447                            1447   \n",
       "2                                    1447                            1447   \n",
       "3                                    1447                            1447   \n",
       "4                                    1447                            1447   \n",
       "...                                   ...                             ...   \n",
       "9999995                                15                              15   \n",
       "9999996                                15                              15   \n",
       "9999997                                15                              15   \n",
       "9999998                                15                              15   \n",
       "9999999                                15                              15   \n",
       "\n",
       "         Modification time in days from today  Acces time in days from today  \\\n",
       "0                                        3131                           1447   \n",
       "1                                        1937                           1447   \n",
       "2                                        3142                           1447   \n",
       "3                                        3142                           1447   \n",
       "4                                        1937                           1447   \n",
       "...                                       ...                            ...   \n",
       "9999995                                   146                              2   \n",
       "9999996                                   146                              2   \n",
       "9999997                                   161                             15   \n",
       "9999998                                   161                             15   \n",
       "9999999                                   111                             15   \n",
       "\n",
       "         GID numeric ID for the group owner of the file  \\\n",
       "0                                                  2035   \n",
       "1                                                  2010   \n",
       "2                                                  2035   \n",
       "3                                                  2035   \n",
       "4                                                  2010   \n",
       "...                                                 ...   \n",
       "9999995                                            2005   \n",
       "9999996                                            2005   \n",
       "9999997                                            2005   \n",
       "9999998                                            2005   \n",
       "9999999                                            2005   \n",
       "\n",
       "         UID numeric ID for the owner of the file  \n",
       "0                                          762231  \n",
       "1                                          614955  \n",
       "2                                          762231  \n",
       "3                                          762231  \n",
       "4                                          614955  \n",
       "...                                           ...  \n",
       "9999995                                   6052828  \n",
       "9999996                                   6052828  \n",
       "9999997                                   6052828  \n",
       "9999998                                   6052828  \n",
       "9999999                                   6052828  \n",
       "\n",
       "[10000000 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_names = [\"Inode (file unique ID)\",\n",
    "\"KB Allocated\",\n",
    "\"File Size\",\n",
    "\"Creation Time in days from today\",\n",
    "\"Change Time in days from today\",\n",
    "\"Modification time in days from today\",\n",
    "\"Acces time in days from today\",\n",
    "\"GID numeric ID for the group owner of the file\",\n",
    "\"UID numeric ID for the owner of the file\"]\n",
    "GPFS = pd.read_csv(\"/mnt/research/CMSE495-SS24-ICER/file_system_usage/gpfs-stats/inode-size-age-jan-23\",header=None, names = column_names, sep=\" \",nrows=1e7)\n",
    "GPFS"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99670e3a",
   "metadata": {},
   "source": [
    "# Users With Many Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7824cb42",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "UID numeric ID for the owner of the file\n",
       "0               1039\n",
       "500120        655392\n",
       "614955        677944\n",
       "638741        835389\n",
       "697349        305850\n",
       "753559        383946\n",
       "758827        264003\n",
       "762231        856363\n",
       "785573        776204\n",
       "799192         21128\n",
       "831677        366721\n",
       "834474         50287\n",
       "839641        922130\n",
       "841154        526494\n",
       "862983        303210\n",
       "875212         84531\n",
       "876249        315345\n",
       "881083        915468\n",
       "1028735        21466\n",
       "1040196       503224\n",
       "1040612       182603\n",
       "6052828       352409\n",
       "1000000088    195906\n",
       "1000000092    481751\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count the number of files for each user  ****************\n",
    "file_count_per_user = GPFS.groupby(\"UID numeric ID for the owner of the file\").size()\n",
    "\n",
    "# Display the result\n",
    "file_countperUID= file_count_per_user[file_count_per_user>1000]\n",
    "file_countperUID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ceb135a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID numeric ID for the owner of the file</th>\n",
       "      <th>Number of files</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>839641</td>\n",
       "      <td>922130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>881083</td>\n",
       "      <td>915468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>762231</td>\n",
       "      <td>856363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>638741</td>\n",
       "      <td>835389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>785573</td>\n",
       "      <td>776204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>614955</td>\n",
       "      <td>677944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>500120</td>\n",
       "      <td>655392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>841154</td>\n",
       "      <td>526494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1040196</td>\n",
       "      <td>503224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1000000092</td>\n",
       "      <td>481751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>753559</td>\n",
       "      <td>383946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>831677</td>\n",
       "      <td>366721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>6052828</td>\n",
       "      <td>352409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>876249</td>\n",
       "      <td>315345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>697349</td>\n",
       "      <td>305850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>862983</td>\n",
       "      <td>303210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>758827</td>\n",
       "      <td>264003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1000000088</td>\n",
       "      <td>195906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1040612</td>\n",
       "      <td>182603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>875212</td>\n",
       "      <td>84531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>834474</td>\n",
       "      <td>50287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1028735</td>\n",
       "      <td>21466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>799192</td>\n",
       "      <td>21128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    UID numeric ID for the owner of the file  Number of files\n",
       "11                                    839641           922130\n",
       "16                                    881083           915468\n",
       "6                                     762231           856363\n",
       "2                                     638741           835389\n",
       "7                                     785573           776204\n",
       "1                                     614955           677944\n",
       "0                                     500120           655392\n",
       "12                                    841154           526494\n",
       "18                                   1040196           503224\n",
       "22                                1000000092           481751\n",
       "4                                     753559           383946\n",
       "9                                     831677           366721\n",
       "20                                   6052828           352409\n",
       "15                                    876249           315345\n",
       "3                                     697349           305850\n",
       "13                                    862983           303210\n",
       "5                                     758827           264003\n",
       "21                                1000000088           195906\n",
       "19                                   1040612           182603\n",
       "14                                    875212            84531\n",
       "10                                    834474            50287\n",
       "17                                   1028735            21466\n",
       "8                                     799192            21128"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# identify users with many files\n",
    "def UsersWithManyFiles(data, file_limit):\n",
    "    '''\n",
    "    Function to identify users with many files using a file limit \n",
    "    as a number integer of the limit of number of files per user.\n",
    "\n",
    "    args:\n",
    "        data - dataset containing columns related to user\n",
    "        file_limit - integer resembling the file limit per user\n",
    "    \n",
    "    returns: \n",
    "        a dataframe of users that are have a greater number of total files than the \n",
    "        file limit.\n",
    "\n",
    "    '''\n",
    "\n",
    "    # Group by the user ID and count the number of files for each user\n",
    "    file_count_per_user = data.groupby(\"UID numeric ID for the owner of the file\").size()\n",
    "    \n",
    "    # Filter users who have more files than the file_limit\n",
    "    users_with_many_files = file_count_per_user[file_count_per_user > file_limit]\n",
    "    \n",
    "    # Convert the filtered Series object to a DataFrame\n",
    "    users_with_many_files_df = users_with_many_files.reset_index()\n",
    "    \n",
    "    users_with_many_files_df.columns = ['UID numeric ID for the owner of the file', 'Number of files']\n",
    "    \n",
    "    \n",
    "    # Sort by 'Number of files' in descending order\n",
    "    users_with_many_files_df.sort_values('Number of files', ascending=False, inplace=True)\n",
    "    \n",
    "    return users_with_many_files_df\n",
    "\n",
    "# test, setting file limit= 10,000\n",
    "UsersWithManyFiles(GPFS,10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe402e99",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}