notebooks/mk_goa_kerchunk.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4344de56-7767-4686-910b-6b1c6b4089e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerchunk.netCDF3 import NetCDF3ToZarr\n",
    "from kerchunk.combine import MultiZarrToZarr\n",
    "import fsspec\n",
    "import os\n",
    "import ujson\n",
    "from pathlib import Path\n",
    "import xarray as xr\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b15f4f7c-176c-4324-90b3-2e61b8813417",
   "metadata": {},
   "outputs": [],
   "source": [
    "fs = fsspec.filesystem('s3')\n",
    "dir = 'ne_pacific_10km/json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1903301c-4713-42fc-ac97-38158f0ecd7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_01.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_02.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_03.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_04.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_05.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_06.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_07.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_08.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_09.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_10.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_11.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/nep_revised_hind_moave_2020_12.nc',\n",
       " 'noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/roms_grd_nep.nc']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flist = fs.glob('s3://noaa3100-oar-pmel-dev-cefi/ne_pacific_10km/*.nc')\n",
    "flist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3ec3842d-3a98-447b-8882-c210fdeca85f",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9822a64b-d6e5-4b8b-b9f9-45dfa8d3215e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_01.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_02.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_03.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_04.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_05.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_06.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_07.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_08.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_09.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_10.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_11.json\n",
      "ne_pacific_10km/json/nep_revised_hind_moave_2020_12.json\n",
      "ne_pacific_10km/json/roms_grd_nep.json\n"
     ]
    }
   ],
   "source": [
    "fs2 = fsspec.filesystem(\"file\")\n",
    "so = dict(default_fill_cache=False, default_cache_type='first')\n",
    "for u in flist:\n",
    "    u = 's3://' + u\n",
    "    dachunks = NetCDF3ToZarr(u, max_chunk_size=10000)\n",
    "    parts = u.split('/') # seperate file path to create a unique name for each json \n",
    "    fstem = Path(u).stem \n",
    "    outf = f'{dir}/{fstem}.json'\n",
    "    print(outf)\n",
    "    with fs2.open(outf, 'wb') as f:\n",
    "        f.write(ujson.dumps(dachunks.translate()).encode());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4e051522-1459-48ba-b1ee-90979431c911",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_01.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_02.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_03.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_04.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_05.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_06.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_07.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_08.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_09.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_10.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_11.json',\n",
       " '/home/rhs/ne_pacific_10km/json/nep_revised_hind_moave_2020_12.json']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_list = fs2.glob(f'{dir}/nep*.json')\n",
    "json_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "691a969f-9f16-469a-91c0-470ef7a012b1",
   "metadata": {},
   "source": [
    "I am pretty much following the example from this blog post and notebook:\n",
    "https://medium.com/pangeo/accessing-netcdf-and-grib-file-collections-as-cloud-native-virtual-datasets-using-kerchunk-625a2d0a9191\n",
    "https://nbviewer.org/github/peterm790/ERA5_Kerchunk_tutorial/blob/master/ERA5_tutorial.ipynb\n",
    "\n",
    "N.B. It seems that hc and Cs_r do not vary from file to file so they are in the identical list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "568337e3-7d4d-4370-8b3a-557a723d3883",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 511 ms, sys: 68.8 ms, total: 580 ms\n",
      "Wall time: 2.61 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "mzz = MultiZarrToZarr(json_list,  \n",
    "    concat_dims=['ocean_time'], #this is the dimension along which the individual files will be merged\n",
    "    identical_dims = ['lat_rho', 'lon_rho', 's_rho', 's_w'],\n",
    ")\n",
    "d = mzz.translate()\n",
    "with fs2.open(f'{dir}/combined.json', 'wb') as f:\n",
    "    f.write(ujson.dumps(d).encode())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}