Update final experiments and submodules

bgroenks96 · Mar 31, 2020 · 9abc589 · 9abc589
1 parent 6dffb1c
commit 9abc589
Show file tree

Hide file tree

Showing 10 changed files with 3,270 additions and 2,012 deletions.
diff --git a/baselines/bcsd.py b/baselines/bcsd.py
@@ -69,7 +69,10 @@ def _log(self, msg):
 
     def fit(self, lr: xr.DataArray, obsv: xr.DataArray, obsv_lr: xr.DataArray=None, batch_size=10):
         if obsv_lr is None:
-            obsv_lr = obsv.interp(lat=lr.lat, lon=lr.lon, method=self.interp)
+            lat_lo = obsv.lat.sel(lat=lr.lat, method='nearest')
+            lon_lo = obsv.lon.sel(lon=lr.lon, method='nearest')
+            obsv_lr = obsv.interp(lat=lat_lo, lon=lon_lo, method='linear')
+        assert not np.any(np.isnan(obsv_lr))
         self.hr_coords = obsv.coords
         self.hr_dims = obsv.dims
         # fit quantile map
@@ -111,7 +114,7 @@ def predict(self, lr: xr.DataArray, batch_size=10):
             y_pred_per_day.append(self.scaling_factors[day]*lr_mapped_interp)
         # concatenate and sort all days
         y_pred = xr.concat(y_pred_per_day, dim=self.time_dim)
-        y_pred = y_pred.isel({self.time_dim: y_pred.Time.argsort().values})
+        y_pred = y_pred.isel({self.time_dim: y_pred.coords[self.time_dim].argsort().values})
         y_pred = y_pred.transpose(self.time_dim, *y_pred.dims[:-1])
         return y_pred
 

diff --git a/climdex b/climdex
diff --git a/downscaling-baselines.ipynb b/downscaling-baselines.ipynb
@@ -180,12 +180,6 @@
     "lr_obs_test = ras_seus_maxt_1.isel(Time=slice(N-365,N+1)).to_array(dim='chan').transpose('Time','lat','lon','chan')\n",
     "hr_train = ras_seus_maxt_14.isel(Time=slice(0,N-365)).to_array(dim='chan').transpose('Time','lat','lon','chan')\n",
     "hr_test = ras_seus_maxt_14.isel(Time=slice(N-365,N+1)).to_array(dim='chan').transpose('Time','lat','lon','chan')\n",
-    "# lr_train, monthly_means_lr = remove_monthly_means(lr_train)\n",
-    "# lr_obs_train, monthly_means_lr_obs = remove_monthly_means(lr_obs_train)\n",
-    "# hr_train, monthly_means_hr = remove_monthly_means(hr_train)\n",
-    "# lr_test,_ = remove_monthly_means(lr_test, monthly_means_lr)\n",
-    "# lr_obs_test,_ = remove_monthly_means(lr_obs_test, monthly_means_lr_obs)\n",
-    "# hr_test,_ = remove_monthly_means(hr_test, monthly_means_hr)\n",
     "bcsd.fit(lr_train, hr_train, lr_obs_train)\n",
     "hr_pred = bcsd.predict(lr_test).values"
    ]
@@ -299,6 +293,44 @@
     "$f: X \\rightarrow \\Theta$, $\\hat{y} \\sim p(\\Theta)$"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model: \"model\"\n",
+      "_________________________________________________________________\n",
+      "Layer (type)                 Output Shape              Param #   \n",
+      "=================================================================\n",
+      "input_1 (InputLayer)         [(None, 8, 16, 1)]        0         \n",
+      "_________________________________________________________________\n",
+      "conv2d (Conv2D)              (None, 8, 16, 50)         500       \n",
+      "_________________________________________________________________\n",
+      "conv2d_1 (Conv2D)            (None, 8, 16, 25)         11275     \n",
+      "_________________________________________________________________\n",
+      "conv2d_2 (Conv2D)            (None, 8, 16, 10)         2260      \n",
+      "_________________________________________________________________\n",
+      "flatten (Flatten)            (None, 1280)              0         \n",
+      "_________________________________________________________________\n",
+      "dense (Dense)                (None, 16384)             20987904  \n",
+      "_________________________________________________________________\n",
+      "reshape (Reshape)            (None, 64, 128, 2)        0         \n",
+      "=================================================================\n",
+      "Total params: 21,001,939\n",
+      "Trainable params: 21,001,939\n",
+      "Non-trainable params: 0\n",
+      "_________________________________________________________________\n"
+     ]
+    }
+   ],
+   "source": [
+    "dscnn.create_bmd_cnn10(8,16, scale=8, c_out=2).summary()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 8,
@@ -563,7 +595,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.6"
+   "version": "3.7.3"
   }
  },
  "nbformat": 4,

diff --git a/experiments/bcsd_final.py b/experiments/bcsd_final.py
@@ -49,7 +49,6 @@ def fit_bcsd_prcp(fold, i):
     assert not np.any(np.isnan(hr_pred.values))
     hr_pred = xr.DataArray(hr_pred, coords=hr_test.coords, dims=hr_test.dims)
     y_true, y_pred = tf.constant(hr_test.values, dtype=tf.float32), tf.constant(hr_pred.values, dtype=tf.float32)
-    #y_true, y_pred = tf.math.pow(y_true, 3.0), tf.math.pow(y_pred, 3.0)
     metrics = prcp.eval_metrics(indices, y_true, y_pred, hr_test.coords)
     np.savez('/tmp/metrics.npz', **metrics)
     mlflow.log_artifact('/tmp/metrics.npz', 'data')

diff --git a/experiments/maxt_experiment_base.py b/experiments/maxt_experiment_base.py
@@ -35,25 +35,25 @@ def train_dataset(self, **kwargs):
     def test_dataset(self, **kwargs):
         return self.to_dataset(*self.test, **kwargs)
 
-def preprocess_fold_maxt(fold):
+def preprocess_fold_maxt(fold, time_dim='Time'):
     (train_lo, train_hi), (test_lo, test_hi) = fold
-    train_lo, monthly_means_lo = remove_monthly_means(train_lo)
-    train_hi, monthly_means_hi = remove_monthly_means(train_hi)
-    test_lo,_ = remove_monthly_means(test_lo, monthly_means_lo)
-    test_hi,_ = remove_monthly_means(test_hi, monthly_means_hi)
+    train_lo, monthly_means_lo = remove_monthly_means(train_lo, time_dim=time_dim)
+    train_hi, monthly_means_hi = remove_monthly_means(train_hi, time_dim=time_dim)
+    test_lo,_ = remove_monthly_means(test_lo, monthly_means_lo, time_dim=time_dim)
+    test_hi,_ = remove_monthly_means(test_hi, monthly_means_hi, time_dim=time_dim)
     return TemperatureDataFold((train_lo, train_hi),
                                (test_lo, test_hi),
                                (monthly_means_lo, monthly_means_hi))
 
-def eval_metrics(indices, true: tf.Tensor, pred: tf.Tensor, coords, monthly_means=None):
+def eval_metrics(indices, true: tf.Tensor, pred: tf.Tensor, coords, monthly_means=None, time_dim='Time'):
     # pointwise metrics
     rmse = eval_rmse(true, pred).numpy()
     bias = eval_bias(true, pred).numpy()
     corr = eval_corr(true, pred).numpy()
     # climdex indices
     pred_arr = xr.DataArray(pred.numpy(), coords=coords)
     if monthly_means is not None:
-        pred_arr = restore_monthly_means(pred_arr, monthly_means)
+        pred_arr = restore_monthly_means(pred_arr, monthly_means, time_dim=time_dim)
     txx = indices.monthly_txx(pred_arr)
     txn = indices.monthly_txn(pred_arr)
     txid = indices.annual_icing_days(pred_arr)

diff --git a/glow-jflvm-noaa-livneh-downscaling.ipynb b/glow-jflvm-noaa-livneh-downscaling.ipynb
diff --git a/noaa-livneh-analysis.ipynb b/noaa-livneh-analysis.ipynb
diff --git a/normalizing_flows b/normalizing_flows
diff --git a/qualitative-analysis.ipynb b/qualitative-analysis.ipynb
diff --git a/quantitative-analysis.ipynb b/quantitative-analysis.ipynb