minor updates to utils and seg_images_folder

make_nd_dataset.py

@@ -648,7 +648,7 @@ def read_seg_dataset_multiclass(example):
                     l = remove_small_objects(lstack[:,:,kk].astype('uint8')>0, np.pi*(FILTER_VALUE**2))
                     l = remove_small_holes(lstack[:,:,kk].astype('uint8')>0, np.pi*(FILTER_VALUE**2))
                     lstack[:,:,kk] = np.round(l).astype(np.uint8)
-                    del l
+                    # del l
 
             datadict={}
             datadict['arr_0'] = im.astype(np.uint8)

utils/recreate_saved_model.py → utils/gen_fullmodel_from_h5.py

@@ -1,5 +1,3 @@
-
-
 # Written by Dr Daniel Buscombe, Marda Science LLC
 # for  the USGS Coastal Change Hazards Program
 #
@@ -26,7 +24,7 @@
 # SOFTWARE.
 
 import sys,os, time
-sys.path.insert(1, '../src')
+# sys.path.insert(1, '../src')
 from tkinter import filedialog
 from tkinter import *
 from tkinter import messagebox
@@ -56,8 +54,8 @@
     for k in config.keys():
         exec(k+'=config["'+k+'"]')
 
-
-    from imports import *
+    from doodleverse_utils.imports import *
+    from doodleverse_utils.model_imports import *
 
     #=======================================================
     # Import the architectures for following models from doodleverse_utils
@@ -92,8 +90,6 @@
                         )
 
     elif MODEL =='simple_resunet':
-        # num_filters = 8 # initial filters
-        # model = res_unet((TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS), num_filters, NCLASSES, (KERNEL_SIZE, KERNEL_SIZE))
 
         model = simple_resunet((TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS),
                     kernel = (2, 2),
@@ -107,7 +103,6 @@
                     filters=FILTERS,#8,
                     num_layers=4,
                     strides=(1,1))
-    #346,564
     elif MODEL=='simple_unet':
         model = simple_unet((TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS),
                     kernel = (2, 2),
@@ -121,10 +116,8 @@
                     filters=FILTERS,#8,
                     num_layers=4,
                     strides=(1,1))
-    #242,812
 
     elif MODEL=='satunet':
-        #model = sat_unet((TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS), num_classes=NCLASSES)
 
         model = custom_satunet((TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS),
                     kernel = (2, 2),
@@ -139,9 +132,6 @@
                     num_layers=4,
                     strides=(1,1))
 
-
-
-    # model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = [mean_iou, dice_coef])
     model.compile(optimizer = 'adam', loss = tf.keras.losses.CategoricalCrossentropy())
 
     model.load_weights(weights)
@@ -150,7 +140,7 @@
     # use gym  make"fullmodel.h5" version which zoo can read "fullmodel.h5" 
     model.save(weights.replace('.h5','_fullmodel.h5'))
 
-    new_model = tf.keras.models.load_model(weights.replace('.h5','_fullmodel.h5'))
+    # new_model = tf.keras.models.load_model(weights.replace('.h5','_fullmodel.h5'))
 
 
 

utils/make_class_balanced_subset.py

@@ -1,22 +1,67 @@
+# Written by Dr Daniel Buscombe, Marda Science LLC
+# for the USGS Coastal Change Hazards Program
+#
+# MIT License
+#
+# Copyright (c) 2022, Marda Science LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
 import numpy as np 
 from glob import glob 
 import os, shutil
 from tqdm import tqdm
+from tkinter import filedialog, messagebox
+from tkinter import *
 
-indirec = 'v4'
-outdirec = 'v5_subset'
-
-# read files
-files = glob('v4/*.npz')
+# Request the folder containing the imagery/npz to segment 
+# sample_direc: full path to the directory
+root = Tk()
+root.filename =  filedialog.askdirectory(title = "Select directory of class-imbalanced npzs")
+indirec = root.filename
+print(indirec)
+root.withdraw()
 
-# make directory for outputs
-os.mkdir(outdirec)
-os.mkdir(outdirec+os.sep+'no_use')
+outdirec = os.path.normpath(os.path.dirname(indirec)+os.sep+os.path.basename(indirec)+'_subset')
+print(outdirec)
 
 # set minimum threshold for any proportion
 # if any normalized class frequency distribution is less than this number
 # it is discounted (moved to 'no_use')
-thres = 1e-2
+# thres = 1e-2
+print("Input threshold for minor class [0 - 1], typically <0.25")
+print("This is the smallest acceptable proportion of the minority class. Samples were minority < threshold will not be used")
+print("The smaller the threshold, the fewer the number of samples used in the subset")
+# print("\n")
+thres = float(input())
+
+print("Threshold chosen: {}".format(thres))
+
+# read files
+files = glob(indirec+os.sep+'*.npz')
+
+try:
+    # make directory for outputs
+    os.mkdir(outdirec)
+    os.mkdir(outdirec+os.sep+'no_use')
+except:
+    pass
 
 # read files one by one
 for file in tqdm(files):
@@ -25,12 +70,11 @@
         label = np.argmax(label,-1)
         # get normalized class distributions
         norm_class_dist = np.bincount(label.flatten())/np.sum(label>-1)
-        # if length > 1, copy the file
         # if below thres
         if np.any(norm_class_dist<thres):
             shutil.copyfile(file,file.replace(indirec,outdirec+os.sep+'no_use'))
             # print('below threshold')
         elif not len(norm_class_dist)==1:
             shutil.copyfile(file,file.replace(indirec,outdirec))            
-        else:
+        else: # if length > 1, copy the file
             shutil.copyfile(file,file.replace(indirec,outdirec+os.sep+'no_use'))