teacher training files are updated for a better logging

Author: alldbi

dataset/__init__.py

@@ -95,14 +95,14 @@ class CIFAR100Instance(datasets.CIFAR100):
 
     def __getitem__(self, index):
 
-        if torch.__version__[0] == '0':
+        # if torch.__version__[0] == '0':
 
-            if self.train:
-                img, target = self.train_data[index], self.train_labels[index]
-            else:
-                img, target = self.test_data[index], self.test_labels[index]
+        if self.train:
+            img, target = self.train_data[index], self.train_labels[index]
         else:
-            img, target = self.data[index], self.targets[index]
+            img, target = self.test_data[index], self.test_labels[index]
+        # else:
+        #     img, target = self.data[index], self.targets[index]
 
         # doing this so that it is consistent with all other datasets
         # to return a PIL Image

dataset/cifar100.py

@@ -162,18 +162,21 @@ def forward(self, x, is_feat=False, preact=False):
     def _initialize_weights(self):
         for m in self.modules():
             if isinstance(m, nn.Conv2d):
-                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels #/ m.groups
+                # print(m.kernel_size[0], m.kernel_size[1], m.in_channels, m.out_channels, m.groups)
                 m.weight.data.normal_(0, math.sqrt(2. / n))
                 if m.bias is not None:
                     m.bias.data.zero_()
+
             elif isinstance(m, nn.BatchNorm2d):
                 m.weight.data.fill_(1)
                 m.bias.data.zero_()
             elif isinstance(m, nn.Linear):
                 n = m.weight.size(1)
                 m.weight.data.normal_(0, 0.01)
                 m.bias.data.zero_()
-
+        print("initializing done!!!")
+        # exit()
 
 def mobilenetv2_T_w(T, W, feature_dim=100):
     model = MobileNetV2(T=T, feature_dim=feature_dim, width_mult=W)

models/mobilenetv2.py

@@ -18,6 +18,7 @@
 from helper.util import get_teacher_name
 from models import model_dict
 import math
+import imageio
 
 
 def load_teacher(model_path, n_cls):
@@ -341,7 +342,7 @@ def augment(plot=True):
 
                 img = img.astype(np.uint8)
 
-                misc.imsave(save_dir + '/' + str(counter + i) + '.png', img)
+                imageio.imwrite(save_dir + '/' + str(counter + i) + '.png', img)
 
             counter += n_suc
 
@@ -385,10 +386,10 @@ def count_parameters(model):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('--path_t', type=str, default='./save/models/wrn_40_2_vanilla/ckpt_epoch_240.pth',
+    parser.add_argument('--path_t', type=str, default='./save/models/resnet110_vanilla/ckpt_epoch_240.pth',
                         help='teacher model snapshot')
     parser.add_argument('--device', type=str, default='cuda:0', help='cuda or cpu')
-    parser.add_argument('--save_dir', type=str, default='/home/aldb/outputs/new2',
+    parser.add_argument('--save_dir', type=str, default='/home/mehdi/output',
                         help='output directory to save results')
     parser.add_argument('--bs', type=int, default=100, help='batch size for dataloader')
     parser.add_argument('--aug_size', type=int, default=500000, help='number of samples to generate')

supermix.py

@@ -81,7 +81,7 @@ def parse_option():
     parser.add_argument('--hint_layer', default=2, type=int, choices=[0, 1, 2, 3, 4])
 
     parser.add_argument('--test_interval', type=int, default=None, help='test interval')
-    parser.add_argument('--seed', default=102, type=int, help='random seed')
+    parser.add_argument('--seed', default=19, type=int, help='random seed')
 
     opt = parser.parse_args()
 
@@ -96,21 +96,21 @@ def parse_option():
 
     # gamma = [0.1, 0.3, 0.5, 0.7, 0.9]
 
-    student_list = [8, 9, 10, 11, 12]
+    student_list =range(10, 13)
 
     k_list = [3]
     k_list.reverse()
-    for k in k_list:
+    for s in student_list:
         opt = parse_option()
         # opt.aug_size = a
         opt.aug_alpha = 3
         opt.aug_lambda = -1
-        opt.gamma = 2
-        opt.alpha = 0
-        opt.aug_type = 'supermix'
-        opt.trial = "07Feb20"
-        s = 0
-        opt.aug_k = k
+        opt.gamma = 1
+        opt.alpha = 0.5
+        opt.aug_type = 'mixup'
+        opt.trial = "12Feb20_originit"
+        # s = 0
+        opt.aug_k = 2
 
 
         if s==0:
@@ -142,9 +142,11 @@ def parse_option():
         elif s==8:
             opt.model_s = 'MobileNetV2'
             opt.path_t = './save/models/ResNet50_vanilla/ckpt_epoch_240.pth'
+            opt.batch_size=64
         elif s==9:
             opt.model_s = 'vgg8'
             opt.path_t = './save/models/ResNet50_vanilla/ckpt_epoch_240.pth'
+            opt.batch_size = 64
         elif s==10:
             opt.model_s = 'ShuffleV1'
             opt.path_t = './save/models/resnet32x4_vanilla/ckpt_epoch_240.pth'

train_series.py

@@ -30,18 +30,20 @@ def parse_option():
     parser.add_argument('--batch_size', type=int, default=128, help='batch_size')
     parser.add_argument('--num_workers', type=int, default=8, help='num of workers to use')
     parser.add_argument('--epochs', type=int, default=600, help='number of training epochs')
-    parser.add_argument('--device', type=str, default='cuda:1', help='batch_size')
+    parser.add_argument('--device', type=str, default='cuda:0', help='batch_size')
 
     # optimization
-    parser.add_argument('--learning_rate', type=float, default=0.1, help='learning rate')
+    parser.add_argument('--learning_rate', type=float, default=0.02, help='learning rate')
     parser.add_argument('--lr_decay_epochs', type=str, default='200, 300, 400, 500', help='where to decay lr, can be a list')
     parser.add_argument('--lr_decay_rate', type=float, default=0.1, help='decay rate for learning rate')
     parser.add_argument('--weight_decay', type=float, default=5e-4, help='weight decay')
     parser.add_argument('--momentum', type=float, default=0.9, help='momentum')
     parser.add_argument('--aug', type=str, default=None,
                         help='address of the augmented dataset')
+    parser.add_argument('--aug_type', type=str, default=None,
+                        help='address of the augmented dataset')
     # dataset
-    parser.add_argument('--model', type=str, default='vgg8',
+    parser.add_argument('--model', type=str, default='MobileNetV2',
                         choices=['resnet8', 'resnet14', 'resnet20', 'resnet32', 'resnet44', 'resnet56', 'resnet110',
                                  'resnet8x4', 'resnet32x4', 'wrn_16_1', 'wrn_16_2', 'wrn_40_1', 'wrn_40_2',
                                  'vgg8', 'vgg11', 'vgg13', 'vgg16', 'vgg19',
@@ -53,8 +55,8 @@ def parse_option():
     opt = parser.parse_args()
 
     # set different learning rate from these 4 models
-    if opt.model in ['MobileNetV2', 'ShuffleV1', 'ShuffleV2']:
-        opt.learning_rate = 0.01
+    # if opt.model in ['MobileNetV2', 'ShuffleV1', 'ShuffleV2']:
+    #     opt.learning_rate = 0.01
 
     # set the path according to the environment
 
@@ -96,6 +98,8 @@ def main():
                           momentum=opt.momentum,
                           weight_decay=opt.weight_decay)
 
+    print("learning rate:", opt.learning_rate)
+
     criterion = nn.CrossEntropyLoss()
 
     if torch.cuda.is_available():