442 fix model checkpoint dictionary issue (Project-MONAI#443)

* [DLMED] fix dictionary issue of checkpoint

* [DLMED] fix flake8 error

* [DLMED] update according to comments

Author: Nic-Ma

monai/handlers/checkpoint_loader.py

@@ -19,7 +19,9 @@ class CheckpointLoader:
     """
     CheckpointLoader acts as an Ignite handler to load checkpoint data from file.
     It can load variables for network, optimizer, lr_scheduler.
-    And also can restore training if load the state_dict of Ignite engine.
+    If saving checkpoint after `torch.nn.DataParallel`, need to save `model.module` instead
+    as PyTorch recommended and then use this loader to load the model.
+    And also can restore training session if load the state_dict of Ignite engine.
 
     Args:
         load_path (str): the file path of checkpoint, it should be a PyTorch pth file.
@@ -48,5 +50,10 @@ def attach(self, engine):
 
     def __call__(self, engine):
         checkpoint = torch.load(self.load_path)
+        if len(self.load_dict) == 1:
+            key = list(self.load_dict.keys())[0]
+            if not (key in checkpoint):
+                checkpoint = {key: checkpoint}
+
         Checkpoint.load_objects(to_load=self.load_dict, checkpoint=checkpoint)
         self.logger.info(f"Restored all variables from {self.load_path}")

tests/test_handler_checkpoint_loader.py

@@ -0,0 +1,93 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import shutil
+import torch
+import unittest
+from ignite.engine import Engine
+import torch.optim as optim
+from monai.handlers import CheckpointSaver, CheckpointLoader
+import logging
+import sys
+
+
+class TestHandlerCheckpointLoader(unittest.TestCase):
+    def test_one_save_one_load(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        net1 = torch.nn.PReLU()
+        data1 = net1.state_dict()
+        data1["weight"] = torch.tensor([0.1])
+        net1.load_state_dict(data1)
+        net2 = torch.nn.PReLU()
+        data2 = net2.state_dict()
+        data2["weight"] = torch.tensor([0.2])
+        net2.load_state_dict(data2)
+        engine = Engine(lambda e, b: None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            save_dir = os.path.join(tempdir, "checkpoint")
+            CheckpointSaver(save_dir=save_dir, save_dict={"net": net1}, save_final=True).attach(engine)
+            engine.run([0] * 8, max_epochs=5)
+            path = save_dir + "/net_final_iteration=40.pth"
+            CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
+            engine.run([0] * 8, max_epochs=1)
+            torch.testing.assert_allclose(net2.state_dict()["weight"], 0.1)
+            shutil.rmtree(save_dir)
+
+    def test_two_save_one_load(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        net1 = torch.nn.PReLU()
+        optimizer = optim.SGD(net1.parameters(), lr=0.02)
+        data1 = net1.state_dict()
+        data1["weight"] = torch.tensor([0.1])
+        net1.load_state_dict(data1)
+        net2 = torch.nn.PReLU()
+        data2 = net2.state_dict()
+        data2["weight"] = torch.tensor([0.2])
+        net2.load_state_dict(data2)
+        engine = Engine(lambda e, b: None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            save_dir = os.path.join(tempdir, "checkpoint")
+            save_dict = {"net": net1, "opt": optimizer}
+            CheckpointSaver(save_dir=save_dir, save_dict=save_dict, save_final=True).attach(engine)
+            engine.run([0] * 8, max_epochs=5)
+            path = save_dir + "/checkpoint_final_iteration=40.pth"
+            CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
+            engine.run([0] * 8, max_epochs=1)
+            torch.testing.assert_allclose(net2.state_dict()["weight"], 0.1)
+            shutil.rmtree(save_dir)
+
+    def test_save_single_device_load_multi_devices(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        net1 = torch.nn.PReLU()
+        data1 = net1.state_dict()
+        data1["weight"] = torch.tensor([0.1])
+        net1.load_state_dict(data1)
+        net2 = torch.nn.PReLU()
+        data2 = net2.state_dict()
+        data2["weight"] = torch.tensor([0.2])
+        net2.load_state_dict(data2)
+        net2 = torch.nn.DataParallel(net2)
+        engine = Engine(lambda e, b: None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            save_dir = os.path.join(tempdir, "checkpoint")
+            CheckpointSaver(save_dir=save_dir, save_dict={"net": net1}, save_final=True).attach(engine)
+            engine.run([0] * 8, max_epochs=5)
+            path = save_dir + "/net_final_iteration=40.pth"
+            CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
+            engine.run([0] * 8, max_epochs=1)
+            torch.testing.assert_allclose(net2.state_dict()["module.weight"], 0.1)
+            shutil.rmtree(save_dir)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_handler_checkpoint_saver.py

@@ -15,7 +15,7 @@
 import torch
 import unittest
 from ignite.engine import Engine
-from monai.handlers import CheckpointSaver, CheckpointLoader
+from monai.handlers import CheckpointSaver
 import torch.optim as optim
 from parameterized import parameterized
 import logging
@@ -97,9 +97,6 @@ def _train_func(engine, batch):
             engine.run(data, max_epochs=5)
             for filename in filenames:
                 self.assertTrue(os.path.exists(os.path.join(save_dir, filename)))
-            loader = CheckpointLoader(load_path=os.path.join(save_dir, filename), load_dict={"net": net})
-            loader.attach(engine)
-            engine.run(data, max_epochs=1)
             shutil.rmtree(save_dir)