From 5322a03e1b46765ffac38bc379063681c4c2621f Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 13 Aug 2023 12:15:47 +0000 Subject: [PATCH 1/3] add command line of TensorRT conversion and inference for brats_mri_generative_diffusion bundle Signed-off-by: binliu --- .../configs/inference_trt.json | 11 ++++++++++ .../configs/metadata.json | 3 ++- .../docs/README.md | 20 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 models/brats_mri_generative_diffusion/configs/inference_trt.json diff --git a/models/brats_mri_generative_diffusion/configs/inference_trt.json b/models/brats_mri_generative_diffusion/configs/inference_trt.json new file mode 100644 index 00000000..b878ebaf --- /dev/null +++ b/models/brats_mri_generative_diffusion/configs/inference_trt.json @@ -0,0 +1,11 @@ +{ + "imports": [ + "$from datetime import datetime", + "$from pathlib import Path", + "$import torch_tensorrt" + ], + "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder_trt.ts'", + "load_autoencoder": "$torch.jit.load(@load_autoencoder_path)", + "load_diffusion_path": "$@bundle_root + '/models/model_trt.ts'", + "load_diffusion": "$torch.jit.load(@load_diffusion_path)" +} diff --git a/models/brats_mri_generative_diffusion/configs/metadata.json b/models/brats_mri_generative_diffusion/configs/metadata.json index c5a22d79..41e39f13 100644 --- a/models/brats_mri_generative_diffusion/configs/metadata.json +++ b/models/brats_mri_generative_diffusion/configs/metadata.json @@ -1,7 +1,8 @@ { "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_ldm_20230507.json", - "version": "1.0.6", + "version": "1.0.7", "changelog": { + "1.0.7": "add support for TensorRT conversion and inference", "1.0.6": "update with new lr scheduler api in inference", "1.0.5": "fix the wrong GPU index issue of multi-node", "1.0.4": "update with new lr scheduler api", diff --git a/models/brats_mri_generative_diffusion/docs/README.md b/models/brats_mri_generative_diffusion/docs/README.md index 1c01d861..692e7a75 100644 --- a/models/brats_mri_generative_diffusion/docs/README.md +++ b/models/brats_mri_generative_diffusion/docs/README.md @@ -151,6 +151,26 @@ The Autoencoder can be exported into a TorchScript file. python -m monai.bundle ckpt_export autoencoder_def --filepath models/model_autoencoder.ts --ckpt_file models/model_autoencoder.pt --meta_file configs/metadata.json --config_file configs/inference.json ``` +#### Export checkpoint to TensorRT based models with fp32 or fp16 precision: + +The Autoencoder can be exported into a TensorRT model. + +``` +python -m monai.bundle trt_export --net_id autoencoder_def --filepath models/model_autoencoder_trt.ts --ckpt_file models/model_autoencoder.pt --meta_file configs/metadata.json --config_file configs/inference.json --use_trace "True" --input_shape "[1, 1, 112, 128, 80]" --precision --converter_kwargs "{'truncate_long_and_double': True}" +``` + +The Diffusion model can be exported into a TensorRT model. + +``` + python -m monai.bundle trt_export --net_id network_def --filepath models/model_trt.ts --ckpt_file models/model.pt --meta_file configs/metadata.json --config_file configs/inference.json --precision --use_trace "True" --input_shape "[[1, 8, 36, 44, 28], [1,]]" --converter_kwargs "{'truncate_long_and_double': True}" +``` + +#### Execute inference with the TensorRT model: + +``` +python -m monai.bundle run --config_file "['configs/inference.json', 'configs/inference_trt.json']" +``` + # References [1] Rombach, Robin, et al. "High-resolution image synthesis with latent diffusion models." Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2022. https://openaccess.thecvf.com/content/CVPR2022/papers/Rombach_High-Resolution_Image_Synthesis_With_Latent_Diffusion_Models_CVPR_2022_paper.pdf From 1ac569bb678f3a9d8363eb235845f00bef31a75d Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 13 Aug 2023 13:53:49 +0000 Subject: [PATCH 2/3] add trt support for the brats_mri_axial_slices_generative_diffusion bundle Signed-off-by: binliu --- .../configs/inference_trt.json | 13 ++++++++++++ .../configs/metadata.json | 3 ++- .../docs/README.md | 20 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 models/brats_mri_axial_slices_generative_diffusion/configs/inference_trt.json diff --git a/models/brats_mri_axial_slices_generative_diffusion/configs/inference_trt.json b/models/brats_mri_axial_slices_generative_diffusion/configs/inference_trt.json new file mode 100644 index 00000000..a32a6519 --- /dev/null +++ b/models/brats_mri_axial_slices_generative_diffusion/configs/inference_trt.json @@ -0,0 +1,13 @@ +{ + "imports": [ + "$from datetime import datetime", + "$from pathlib import Path", + "$from PIL import Image", + "$from scripts.utils import visualize_2d_image", + "$import torch_tensorrt" + ], + "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder_trt.ts'", + "load_autoencoder": "$torch.jit.load(@load_autoencoder_path)", + "load_diffusion_path": "$@bundle_root + '/models/model_trt.ts'", + "load_diffusion": "$torch.jit.load(@load_diffusion_path)" +} diff --git a/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json b/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json index 19be69af..42f82db4 100644 --- a/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json +++ b/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json @@ -1,7 +1,8 @@ { "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_ldm_20230507.json", - "version": "1.0.6", + "version": "1.0.7", "changelog": { + "1.0.7": "add support for TensorRT conversion and inference", "1.0.6": "update with new lr scheduler api in inference", "1.0.5": "fix the wrong GPU index issue of multi-node", "1.0.4": "update with new lr scheduler api", diff --git a/models/brats_mri_axial_slices_generative_diffusion/docs/README.md b/models/brats_mri_axial_slices_generative_diffusion/docs/README.md index 5c471a8f..74c3ce1f 100644 --- a/models/brats_mri_axial_slices_generative_diffusion/docs/README.md +++ b/models/brats_mri_axial_slices_generative_diffusion/docs/README.md @@ -143,6 +143,26 @@ The following code generates a synthetic image from a random sampled noise. python -m monai.bundle run --config_file configs/inference.json ``` +#### Export checkpoint to TensorRT based models with fp32 or fp16 precision: + +The Autoencoder can be exported into a TensorRT model. + +``` +python -m monai.bundle trt_export --net_id autoencoder_def --filepath models/model_autoencoder_trt.ts --ckpt_file models/model_autoencoder.pt --meta_file configs/metadata.json --config_file configs/inference.json --use_trace "True" --input_shape "[1, 1, 240, 240]" --precision --converter_kwargs "{'truncate_long_and_double': True}" +``` + +The Diffusion model can be exported into a TensorRT model. + +``` + python -m monai.bundle trt_export --net_id network_def --filepath models/model_trt.ts --ckpt_file models/model.pt --meta_file configs/metadata.json --config_file configs/inference.json --precision --use_trace "True" --input_shape "[[1, 1, 64, 64], [1,]]" --converter_kwargs "{'truncate_long_and_double': True}" +``` + +#### Execute inference with the TensorRT model: + +``` +python -m monai.bundle run --config_file "['configs/inference.json', 'configs/inference_trt.json']" +``` + # References [1] Rombach, Robin, et al. "High-resolution image synthesis with latent diffusion models." Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2022. https://openaccess.thecvf.com/content/CVPR2022/papers/Rombach_High-Resolution_Image_Synthesis_With_Latent_Diffusion_Models_CVPR_2022_paper.pdf From 934e839e8a3af3d5c43d2581b307ab6f9c6bd673 Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 20 Aug 2023 13:37:22 +0000 Subject: [PATCH 3/3] add suppor t of trt inference of the pathology nuclei segmentation classification bundle Signed-off-by: binliu --- .../configs/inference.json | 15 ++++++++++++++- .../configs/inference_trt.json | 11 +++++++++++ .../docs/README.md | 12 ++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 models/pathology_nuclei_segmentation_classification/configs/inference_trt.json diff --git a/models/pathology_nuclei_segmentation_classification/configs/inference.json b/models/pathology_nuclei_segmentation_classification/configs/inference.json index 53c9adab..adf7c87f 100644 --- a/models/pathology_nuclei_segmentation_classification/configs/inference.json +++ b/models/pathology_nuclei_segmentation_classification/configs/inference.json @@ -12,13 +12,16 @@ "hovernet_mode": "fast", "patch_size": 256, "out_size": 164, + "non_trt_inference": true, + "use_list_output": "$not @non_trt_inference", "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", "network_def": { "_target_": "HoVerNet", "mode": "@hovernet_mode", "adapt_standard_resnet": true, "in_channels": 3, - "out_classes": 5 + "out_classes": 5, + "use_list_output": "@use_list_output" }, "network": "$@network_def.to(@device)", "preprocessing": { @@ -77,6 +80,16 @@ "postprocessing": { "_target_": "Compose", "transforms": [ + { + "_target_": "ExtendSubKeysd", + "_disabled_": "@non_trt_inference", + "keys": "pred", + "map_names": [ + "horizontal_vertical", + "nucleus_prediction", + "type_prediction" + ] + }, { "_target_": "FlattenSubKeysd", "keys": "pred", diff --git a/models/pathology_nuclei_segmentation_classification/configs/inference_trt.json b/models/pathology_nuclei_segmentation_classification/configs/inference_trt.json new file mode 100644 index 00000000..2bd6b253 --- /dev/null +++ b/models/pathology_nuclei_segmentation_classification/configs/inference_trt.json @@ -0,0 +1,11 @@ +{ + "imports": [ + "$import glob", + "$import os", + "$import torch_tensorrt" + ], + "non_trt_inference": false, + "handlers#0#_disabled_": true, + "network_def": "$torch.jit.load(@bundle_root + '/models/model_trt.ts')", + "evaluator#amp": false +} diff --git a/models/pathology_nuclei_segmentation_classification/docs/README.md b/models/pathology_nuclei_segmentation_classification/docs/README.md index c9dd9b83..509c0f40 100644 --- a/models/pathology_nuclei_segmentation_classification/docs/README.md +++ b/models/pathology_nuclei_segmentation_classification/docs/README.md @@ -135,6 +135,18 @@ python -m monai.bundle run --config_file "['configs/train.json','configs/evaluat python -m monai.bundle run --config_file configs/inference.json ``` +#### Export checkpoint to TensorRT based models with fp32 or fp16 precision: + +``` +python -m monai.bundle trt_export --net_id network_def --filepath models/model_trt.ts --ckpt_file models/model.pt --meta_file configs/metadata.json --config_file configs/inference.json --precision --use_trace "True" --use_onnx "True" --dynamic_batchsize "[1, 16, 16]" --onnx_output_names "['output_0', 'output_1', 'output_2']" --use_list_output "True" +``` + +#### Execute inference with the TensorRT model: + +``` +python -m monai.bundle run --config_file "['configs/inference.json', 'configs/inference_trt.json']" +``` + # References [1] Simon Graham, Quoc Dang Vu, Shan E Ahmed Raza, Ayesha Azam, Yee Wah Tsang, Jin Tae Kwak, Nasir Rajpoot, Hover-Net: Simultaneous segmentation and classification of nuclei in multi-tissue histology images, Medical Image Analysis, 2019 https://doi.org/10.1016/j.media.2019.101563