From 578516058ff1d7335e1acd7fb660751181bc01c3 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 29 Apr 2024 09:12:19 -0400 Subject: [PATCH 01/28] Download and extract VisDrone2019 zip files --- .../object_detection/datasets/visdrone.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 examples/src/armory/examples/object_detection/datasets/visdrone.py diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py new file mode 100644 index 00000000..3ed4f9b1 --- /dev/null +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -0,0 +1,85 @@ +"""Utilities to load the VisDrone 2019 dataset.""" + +from pathlib import Path +import zipfile + +import requests +from tqdm import tqdm + +from armory.evaluation import SysConfig + + +def download_from_gdrive(fileid: str, filepath: Path) -> None: + url = "https://drive.usercontent.google.com/download" + response = requests.get(url, params={"id": fileid, "confirm": "1"}, stream=True) + + total_size = int(response.headers.get("content-length", 0)) + block_size = 1024 + + with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: + with open(filepath, "wb") as f: + for data in response.iter_content(block_size): + f.write(data) + pbar.update(len(data)) + + if total_size != 0 and pbar.n != total_size: + raise RuntimeError("Download failed") + + +def download_visdrone(cachedir: Path): + train_fileid = "1a2oHjcEcwXP8oUF95qiwrqzACb2YlUhn" + train_filename = "VisDrone2019-DET-train.zip" + train_filepath = cachedir / train_filename + if not train_filepath.exists(): + print(f"Downloading to {train_filepath}") + download_from_gdrive(train_fileid, train_filepath) + else: + print(f"Using cached {train_filepath}") + train_dir = cachedir / "train" / "VisDrone2019-DET-train" + if not train_dir.exists(): + if zipfile.is_zipfile(train_filepath): + with zipfile.ZipFile(train_filepath, "r") as zip_ref: + zip_ref.extractall(cachedir / "train") + else: + raise RuntimeError( + f"Download of training data failed, {train_filepath} not a zip file" + ) + train_dir = cachedir / "train" / "VisDrone2019-DET-train" + if not train_dir.exists(): + raise RuntimeError( + f"Extraction of training data failed, {train_dir} not found" + ) + else: + print(f"Using cached {train_dir}") + + test_fileid = "1bxK5zgLn0_L8x276eKkuYA_FzwCIjb59" + test_filename = "VisDrone2019-DET-val.zip" + test_filepath = cachedir / test_filename + if not test_filepath.exists(): + print(f"Downloading to {test_filepath}") + download_from_gdrive(test_fileid, test_filepath) + else: + print(f"Using cached {test_filepath}") + test_dir = cachedir / "test" / "VisDrone2019-DET-val" + if not test_dir.exists(): + if zipfile.is_zipfile(test_filepath): + with zipfile.ZipFile(test_filepath, "r") as zip_ref: + zip_ref.extractall(cachedir / "test") + else: + raise RuntimeError( + f"Download of test data failed, {test_filepath} not a zip file" + ) + if not test_dir.exists(): + raise RuntimeError(f"Extraction of test data failed, {test_dir} not found") + else: + print(f"Using cached {test_dir}") + + return train_dir, test_dir + + +if __name__ == "__main__": + sysconfig = SysConfig() + cachedir = sysconfig.dataset_cache / "visdrone2019" + cachedir.mkdir(parents=True, exist_ok=True) + + train_dir, test_dir = download_visdrone(cachedir) From 19c5a559d5b61afa879756a4ab484d91ab4c1ceb Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 29 Apr 2024 12:55:07 -0400 Subject: [PATCH 02/28] Load dataset with HuggingFace and provider Armory wrapper --- .../object_detection/datasets/visdrone.py | 221 ++++++++++++------ 1 file changed, 152 insertions(+), 69 deletions(-) diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py index 3ed4f9b1..a29569a1 100644 --- a/examples/src/armory/examples/object_detection/datasets/visdrone.py +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -1,85 +1,168 @@ """Utilities to load the VisDrone 2019 dataset.""" +import csv +import io from pathlib import Path -import zipfile +from pprint import pprint +from typing import Any, Dict, Iterator, List, Tuple -import requests -from tqdm import tqdm +import datasets -from armory.evaluation import SysConfig +import armory.data +import armory.dataset -def download_from_gdrive(fileid: str, filepath: Path) -> None: - url = "https://drive.usercontent.google.com/download" - response = requests.get(url, params={"id": fileid, "confirm": "1"}, stream=True) +def create_dataloader( + dataset: datasets.Dataset, **kwargs +) -> armory.dataset.ObjectDetectionDataLoader: + """ + Create an Armory object detection dataloader for the given VisDrone2019 dataset split. - total_size = int(response.headers.get("content-length", 0)) - block_size = 1024 + Args: + dataset: VisDrone2019 dataset split + **kwargs: Additional keyword arguments to pass to the dataloader constructor - with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: - with open(filepath, "wb") as f: - for data in response.iter_content(block_size): - f.write(data) - pbar.update(len(data)) + Return: + Armory object detection dataloader + """ + return armory.dataset.ObjectDetectionDataLoader( + dataset, + image_key="image", + dim=armory.data.ImageDimensions.CHW, + scale=armory.data.Scale( + dtype=armory.data.DataType.UINT8, + max=255, + ), + objects_key="objects", + boxes_key="bbox", + format=armory.data.BBoxFormat.XYWH, + labels_key="category", + **kwargs, + ) - if total_size != 0 and pbar.n != total_size: - raise RuntimeError("Download failed") +GDRIVE_VAL_URL = "https://drive.usercontent.google.com/download?id=1bxK5zgLn0_L8x276eKkuYA_FzwCIjb59&confirm=1" +GDRIVE_TRAIN_URL = "https://drive.usercontent.google.com/download?id=1a2oHjcEcwXP8oUF95qiwrqzACb2YlUhn&confirm=1" -def download_visdrone(cachedir: Path): - train_fileid = "1a2oHjcEcwXP8oUF95qiwrqzACb2YlUhn" - train_filename = "VisDrone2019-DET-train.zip" - train_filepath = cachedir / train_filename - if not train_filepath.exists(): - print(f"Downloading to {train_filepath}") - download_from_gdrive(train_fileid, train_filepath) - else: - print(f"Using cached {train_filepath}") - train_dir = cachedir / "train" / "VisDrone2019-DET-train" - if not train_dir.exists(): - if zipfile.is_zipfile(train_filepath): - with zipfile.ZipFile(train_filepath, "r") as zip_ref: - zip_ref.extractall(cachedir / "train") - else: - raise RuntimeError( - f"Download of training data failed, {train_filepath} not a zip file" - ) - train_dir = cachedir / "train" / "VisDrone2019-DET-train" - if not train_dir.exists(): - raise RuntimeError( - f"Extraction of training data failed, {train_dir} not found" - ) - else: - print(f"Using cached {train_dir}") - - test_fileid = "1bxK5zgLn0_L8x276eKkuYA_FzwCIjb59" - test_filename = "VisDrone2019-DET-val.zip" - test_filepath = cachedir / test_filename - if not test_filepath.exists(): - print(f"Downloading to {test_filepath}") - download_from_gdrive(test_fileid, test_filepath) - else: - print(f"Using cached {test_filepath}") - test_dir = cachedir / "test" / "VisDrone2019-DET-val" - if not test_dir.exists(): - if zipfile.is_zipfile(test_filepath): - with zipfile.ZipFile(test_filepath, "r") as zip_ref: - zip_ref.extractall(cachedir / "test") - else: - raise RuntimeError( - f"Download of test data failed, {test_filepath} not a zip file" - ) - if not test_dir.exists(): - raise RuntimeError(f"Extraction of test data failed, {test_dir} not found") - else: - print(f"Using cached {test_dir}") - return train_dir, test_dir +def load_dataset() -> datasets.DatasetDict: + """ + Load the train and validation splits of the VisDrone2019 dataset. + Return: + Dictionary containing the train and validation splits + """ + dl_manager = datasets.DownloadManager(dataset_name="VisDrone2019") + ds_features = features() + paths = dl_manager.download({"train": GDRIVE_TRAIN_URL, "val": GDRIVE_VAL_URL}) + train_files = dl_manager.iter_archive(paths["train"]) + val_files = dl_manager.iter_archive(paths["val"]) + return datasets.DatasetDict( + { + "train": datasets.Dataset.from_generator( + generate_samples, + gen_kwargs={"files": train_files}, + features=ds_features, + ), + "val": datasets.Dataset.from_generator( + generate_samples, + gen_kwargs={"files": val_files}, + features=ds_features, + ), + } + ) -if __name__ == "__main__": - sysconfig = SysConfig() - cachedir = sysconfig.dataset_cache / "visdrone2019" - cachedir.mkdir(parents=True, exist_ok=True) - train_dir, test_dir = download_visdrone(cachedir) +CATEGORIES = [ + "ignored", + "pedestrian", + "people", + "bicycle", + "car", + "van", + "truck", + "tricycle", + "awning-tricycle", + "bus", + "motor", + "other", +] + + +def features() -> datasets.Features: + """Create VisDrone2019 dataset features""" + return datasets.Features( + { + "image_id": datasets.Value("int64"), + "file_name": datasets.Value("string"), + "image": datasets.Image(), + "objects": datasets.Sequence( + { + "id": datasets.Value("int64"), + "bbox": datasets.Sequence(datasets.Value("float32"), length=4), + "category": datasets.ClassLabel( + num_classes=len(CATEGORIES), names=CATEGORIES + ), + "truncation": datasets.Value("int32"), + "occlusion": datasets.Value("int32"), + } + ), + } + ) + + +ANNOTATION_FIELDS = [ + "x", + "y", + "width", + "height", + "score", + "category_id", + "truncation", + "occlusion", +] + + +def load_annotations(file: io.BufferedReader) -> List[Dict[str, Any]]: + """Load annotations/objects from the given file""" + reader = csv.DictReader( + io.StringIO(file.read().decode("utf-8")), fieldnames=ANNOTATION_FIELDS + ) + annotations = [] + for idx, row in enumerate(reader): + annotations.append( + { + "id": idx, + "bbox": list(map(float, [row[k] for k in ANNOTATION_FIELDS[:4]])), + "category": int(row["category_id"]), + "truncation": row["truncation"], + "occlusion": row["occlusion"], + } + ) + return annotations + + +def generate_samples( + files: Iterator[Tuple[str, io.BufferedReader]], annotation_file_ext: str = ".txt" +) -> Iterator[Dict[str, Any]]: + """Generate dataset samples from the given files in a VisDrone2019 archive""" + image_to_annotations = {} + # This loop relies on the ordering of the files in the archive: + # Annotation files come first, then the images. + for idx, (path, file) in enumerate(files): + file_name = Path(path).stem + if Path(path).suffix == annotation_file_ext: + image_to_annotations[file_name] = load_annotations(file) + elif file_name in image_to_annotations: + yield { + "image_id": idx, + "file_name": file_name, + "image": {"path": path, "bytes": file.read()}, + "objects": image_to_annotations[file_name], + } + else: + raise ValueError(f"Image {file_name} has no annotations") + + +if __name__ == "__main__": + pprint(load_dataset()) From 6b52a4f5754ed15627a4864d984d900f8f982a56 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 29 Apr 2024 15:32:00 -0400 Subject: [PATCH 03/28] Create benign evaluation for yolov5 with VisDrone dataset --- .../object_detection/datasets/visdrone.py | 49 ++++++- .../object_detection/visdrone_yolov5.py | 129 ++++++++++++++++++ 2 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 examples/src/armory/examples/object_detection/visdrone_yolov5.py diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py index a29569a1..4d614e97 100644 --- a/examples/src/armory/examples/object_detection/datasets/visdrone.py +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -6,35 +6,76 @@ from pprint import pprint from typing import Any, Dict, Iterator, List, Tuple +import albumentations as A +import albumentations.pytorch import datasets +import numpy as np import armory.data import armory.dataset def create_dataloader( - dataset: datasets.Dataset, **kwargs + dataset: datasets.Dataset, max_size: int, **kwargs ) -> armory.dataset.ObjectDetectionDataLoader: """ Create an Armory object detection dataloader for the given VisDrone2019 dataset split. Args: dataset: VisDrone2019 dataset split + max_size: Maximum image size to which to resize and pad image samples **kwargs: Additional keyword arguments to pass to the dataloader constructor Return: Armory object detection dataloader """ + resize = A.Compose( + [ + A.LongestMaxSize(max_size=max_size), + A.PadIfNeeded( + min_height=max_size, + min_width=max_size, + border_mode=0, + value=(0, 0, 0), + ), + A.ToFloat(max_value=255), + albumentations.pytorch.ToTensorV2(), + ], + bbox_params=A.BboxParams( + format="coco", + label_fields=["id", "category", "occlusion", "truncation"], + ), + ) + + def transform(sample): + tmp = dict(**sample) + tmp["image"] = [] + tmp["objects"] = [] + for image, objects in zip(sample["image"], sample["objects"]): + res = resize( + image=np.asarray(image), + bboxes=objects["bbox"], + id=objects["id"], + category=objects["category"], + occlusion=objects["occlusion"], + truncation=objects["truncation"], + ) + tmp["image"].append(res.pop("image")) + tmp["objects"].append(res) + return tmp + + dataset.set_transform(transform) + return armory.dataset.ObjectDetectionDataLoader( dataset, image_key="image", dim=armory.data.ImageDimensions.CHW, scale=armory.data.Scale( - dtype=armory.data.DataType.UINT8, - max=255, + dtype=armory.data.DataType.FLOAT, + max=1.0, ), objects_key="objects", - boxes_key="bbox", + boxes_key="bboxes", format=armory.data.BBoxFormat.XYWH, labels_key="category", **kwargs, diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5.py b/examples/src/armory/examples/object_detection/visdrone_yolov5.py new file mode 100644 index 00000000..8da0ba95 --- /dev/null +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5.py @@ -0,0 +1,129 @@ +""" +Example Armory evaluation of VisDrone object detection with YOLOv5 against +a custom Robust DPatch attack +""" + +from pprint import pprint +from typing import Optional + +import torchmetrics.detection +import yolov5 + +import armory.data +import armory.engine +import armory.evaluation +import armory.examples.object_detection.datasets.visdrone +import armory.export.criteria +import armory.export.object_detection +import armory.metric +import armory.metrics.compute +import armory.metrics.detection +import armory.metrics.tide +import armory.model.object_detection +import armory.perturbation +import armory.track + + +def parse_cli_args(): + """Parse command-line arguments""" + from armory.examples.utils.args import create_parser + + parser = create_parser( + description="Perform VisDrone object detection", + batch_size=4, + export_every_n_batches=5, + num_batches=20, + ) + return parser.parse_args() + + +def load_dataset( + evaluation: armory.evaluation.Evaluation, + batch_size: int, + shuffle: bool, + seed: Optional[int] = None, +): + """Load VisDrone dataset""" + with evaluation.autotrack(): + hf_dataset = armory.examples.object_detection.datasets.visdrone.load_dataset() + dataloader = ( + armory.examples.object_detection.datasets.visdrone.create_dataloader( + hf_dataset["val"], + max_size=640, + batch_size=batch_size, + shuffle=shuffle, + seed=seed, + ) + ) + dataset = armory.evaluation.Dataset( + name="VisDrone2019", + dataloader=dataloader, + ) + return dataset + + +def load_model(evaluation: armory.evaluation.Evaluation): + """Load YOLOv5 model from HuggingFace""" + with evaluation.autotrack() as track_call: + hf_model = track_call(yolov5.load, model_path="smidm/yolov5-visdrone") + + armory_model = armory.model.object_detection.YoloV5ObjectDetector( + name="YOLOv5", + model=hf_model, + ) + + return armory_model + + +def create_metrics(): + return { + "map": armory.metric.PredictionMetric( + torchmetrics.detection.MeanAveragePrecision(class_metrics=False), + armory.data.TorchBoundingBoxSpec(format=armory.data.BBoxFormat.XYXY), + ), + "tide": armory.metrics.tide.TIDE.create(), + "detection": armory.metrics.detection.ObjectDetectionRates.create(), + } + + +def create_exporters(model, export_every_n_batches): + """Create sample exporters""" + return [ + armory.export.object_detection.ObjectDetectionExporter( + criterion=armory.export.criteria.every_n_batches(export_every_n_batches) + ), + ] + + +@armory.track.track_params +def main(batch_size, export_every_n_batches, num_batches, seed, shuffle): + """Perform the evaluation""" + evaluation = armory.evaluation.Evaluation( + name="visdrone-object-detection-yolov5", + description="VisDrone object detection using YOLOv5", + author="TwoSix", + ) + + dataset = load_dataset(evaluation, batch_size, shuffle, seed) + model = load_model(evaluation) + + evaluation.use_dataset(dataset) + evaluation.use_model(model) + evaluation.use_metrics(create_metrics()) + evaluation.use_exporters(create_exporters(model, export_every_n_batches)) + + with evaluation.add_chain("benign"): + pass + + eval_engine = armory.engine.EvaluationEngine( + evaluation, + profiler=armory.metrics.compute.BasicProfiler(), + limit_test_batches=num_batches, + ) + eval_results = eval_engine.run() + + pprint(eval_results) + + +if __name__ == "__main__": + main(**vars(parse_cli_args())) From 5f21e8023f6cb3d85915fc213fe48dc771ed21a5 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 30 Apr 2024 09:22:46 -0400 Subject: [PATCH 04/28] Record OD rates as metrics --- .../armory/examples/object_detection/visdrone_yolov5.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5.py b/examples/src/armory/examples/object_detection/visdrone_yolov5.py index 8da0ba95..707518c6 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5.py @@ -82,7 +82,14 @@ def create_metrics(): armory.data.TorchBoundingBoxSpec(format=armory.data.BBoxFormat.XYXY), ), "tide": armory.metrics.tide.TIDE.create(), - "detection": armory.metrics.detection.ObjectDetectionRates.create(), + "detection": armory.metrics.detection.ObjectDetectionRates.create( + record_as_metrics=[ + "true_positive_rate_mean", + "misclassification_rate_mean", + "disappearance_rate_mean", + "hallucinations_mean", + ], + ), } From 09e6a8e44911001f09f1c5d7f18f2a9bd7a46cbf Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 30 Apr 2024 11:19:03 -0400 Subject: [PATCH 05/28] Revise YOLOv5 wrapper to work with yolov5 backends with slightly different detection model types --- .../yolov5_object_detector.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/library/src/armory/model/object_detection/yolov5_object_detector.py b/library/src/armory/model/object_detection/yolov5_object_detector.py index 0adf926d..e09008bf 100644 --- a/library/src/armory/model/object_detection/yolov5_object_detector.py +++ b/library/src/armory/model/object_detection/yolov5_object_detector.py @@ -1,8 +1,7 @@ from functools import partial -from typing import Optional +from typing import TYPE_CHECKING, Optional import torch -from yolov5.models.yolo import DetectionModel from armory.data import ( BBoxFormat, @@ -18,6 +17,9 @@ from armory.model.object_detection.object_detector import ObjectDetector from armory.track import track_init_params +if TYPE_CHECKING: + from yolov5.models.yolo import DetectionModel + @track_init_params class YoloV5ObjectDetector(ObjectDetector): @@ -40,7 +42,8 @@ class YoloV5ObjectDetector(ObjectDetector): def __init__( self, name: str, - model: DetectionModel, + model, + detection_model: Optional["DetectionModel"] = None, inputs_spec: Optional[ImageSpec] = None, predictions_spec: Optional[BoundingBoxSpec] = None, iou_threshold: Optional[float] = None, @@ -53,6 +56,12 @@ def __init__( Args: name: Name of the model. model: YOLOv5 model being wrapped. + detection_model: Optional, the inner YOLOv5 detection model to use + for computing loss. By default, the detection model is assumed + to be a property of the inner model property of the given YOLOv5 + model--that is, `model.model.model`. It is unlikely that this + argument will ever be necessary, and may only be required if + the upstream `yolov5` package changes its model structure. inputs_spec: Optional, data specification used to obtain raw image data from the image inputs contained in object detection batches. Defaults to a specification compatible with typical @@ -85,7 +94,9 @@ def __init__( from yolov5.utils.general import non_max_suppression from yolov5.utils.loss import ComputeLoss - self._detection_model = self._get_detection_model(self._model) + self._detection_model: "DetectionModel" = ( + detection_model if detection_model is not None else self._model.model.model + ) self.compute_loss = ComputeLoss(self._detection_model) if score_threshold is not None: @@ -94,16 +105,6 @@ def __init__( kwargs["iou_thres"] = iou_threshold self.nms = partial(non_max_suppression, **kwargs) - @staticmethod - def _get_detection_model(model: DetectionModel) -> DetectionModel: - detect_model = model - try: - while type(detect_model) is not DetectionModel: - detect_model = detect_model.model - except AttributeError: - raise TypeError(f"{model} is not a {DetectionModel.__name__}") - return detect_model - def forward(self, x, targets=None): """ Invokes the wrapped model. If in training and given targets, then the From 9d294361c132270a3a11c019a92c2004a36cb99f Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Wed, 1 May 2024 14:35:27 -0400 Subject: [PATCH 06/28] Remove argmax on YOLOv5 output as the NMS processing already does that --- .../src/armory/model/object_detection/yolov5_object_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/src/armory/model/object_detection/yolov5_object_detector.py b/library/src/armory/model/object_detection/yolov5_object_detector.py index e09008bf..f90c3987 100644 --- a/library/src/armory/model/object_detection/yolov5_object_detector.py +++ b/library/src/armory/model/object_detection/yolov5_object_detector.py @@ -139,7 +139,7 @@ def predict(self, batch: ObjectDetectionBatch): outputs = [ { "boxes": output[:, 0:4], - "labels": torch.argmax(output[:, 5:], dim=1, keepdim=False), + "labels": output[:, 5].to(torch.int64), "scores": output[:, 4], } for output in outputs From 835b90614ee4254390ca55482f8a39d83b3ad2cc Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Wed, 1 May 2024 14:59:04 -0400 Subject: [PATCH 07/28] Download VisDrone from GitHub and adjust labels to match what YOLOv5 was trained on --- .../object_detection/datasets/visdrone.py | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py index 4d614e97..165df538 100644 --- a/examples/src/armory/examples/object_detection/datasets/visdrone.py +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -82,8 +82,8 @@ def transform(sample): ) -GDRIVE_VAL_URL = "https://drive.usercontent.google.com/download?id=1bxK5zgLn0_L8x276eKkuYA_FzwCIjb59&confirm=1" -GDRIVE_TRAIN_URL = "https://drive.usercontent.google.com/download?id=1a2oHjcEcwXP8oUF95qiwrqzACb2YlUhn&confirm=1" +TRAIN_URL = "https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-train.zip" +VAL_URL = "https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-val.zip" def load_dataset() -> datasets.DatasetDict: @@ -95,7 +95,7 @@ def load_dataset() -> datasets.DatasetDict: """ dl_manager = datasets.DownloadManager(dataset_name="VisDrone2019") ds_features = features() - paths = dl_manager.download({"train": GDRIVE_TRAIN_URL, "val": GDRIVE_VAL_URL}) + paths = dl_manager.download({"train": TRAIN_URL, "val": VAL_URL}) train_files = dl_manager.iter_archive(paths["train"]) val_files = dl_manager.iter_archive(paths["val"]) return datasets.DatasetDict( @@ -115,7 +115,9 @@ def load_dataset() -> datasets.DatasetDict: CATEGORIES = [ - "ignored", + # The YOLOv5 model removed this class and shifted all others down by 1 when + # it trained on the VisDrone data + # "ignored", "pedestrian", "people", "bicycle", @@ -171,15 +173,18 @@ def load_annotations(file: io.BufferedReader) -> List[Dict[str, Any]]: ) annotations = [] for idx, row in enumerate(reader): - annotations.append( - { - "id": idx, - "bbox": list(map(float, [row[k] for k in ANNOTATION_FIELDS[:4]])), - "category": int(row["category_id"]), - "truncation": row["truncation"], - "occlusion": row["occlusion"], - } - ) + category = int(row["category_id"]) + if category != 0: # Drop class-0 annotations + category -= 1 # The model was trained with 0-indexed categories starting at pedestrian + annotations.append( + { + "id": idx, + "bbox": list(map(float, [row[k] for k in ANNOTATION_FIELDS[:4]])), + "category": category, + "truncation": row["truncation"], + "occlusion": row["occlusion"], + } + ) return annotations From 78e118283fb373291e5ce08971831e1b20cd7db4 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Fri, 3 May 2024 12:32:00 -0400 Subject: [PATCH 08/28] Ignore all classes with scores of 0 in visdrone dataset --- .../armory/examples/object_detection/datasets/visdrone.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py index 165df538..651a04b0 100644 --- a/examples/src/armory/examples/object_detection/datasets/visdrone.py +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -128,7 +128,8 @@ def load_dataset() -> datasets.DatasetDict: "awning-tricycle", "bus", "motor", - "other", + # The YOLOv5 model also ignored/removed this class + # "other", ] @@ -173,8 +174,9 @@ def load_annotations(file: io.BufferedReader) -> List[Dict[str, Any]]: ) annotations = [] for idx, row in enumerate(reader): + score = int(row["score"]) category = int(row["category_id"]) - if category != 0: # Drop class-0 annotations + if score != 0: # Drop annotations with score of 0 (class-0 & class-11) category -= 1 # The model was trained with 0-indexed categories starting at pedestrian annotations.append( { From 9b6f67e1b29cb3cf198da4636cd7c57dd43124ce Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Fri, 3 May 2024 17:08:45 -0400 Subject: [PATCH 09/28] Create initial lightning module to generate robust DPatch against visdrone --- .../visdrone_yolov5_robustdpatch.py | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py new file mode 100644 index 00000000..3d7224bf --- /dev/null +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -0,0 +1,153 @@ +import logging +from typing import Optional, Sequence + +from lightning.pytorch import LightningModule, Trainer +import torch +import yolov5 + +import armory.data +import armory.examples.object_detection.datasets.visdrone +import armory.model.object_detection + +logger = logging.getLogger(__name__) + + +class RobustDPatchModule(LightningModule): + + def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): + super().__init__() + self.model = model + self.target_spec = armory.data.TorchBoundingBoxSpec( + format=armory.data.BBoxFormat.CXCYWH + ) + self.patch_shape = (3, 50, 50) + self.patch_location = (295, 295) + self.patch = torch.zeros(self.patch_shape) + self.targeted = False + self.learning_rate = 0.01 + + # def forward(self, inputs, target): + # # return self.model(inputs) + # return super().forward(inputs, target) + + def configure_optimizers(self): + # return torch.optim.Adam(self.parameters(), lr=1e-3) + return super().configure_optimizers() + + def on_train_epoch_start(self): + self.patch_gradients = torch.zeros_like(self.patch, device=self.model.device) + self.patch = self.patch.to(self.model.device) + + def on_train_epoch_end(self): + self.patch = ( + self.patch + + torch.sign(self.patch_gradients) + * (1 - 2 * int(self.targeted)) + * self.learning_rate + ) + # TODO handle normalized min/max + self.patch = torch.clip(self.patch, 0, self.model.inputs_spec.scale.max) + + def training_step(self, batch: armory.data.Batch, batch_idx: int): + # TODO transformations + + # Get inputs as Tensor with gradients required + inputs = batch.inputs.get(self.model.inputs_spec) + assert isinstance(inputs, torch.Tensor) + if inputs.is_leaf: + inputs.requires_grad = True + else: + inputs.retain_grad() + + # TODO apply patch to image + + # Get targets as Tensor + _, _, height, width = inputs.shape + targets = batch.targets.get(self.target_spec) + yolo_targets = self._to_yolo_targets(targets, height, width, self.model.device) + + # Get loss from model outputs + self.model.train() + loss_components = self.model(inputs, yolo_targets) + loss = loss_components["loss_total"] + + # Clean gradients + self.model.zero_grad() + + # Compute gradients + loss.backward(retain_graph=True) + assert inputs.grad is not None + grads = inputs.grad.clone() + assert grads.shape == inputs.shape + + # Extract patch gradients + x_1, y_1 = self.patch_location + x_2 = x_1 + self.patch_shape[1] + y_2 = y_1 + self.patch_shape[2] + grads = grads[:, :, x_1:x_2, y_1:y_2] + + patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) + logger.debug( + "Gradient percentage diff: %f)", + torch.mean( + torch.sign(patch_gradients) != torch.sign(self.patch_gradients), + dtype=torch.float64, + ), + ) + self.patch_gradients = patch_gradients + + @staticmethod + def _to_yolo_targets( + targets: Sequence[armory.data.BoundingBoxes.BoxesTorch], + height: int, + width: int, + device, + ) -> torch.Tensor: + targets_list = [] + + for i, target in enumerate(targets): + labels = torch.zeros(len(target["boxes"]), 6, device=device) + labels[:, 0] = i + labels[:, 1] = target["labels"] + labels[:, 2:6] = target["boxes"] + + # normalize bounding boxes to [0, 1]} + labels[:, 2:6:2] = labels[:, 2:6:2] / width + labels[:, 3:6:2] = labels[:, 3:6:2] / height + + targets_list.append(labels) + + r = torch.vstack(targets_list) + return r + + +def load_model(): + hf_model = yolov5.load(model_path="smidm/yolov5-visdrone") + + armory_model = armory.model.object_detection.YoloV5ObjectDetector( + name="YOLOv5", + model=hf_model, + ) + + return armory_model + + +def load_dataset(batch_size: int, shuffle: bool, seed: Optional[int] = None): + hf_dataset = armory.examples.object_detection.datasets.visdrone.load_dataset() + dataloader = armory.examples.object_detection.datasets.visdrone.create_dataloader( + hf_dataset["val"], + max_size=640, + batch_size=batch_size, + shuffle=shuffle, + seed=seed, + ) + return dataloader + + +if __name__ == "__main__": + dataloader = load_dataset(batch_size=2, shuffle=False) + model = load_model() + + module = RobustDPatchModule(model) + trainer = Trainer(limit_train_batches=10, max_epochs=2) + trainer.fit(module, dataloader) From 1ba82cad21ae4596756f39a539a9b2a14e1721c7 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 7 May 2024 08:51:55 -0400 Subject: [PATCH 10/28] Apply patch to image in training step --- .../visdrone_yolov5_robustdpatch.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 3d7224bf..6d77bd6e 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -51,15 +51,20 @@ def on_train_epoch_end(self): def training_step(self, batch: armory.data.Batch, batch_idx: int): # TODO transformations - # Get inputs as Tensor with gradients required + # Get inputs as Tensor inputs = batch.inputs.get(self.model.inputs_spec) assert isinstance(inputs, torch.Tensor) - if inputs.is_leaf: - inputs.requires_grad = True - else: - inputs.retain_grad() - # TODO apply patch to image + # Get patch as Tensor with gradients required + patch = self.patch.clone() + patch.requires_grad = True + + # Apply patch to image + x_1, y_1 = self.patch_location + x_2 = x_1 + self.patch_shape[1] + y_2 = y_1 + self.patch_shape[2] + inputs_with_patch = inputs.clone() + inputs_with_patch[:, :, x_1:x_2, y_1:y_2] = patch # Get targets as Tensor _, _, height, width = inputs.shape @@ -68,7 +73,7 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): # Get loss from model outputs self.model.train() - loss_components = self.model(inputs, yolo_targets) + loss_components = self.model(inputs_with_patch, yolo_targets) loss = loss_components["loss_total"] # Clean gradients @@ -76,16 +81,11 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): # Compute gradients loss.backward(retain_graph=True) - assert inputs.grad is not None - grads = inputs.grad.clone() - assert grads.shape == inputs.shape - - # Extract patch gradients - x_1, y_1 = self.patch_location - x_2 = x_1 + self.patch_shape[1] - y_2 = y_1 + self.patch_shape[2] - grads = grads[:, :, x_1:x_2, y_1:y_2] + assert patch.grad is not None + grads = patch.grad.clone() + assert grads.shape == self.patch.shape + # Accumulate gradients patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) logger.debug( "Gradient percentage diff: %f)", From bc68b4fc2f4cc43d1f6239febe3f9311470fb0a0 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 7 May 2024 11:21:20 -0400 Subject: [PATCH 11/28] Save final patch to image file --- .../visdrone_yolov5_robustdpatch.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 6d77bd6e..894ee3eb 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -1,6 +1,7 @@ import logging from typing import Optional, Sequence +import PIL.Image from lightning.pytorch import LightningModule, Trainer import torch import yolov5 @@ -21,8 +22,13 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): format=armory.data.BBoxFormat.CXCYWH ) self.patch_shape = (3, 50, 50) - self.patch_location = (295, 295) - self.patch = torch.zeros(self.patch_shape) + self.patch_location = (295, 295) # middle of 640x640 + # TODO non-zero min value + self.patch = ( + torch.randint(0, 255, self.patch_shape) + / 255 + * self.model.inputs_spec.scale.max + ) self.targeted = False self.learning_rate = 0.01 @@ -149,5 +155,9 @@ def load_dataset(batch_size: int, shuffle: bool, seed: Optional[int] = None): model = load_model() module = RobustDPatchModule(model) - trainer = Trainer(limit_train_batches=10, max_epochs=2) + trainer = Trainer(limit_train_batches=10, max_epochs=20) trainer.fit(module, dataloader) + + patch_np = (module.patch.cpu().numpy().transpose(1, 2, 0) * 255).astype("uint8") + patch = PIL.Image.fromarray(patch_np) + patch.save("patch.png") From 3aef3d0136ba950121dab085555e7e33c80e2446 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 7 May 2024 11:44:44 -0400 Subject: [PATCH 12/28] Generate patch then use it in Armory evaluation --- .../object_detection/visdrone_yolov5.py | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5.py b/examples/src/armory/examples/object_detection/visdrone_yolov5.py index 707518c6..39ae6f6f 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5.py @@ -4,8 +4,9 @@ """ from pprint import pprint -from typing import Optional +from typing import Literal, Optional, Union +import torch import torchmetrics.detection import yolov5 @@ -42,13 +43,14 @@ def load_dataset( batch_size: int, shuffle: bool, seed: Optional[int] = None, + split: Union[Literal["val"], Literal["train"]] = "val", ): """Load VisDrone dataset""" with evaluation.autotrack(): hf_dataset = armory.examples.object_detection.datasets.visdrone.load_dataset() dataloader = ( armory.examples.object_detection.datasets.visdrone.create_dataloader( - hf_dataset["val"], + hf_dataset[split], max_size=640, batch_size=batch_size, shuffle=shuffle, @@ -102,6 +104,20 @@ def create_exporters(model, export_every_n_batches): ] +def generate_patch(dataloader, model) -> torch.Tensor: + from lightning.pytorch import Trainer + + from armory.examples.object_detection.visdrone_yolov5_robustdpatch import ( + RobustDPatchModule, + ) + + module = RobustDPatchModule(model) + trainer = Trainer(limit_train_batches=10, max_epochs=20) + trainer.fit(module, dataloader) + + return module.patch + + @armory.track.track_params def main(batch_size, export_every_n_batches, num_batches, seed, shuffle): """Perform the evaluation""" @@ -114,6 +130,10 @@ def main(batch_size, export_every_n_batches, num_batches, seed, shuffle): dataset = load_dataset(evaluation, batch_size, shuffle, seed) model = load_model(evaluation) + # Generate patch + train_dataset = load_dataset(evaluation, batch_size, shuffle, seed, split="train") + patch = generate_patch(train_dataset.dataloader, model) + evaluation.use_dataset(dataset) evaluation.use_model(model) evaluation.use_metrics(create_metrics()) @@ -122,6 +142,23 @@ def main(batch_size, export_every_n_batches, num_batches, seed, shuffle): with evaluation.add_chain("benign"): pass + with evaluation.add_chain("patch") as chain: + x_1, y_1 = 295, 295 # middle of 640x640 + x_2 = x_1 + patch.shape[1] + y_2 = y_1 + patch.shape[2] + + def apply_patch(inputs: torch.Tensor) -> torch.Tensor: + with_patch = inputs.clone() + with_patch[:, :, x_1:x_2, y_1:y_2] = patch + return with_patch + + attack = armory.perturbation.CallablePerturbation( + name="RobustDPatch", + perturbation=apply_patch, + inputs_spec=model.inputs_spec, + ) + chain.add_perturbation(attack) + eval_engine = armory.engine.EvaluationEngine( evaluation, profiler=armory.metrics.compute.BasicProfiler(), From 538b6345b078a981faea7a039014dcc8cc4042c9 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Wed, 8 May 2024 12:18:01 -0400 Subject: [PATCH 13/28] Apply random augmentations to image during patch generation --- .../visdrone_yolov5_robustdpatch.py | 104 ++++++++++-------- 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 894ee3eb..e338f94f 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -2,6 +2,7 @@ from typing import Optional, Sequence import PIL.Image +import kornia from lightning.pytorch import LightningModule, Trainer import torch import yolov5 @@ -31,6 +32,13 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): ) self.targeted = False self.learning_rate = 0.01 + self.sample_size = 10 + self.augmentation = kornia.augmentation.container.ImageSequential( + kornia.augmentation.RandomHorizontalFlip(p=0.5), + kornia.augmentation.RandomBrightness(brightness=(0.5, 2.0), p=0.5), + kornia.augmentation.RandomRotation(degrees=15, p=0.5), + random_apply=True, + ) # def forward(self, inputs, target): # # return self.model(inputs) @@ -55,52 +63,56 @@ def on_train_epoch_end(self): self.patch = torch.clip(self.patch, 0, self.model.inputs_spec.scale.max) def training_step(self, batch: armory.data.Batch, batch_idx: int): - # TODO transformations - - # Get inputs as Tensor - inputs = batch.inputs.get(self.model.inputs_spec) - assert isinstance(inputs, torch.Tensor) - - # Get patch as Tensor with gradients required - patch = self.patch.clone() - patch.requires_grad = True - - # Apply patch to image - x_1, y_1 = self.patch_location - x_2 = x_1 + self.patch_shape[1] - y_2 = y_1 + self.patch_shape[2] - inputs_with_patch = inputs.clone() - inputs_with_patch[:, :, x_1:x_2, y_1:y_2] = patch - - # Get targets as Tensor - _, _, height, width = inputs.shape - targets = batch.targets.get(self.target_spec) - yolo_targets = self._to_yolo_targets(targets, height, width, self.model.device) - - # Get loss from model outputs - self.model.train() - loss_components = self.model(inputs_with_patch, yolo_targets) - loss = loss_components["loss_total"] - - # Clean gradients - self.model.zero_grad() - - # Compute gradients - loss.backward(retain_graph=True) - assert patch.grad is not None - grads = patch.grad.clone() - assert grads.shape == self.patch.shape - - # Accumulate gradients - patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) - logger.debug( - "Gradient percentage diff: %f)", - torch.mean( - torch.sign(patch_gradients) != torch.sign(self.patch_gradients), - dtype=torch.float64, - ), - ) - self.patch_gradients = patch_gradients + for _ in range(self.sample_size): + # Get inputs as Tensor + inputs = batch.inputs.get(self.model.inputs_spec) + assert isinstance(inputs, torch.Tensor) + + # Get patch as Tensor with gradients required + patch = self.patch.clone() + patch.requires_grad = True + + # Apply patch to image + x_1, y_1 = self.patch_location + x_2 = x_1 + self.patch_shape[1] + y_2 = y_1 + self.patch_shape[2] + inputs_with_patch = inputs.clone() + inputs_with_patch[:, :, x_1:x_2, y_1:y_2] = patch + + # Apply random augmentations to images + inputs_with_augmentations = self.augmentation(inputs_with_patch) + + # Get targets as Tensor + _, _, height, width = inputs.shape + targets = batch.targets.get(self.target_spec) + yolo_targets = self._to_yolo_targets( + targets, height, width, self.model.device + ) + + # Get loss from model outputs + self.model.train() + loss_components = self.model(inputs_with_augmentations, yolo_targets) + loss = loss_components["loss_total"] + + # Clean gradients + self.model.zero_grad() + + # Compute gradients + loss.backward(retain_graph=True) + assert patch.grad is not None + grads = patch.grad.clone() + assert grads.shape == self.patch.shape + + # Accumulate gradients + patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) + logger.debug( + "Gradient percentage diff: %f)", + torch.mean( + torch.sign(patch_gradients) != torch.sign(self.patch_gradients), + dtype=torch.float64, + ), + ) + self.patch_gradients = patch_gradients @staticmethod def _to_yolo_targets( From eb0eff2040a53a8d9c6c918080e3566e043bd654 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Wed, 8 May 2024 12:18:48 -0400 Subject: [PATCH 14/28] Add args for patch generation parameters --- .../object_detection/visdrone_yolov5.py | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5.py b/examples/src/armory/examples/object_detection/visdrone_yolov5.py index 39ae6f6f..7a86bf98 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5.py @@ -35,6 +35,24 @@ def parse_cli_args(): export_every_n_batches=5, num_batches=20, ) + parser.add_argument( + "--patch-batch-size", + default=2, + help="Batch size used to generate the patch", + type=int, + ) + parser.add_argument( + "--patch-num-batches", + default=10, + help="Number of batches used to generate the patch", + type=int, + ) + parser.add_argument( + "--patch-num-epochs", + default=20, + help="Number of epochs used to generate the patch", + type=int, + ) return parser.parse_args() @@ -104,7 +122,7 @@ def create_exporters(model, export_every_n_batches): ] -def generate_patch(dataloader, model) -> torch.Tensor: +def generate_patch(dataloader, model, num_batches=10, num_epochs=20) -> torch.Tensor: from lightning.pytorch import Trainer from armory.examples.object_detection.visdrone_yolov5_robustdpatch import ( @@ -112,14 +130,23 @@ def generate_patch(dataloader, model) -> torch.Tensor: ) module = RobustDPatchModule(model) - trainer = Trainer(limit_train_batches=10, max_epochs=20) + trainer = Trainer(limit_train_batches=num_batches, max_epochs=num_epochs) trainer.fit(module, dataloader) return module.patch @armory.track.track_params -def main(batch_size, export_every_n_batches, num_batches, seed, shuffle): +def main( + batch_size, + export_every_n_batches, + num_batches, + seed, + shuffle, + patch_batch_size, + patch_num_batches, + patch_num_epochs, +): """Perform the evaluation""" evaluation = armory.evaluation.Evaluation( name="visdrone-object-detection-yolov5", @@ -131,8 +158,12 @@ def main(batch_size, export_every_n_batches, num_batches, seed, shuffle): model = load_model(evaluation) # Generate patch - train_dataset = load_dataset(evaluation, batch_size, shuffle, seed, split="train") - patch = generate_patch(train_dataset.dataloader, model) + train_dataset = load_dataset( + evaluation, patch_batch_size, shuffle, seed, split="train" + ) + patch = generate_patch( + train_dataset.dataloader, model, patch_num_batches, patch_num_epochs + ) evaluation.use_dataset(dataset) evaluation.use_model(model) From d124aa260303e5881bcfa790cb5e44485f77424b Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Wed, 8 May 2024 13:58:56 -0400 Subject: [PATCH 15/28] Log loss to MLFlow during patch generation --- .../examples/object_detection/visdrone_yolov5.py | 13 ++++++++++++- .../visdrone_yolov5_robustdpatch.py | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5.py b/examples/src/armory/examples/object_detection/visdrone_yolov5.py index 7a86bf98..06ddc000 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5.py @@ -124,13 +124,24 @@ def create_exporters(model, export_every_n_batches): def generate_patch(dataloader, model, num_batches=10, num_epochs=20) -> torch.Tensor: from lightning.pytorch import Trainer + from lightning.pytorch.loggers import MLFlowLogger + from armory.evaluation import SysConfig from armory.examples.object_detection.visdrone_yolov5_robustdpatch import ( RobustDPatchModule, ) + from armory.track import init_tracking_uri + + sysconfig = SysConfig() + logger = MLFlowLogger( + experiment_name="visdrone-yolov5-robustdpatch-generation", + tracking_uri=init_tracking_uri(sysconfig.armory_home), + ) module = RobustDPatchModule(model) - trainer = Trainer(limit_train_batches=num_batches, max_epochs=num_epochs) + trainer = Trainer( + limit_train_batches=num_batches, max_epochs=num_epochs, logger=logger + ) trainer.fit(module, dataloader) return module.patch diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index e338f94f..fc4169cc 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -99,6 +99,7 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): # Compute gradients loss.backward(retain_graph=True) + self.log("loss", loss) assert patch.grad is not None grads = patch.grad.clone() assert grads.shape == self.patch.shape From 5121d35b0b261da276a6cd6f16acf42504100284 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Wed, 8 May 2024 14:33:30 -0400 Subject: [PATCH 16/28] Randomize patch location --- .../src/armory/examples/object_detection/visdrone_yolov5.py | 3 ++- .../object_detection/visdrone_yolov5_robustdpatch.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5.py b/examples/src/armory/examples/object_detection/visdrone_yolov5.py index 06ddc000..0906dbc9 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5.py @@ -185,7 +185,8 @@ def main( pass with evaluation.add_chain("patch") as chain: - x_1, y_1 = 295, 295 # middle of 640x640 + # x_1, y_1 = 295, 295 # middle of 640x640 + x_1, y_1 = 50, 50 x_2 = x_1 + patch.shape[1] y_2 = y_1 + patch.shape[2] diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index fc4169cc..72f82ef7 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -1,4 +1,5 @@ import logging +import random from typing import Optional, Sequence import PIL.Image @@ -73,7 +74,9 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): patch.requires_grad = True # Apply patch to image - x_1, y_1 = self.patch_location + x_1 = random.randint(0, inputs.shape[3] - self.patch_shape[2]) + y_1 = random.randint(0, inputs.shape[2] - self.patch_shape[1]) + # x_1, y_1 = self.patch_location x_2 = x_1 + self.patch_shape[1] y_2 = y_1 + self.patch_shape[2] inputs_with_patch = inputs.clone() From ba2ea7c234e39740dc430e681fd1e1a5fe5d36d3 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Thu, 9 May 2024 08:44:47 -0400 Subject: [PATCH 17/28] Omit boxes with 0 height or width --- .../armory/examples/object_detection/datasets/visdrone.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py index 651a04b0..f322c05d 100644 --- a/examples/src/armory/examples/object_detection/datasets/visdrone.py +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -178,10 +178,13 @@ def load_annotations(file: io.BufferedReader) -> List[Dict[str, Any]]: category = int(row["category_id"]) if score != 0: # Drop annotations with score of 0 (class-0 & class-11) category -= 1 # The model was trained with 0-indexed categories starting at pedestrian + bbox = list(map(float, [row[k] for k in ANNOTATION_FIELDS[:4]])) + if bbox[2] == 0 or bbox[3] == 0: + continue annotations.append( { "id": idx, - "bbox": list(map(float, [row[k] for k in ANNOTATION_FIELDS[:4]])), + "bbox": bbox, "category": category, "truncation": row["truncation"], "occlusion": row["occlusion"], From ff23c18ff302e2d6e17f444fb5d998ecf4d3a4f6 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Thu, 9 May 2024 08:45:34 -0400 Subject: [PATCH 18/28] Initial attempt using lightning automatic optimization --- .../visdrone_yolov5_robustdpatch.py | 83 ++++++++++--------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 72f82ef7..ef22d3bc 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -5,6 +5,7 @@ import PIL.Image import kornia from lightning.pytorch import LightningModule, Trainer +from lightning.pytorch.loggers import MLFlowLogger import torch import yolov5 @@ -31,6 +32,7 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): / 255 * self.model.inputs_spec.scale.max ) + self.initial_patch = self.patch.clone() self.targeted = False self.learning_rate = 0.01 self.sample_size = 10 @@ -41,27 +43,28 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): random_apply=True, ) - # def forward(self, inputs, target): - # # return self.model(inputs) - # return super().forward(inputs, target) - def configure_optimizers(self): - # return torch.optim.Adam(self.parameters(), lr=1e-3) - return super().configure_optimizers() - - def on_train_epoch_start(self): - self.patch_gradients = torch.zeros_like(self.patch, device=self.model.device) - self.patch = self.patch.to(self.model.device) + return torch.optim.SGD([self.patch], lr=self.learning_rate) def on_train_epoch_end(self): - self.patch = ( - self.patch - + torch.sign(self.patch_gradients) - * (1 - 2 * int(self.targeted)) - * self.learning_rate - ) + if isinstance(self.logger, MLFlowLogger): + if self.current_epoch % 5 == 0: + patch_np = ( + self.patch.detach().cpu().numpy().transpose(1, 2, 0) * 255 + ).astype("uint8") + patch = PIL.Image.fromarray(patch_np) + self.logger.experiment.log_image( + self.logger.run_id, patch, f"patch_epoch_{self.current_epoch}.png" + ) + + # self.patch = ( + # self.patch + # + torch.sign(self.patch_gradients) + # * (1 - 2 * int(self.targeted)) + # * self.learning_rate + # ) # TODO handle normalized min/max - self.patch = torch.clip(self.patch, 0, self.model.inputs_spec.scale.max) + # self.patch = torch.clip(self.patch, 0, self.model.inputs_spec.scale.max) def training_step(self, batch: armory.data.Batch, batch_idx: int): for _ in range(self.sample_size): @@ -69,9 +72,8 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): inputs = batch.inputs.get(self.model.inputs_spec) assert isinstance(inputs, torch.Tensor) - # Get patch as Tensor with gradients required - patch = self.patch.clone() - patch.requires_grad = True + # Require gradients on patch Tensor + self.patch.requires_grad = True # Apply patch to image x_1 = random.randint(0, inputs.shape[3] - self.patch_shape[2]) @@ -80,7 +82,7 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): x_2 = x_1 + self.patch_shape[1] y_2 = y_1 + self.patch_shape[2] inputs_with_patch = inputs.clone() - inputs_with_patch[:, :, x_1:x_2, y_1:y_2] = patch + inputs_with_patch[:, :, x_1:x_2, y_1:y_2] = self.patch # Apply random augmentations to images inputs_with_augmentations = self.augmentation(inputs_with_patch) @@ -97,26 +99,21 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): loss_components = self.model(inputs_with_augmentations, yolo_targets) loss = loss_components["loss_total"] - # Clean gradients - self.model.zero_grad() - - # Compute gradients - loss.backward(retain_graph=True) self.log("loss", loss) - assert patch.grad is not None - grads = patch.grad.clone() - assert grads.shape == self.patch.shape + loss = -loss + # Clean gradients + # self.model.zero_grad() + # Compute gradients + # loss.backward(retain_graph=True) + # assert patch.grad is not None + # grads = patch.grad.clone() + # assert grads.shape == self.patch.shape # Accumulate gradients - patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) - logger.debug( - "Gradient percentage diff: %f)", - torch.mean( - torch.sign(patch_gradients) != torch.sign(self.patch_gradients), - dtype=torch.float64, - ), - ) - self.patch_gradients = patch_gradients + # patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) + # self.patch_gradients = patch_gradients + + return loss @staticmethod def _to_yolo_targets( @@ -174,6 +171,14 @@ def load_dataset(batch_size: int, shuffle: bool, seed: Optional[int] = None): trainer = Trainer(limit_train_batches=10, max_epochs=20) trainer.fit(module, dataloader) - patch_np = (module.patch.cpu().numpy().transpose(1, 2, 0) * 255).astype("uint8") + patch_np = ( + module.initial_patch.detach().cpu().numpy().transpose(1, 2, 0) * 255 + ).astype("uint8") + patch = PIL.Image.fromarray(patch_np) + patch.save("initial_patch.png") + + patch_np = (module.patch.detach().cpu().numpy().transpose(1, 2, 0) * 255).astype( + "uint8" + ) patch = PIL.Image.fromarray(patch_np) patch.save("patch.png") From 47110edfae8dc6324bba3c6d47c643f481203215 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 20 May 2024 11:45:55 -0400 Subject: [PATCH 19/28] Use adjusted learning rate and momentum --- .../visdrone_yolov5_robustdpatch.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index ef22d3bc..060a2d17 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -1,5 +1,6 @@ import logging -import random + +# import random from typing import Optional, Sequence import PIL.Image @@ -32,19 +33,20 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): / 255 * self.model.inputs_spec.scale.max ) + self.patch.requires_grad = True self.initial_patch = self.patch.clone() self.targeted = False - self.learning_rate = 0.01 + self.learning_rate = 0.1 self.sample_size = 10 self.augmentation = kornia.augmentation.container.ImageSequential( kornia.augmentation.RandomHorizontalFlip(p=0.5), - kornia.augmentation.RandomBrightness(brightness=(0.5, 2.0), p=0.5), + kornia.augmentation.RandomBrightness(brightness=(0.75, 1.25), p=0.5), kornia.augmentation.RandomRotation(degrees=15, p=0.5), random_apply=True, ) def configure_optimizers(self): - return torch.optim.SGD([self.patch], lr=self.learning_rate) + return torch.optim.SGD([self.patch], lr=self.learning_rate, momentum=0.9) def on_train_epoch_end(self): if isinstance(self.logger, MLFlowLogger): @@ -72,12 +74,10 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): inputs = batch.inputs.get(self.model.inputs_spec) assert isinstance(inputs, torch.Tensor) - # Require gradients on patch Tensor - self.patch.requires_grad = True - # Apply patch to image - x_1 = random.randint(0, inputs.shape[3] - self.patch_shape[2]) - y_1 = random.randint(0, inputs.shape[2] - self.patch_shape[1]) + # x_1 = random.randint(0, inputs.shape[3] - self.patch_shape[2]) + # y_1 = random.randint(0, inputs.shape[2] - self.patch_shape[1]) + x_1, y_1 = self.patch_location # x_1, y_1 = self.patch_location x_2 = x_1 + self.patch_shape[1] y_2 = y_1 + self.patch_shape[2] @@ -95,12 +95,12 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): ) # Get loss from model outputs - self.model.train() + self.model.eval() loss_components = self.model(inputs_with_augmentations, yolo_targets) loss = loss_components["loss_total"] self.log("loss", loss) - loss = -loss + # loss = -loss # Clean gradients # self.model.zero_grad() From e9ce9bd879483def1f2d583d851360b986220af2 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 20 May 2024 11:48:36 -0400 Subject: [PATCH 20/28] Fix _apply methods to return for chaining --- library/src/armory/model/base.py | 2 +- library/src/armory/model/object_detection/object_detector.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/library/src/armory/model/base.py b/library/src/armory/model/base.py index a1383418..3d197954 100644 --- a/library/src/armory/model/base.py +++ b/library/src/armory/model/base.py @@ -68,8 +68,8 @@ def __init__( self.device = torch.device("cpu") def _apply(self, fn, *args, **kwargs): - super()._apply(fn, *args, **kwargs) self.device = fn(torch.zeros(1)).device + return super()._apply(fn, *args, **kwargs) def forward(self, *args, **kwargs): """ diff --git a/library/src/armory/model/object_detection/object_detector.py b/library/src/armory/model/object_detection/object_detector.py index ca90bd69..db4f6c3b 100644 --- a/library/src/armory/model/object_detection/object_detector.py +++ b/library/src/armory/model/object_detection/object_detector.py @@ -85,9 +85,10 @@ def __init__( self.score_threshold = score_threshold def _apply(self, *args, **kwargs): - super()._apply(*args, **kwargs) + res = super()._apply(*args, **kwargs) if isinstance(self.inputs_spec, TorchSpec): self.inputs_spec.to(device=self.device) + return res def _filter_predictions(self, preds): for pred in preds: From 373d2ebd1359d22b7fad6be6a4238306e36667b4 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 20 May 2024 11:49:43 -0400 Subject: [PATCH 21/28] Allow custom loss function in YOLOv5 wrapper --- .../object_detection/yolov5_object_detector.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/library/src/armory/model/object_detection/yolov5_object_detector.py b/library/src/armory/model/object_detection/yolov5_object_detector.py index f90c3987..8a2ef1fc 100644 --- a/library/src/armory/model/object_detection/yolov5_object_detector.py +++ b/library/src/armory/model/object_detection/yolov5_object_detector.py @@ -1,5 +1,5 @@ from functools import partial -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional, Tuple import torch @@ -48,6 +48,7 @@ def __init__( predictions_spec: Optional[BoundingBoxSpec] = None, iou_threshold: Optional[float] = None, score_threshold: Optional[float] = None, + compute_loss: Optional[Callable[[Any, Any], Tuple[Any, Any]]] = None, **kwargs, ): """ @@ -69,6 +70,12 @@ def __init__( predictions_spec: Optional, data specification used to update the object detection predictions in the batch. Defaults to a bounding box specification compatible with typical YOLOv5 models. + compute_loss: Optional, loss function used to calculate loss when the + model is in training mode. By default, the standard YOLOv5 loss + function is used. The function must accept two arguments: the + model predictions and the ground truth targets. The function + must return a tuple of the loss and the loss items (the second + element is unused). **kwargs: All other keyword arguments will be forwarded to the `yolov5.utils.general.non_max_suppression` function used to postprocess the model outputs. @@ -97,7 +104,11 @@ def __init__( self._detection_model: "DetectionModel" = ( detection_model if detection_model is not None else self._model.model.model ) - self.compute_loss = ComputeLoss(self._detection_model) + self.compute_loss = ( + compute_loss + if compute_loss is not None + else ComputeLoss(self._detection_model) + ) if score_threshold is not None: kwargs["conf_thres"] = score_threshold @@ -114,7 +125,7 @@ def forward(self, x, targets=None): """ # inputs: CHW images, 0.0-1.0 float # outputs: (N,6) detections (cx,cy,w,h,scores,labels) - if self.training and targets is not None: + if targets is not None: outputs = self._detection_model(x) loss, _ = self.compute_loss(outputs, targets) return dict(loss_total=loss) From 10ce3c1cc88b39fc19bbea636da8b8dfbb980582 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 20 May 2024 13:36:38 -0400 Subject: [PATCH 22/28] Use randomized locations for patch --- .../visdrone_yolov5_robustdpatch.py | 90 +++++++++---------- 1 file changed, 42 insertions(+), 48 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 060a2d17..4e94a004 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -1,6 +1,5 @@ import logging - -# import random +import random from typing import Optional, Sequence import PIL.Image @@ -37,7 +36,6 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): self.initial_patch = self.patch.clone() self.targeted = False self.learning_rate = 0.1 - self.sample_size = 10 self.augmentation = kornia.augmentation.container.ImageSequential( kornia.augmentation.RandomHorizontalFlip(p=0.5), kornia.augmentation.RandomBrightness(brightness=(0.75, 1.25), p=0.5), @@ -69,51 +67,47 @@ def on_train_epoch_end(self): # self.patch = torch.clip(self.patch, 0, self.model.inputs_spec.scale.max) def training_step(self, batch: armory.data.Batch, batch_idx: int): - for _ in range(self.sample_size): - # Get inputs as Tensor - inputs = batch.inputs.get(self.model.inputs_spec) - assert isinstance(inputs, torch.Tensor) - - # Apply patch to image - # x_1 = random.randint(0, inputs.shape[3] - self.patch_shape[2]) - # y_1 = random.randint(0, inputs.shape[2] - self.patch_shape[1]) - x_1, y_1 = self.patch_location - # x_1, y_1 = self.patch_location - x_2 = x_1 + self.patch_shape[1] - y_2 = y_1 + self.patch_shape[2] - inputs_with_patch = inputs.clone() - inputs_with_patch[:, :, x_1:x_2, y_1:y_2] = self.patch - - # Apply random augmentations to images - inputs_with_augmentations = self.augmentation(inputs_with_patch) - - # Get targets as Tensor - _, _, height, width = inputs.shape - targets = batch.targets.get(self.target_spec) - yolo_targets = self._to_yolo_targets( - targets, height, width, self.model.device - ) - - # Get loss from model outputs - self.model.eval() - loss_components = self.model(inputs_with_augmentations, yolo_targets) - loss = loss_components["loss_total"] - - self.log("loss", loss) - # loss = -loss - - # Clean gradients - # self.model.zero_grad() - # Compute gradients - # loss.backward(retain_graph=True) - # assert patch.grad is not None - # grads = patch.grad.clone() - # assert grads.shape == self.patch.shape - # Accumulate gradients - # patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) - # self.patch_gradients = patch_gradients - - return loss + # Get inputs as Tensor + inputs = batch.inputs.get(self.model.inputs_spec) + assert isinstance(inputs, torch.Tensor) + + # Apply patch to image + x_1 = random.randint(0, inputs.shape[3] - self.patch_shape[2]) + y_1 = random.randint(0, inputs.shape[2] - self.patch_shape[1]) + # x_1, y_1 = self.patch_location + x_2 = x_1 + self.patch_shape[1] + y_2 = y_1 + self.patch_shape[2] + inputs_with_patch = inputs.clone() + inputs_with_patch[:, :, x_1:x_2, y_1:y_2] = self.patch + + # Apply random augmentations to images + inputs_with_augmentations = self.augmentation(inputs_with_patch) + + # Get targets as Tensor + _, _, height, width = inputs.shape + targets = batch.targets.get(self.target_spec) + yolo_targets = self._to_yolo_targets(targets, height, width, self.model.device) + + # Get loss from model outputs + self.model.eval() + loss_components = self.model(inputs_with_augmentations, yolo_targets) + loss = loss_components["loss_total"] + + self.log("loss", loss) + # loss = -loss + + # Clean gradients + # self.model.zero_grad() + # Compute gradients + # loss.backward(retain_graph=True) + # assert patch.grad is not None + # grads = patch.grad.clone() + # assert grads.shape == self.patch.shape + # Accumulate gradients + # patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) + # self.patch_gradients = patch_gradients + + return loss @staticmethod def _to_yolo_targets( From 0056c67f87d14b9dfb2089ccdfbd27272e560fb1 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 21 May 2024 10:38:02 -0400 Subject: [PATCH 23/28] Clean up patch lightning module --- .../visdrone_yolov5_robustdpatch.py | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 4e94a004..3311c57c 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -27,11 +27,9 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): self.patch_shape = (3, 50, 50) self.patch_location = (295, 295) # middle of 640x640 # TODO non-zero min value - self.patch = ( - torch.randint(0, 255, self.patch_shape) - / 255 - * self.model.inputs_spec.scale.max - ) + self.patch_min = 0 + self.patch_max = self.model.inputs_spec.scale.max + self.patch = torch.randint(0, 255, self.patch_shape) / 255 * self.patch_max self.patch.requires_grad = True self.initial_patch = self.patch.clone() self.targeted = False @@ -57,15 +55,6 @@ def on_train_epoch_end(self): self.logger.run_id, patch, f"patch_epoch_{self.current_epoch}.png" ) - # self.patch = ( - # self.patch - # + torch.sign(self.patch_gradients) - # * (1 - 2 * int(self.targeted)) - # * self.learning_rate - # ) - # TODO handle normalized min/max - # self.patch = torch.clip(self.patch, 0, self.model.inputs_spec.scale.max) - def training_step(self, batch: armory.data.Batch, batch_idx: int): # Get inputs as Tensor inputs = batch.inputs.get(self.model.inputs_spec) @@ -94,18 +83,6 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): loss = loss_components["loss_total"] self.log("loss", loss) - # loss = -loss - - # Clean gradients - # self.model.zero_grad() - # Compute gradients - # loss.backward(retain_graph=True) - # assert patch.grad is not None - # grads = patch.grad.clone() - # assert grads.shape == self.patch.shape - # Accumulate gradients - # patch_gradients = self.patch_gradients + torch.sum(grads, dim=0) - # self.patch_gradients = patch_gradients return loss From fb32dfefbda1fe922ae4c7a610274817c4e992a4 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 21 May 2024 11:28:45 -0400 Subject: [PATCH 24/28] Put model in training mode so loss can be calculated --- .../examples/object_detection/visdrone_yolov5_robustdpatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 3311c57c..cb2720a3 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -78,7 +78,7 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): yolo_targets = self._to_yolo_targets(targets, height, width, self.model.device) # Get loss from model outputs - self.model.eval() + self.model.train() loss_components = self.model(inputs_with_augmentations, yolo_targets) loss = loss_components["loss_total"] From dd59429af909e64a642891f4ca807c713b6c0cb6 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 3 Jun 2024 15:52:11 -0400 Subject: [PATCH 25/28] Remove dependency on order of files in visdrone archive --- .../object_detection/datasets/visdrone.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py index f322c05d..59cdec94 100644 --- a/examples/src/armory/examples/object_detection/datasets/visdrone.py +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -197,22 +197,22 @@ def generate_samples( files: Iterator[Tuple[str, io.BufferedReader]], annotation_file_ext: str = ".txt" ) -> Iterator[Dict[str, Any]]: """Generate dataset samples from the given files in a VisDrone2019 archive""" - image_to_annotations = {} - # This loop relies on the ordering of the files in the archive: - # Annotation files come first, then the images. - for idx, (path, file) in enumerate(files): + annotations = {} + images = {} + for path, file in files: file_name = Path(path).stem if Path(path).suffix == annotation_file_ext: - image_to_annotations[file_name] = load_annotations(file) - elif file_name in image_to_annotations: - yield { - "image_id": idx, - "file_name": file_name, - "image": {"path": path, "bytes": file.read()}, - "objects": image_to_annotations[file_name], - } + annotations[file_name] = load_annotations(file) else: - raise ValueError(f"Image {file_name} has no annotations") + images[file_name] = {"path": path, "bytes": file.read()} + + for idx, (file_name, annotation) in enumerate(annotations.items()): + yield { + "image_id": idx, + "file_name": file_name, + "image": images[file_name], + "objects": annotation, + } if __name__ == "__main__": From 6dc7043c8f875c8f732280b0534c13e350017155 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 10 Jun 2024 17:00:50 -0400 Subject: [PATCH 26/28] Include test split in visdrone dataset --- .../examples/object_detection/datasets/visdrone.py | 13 ++++++++++--- .../examples/object_detection/visdrone_yolov5.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/src/armory/examples/object_detection/datasets/visdrone.py b/examples/src/armory/examples/object_detection/datasets/visdrone.py index 59cdec94..377ba3c3 100644 --- a/examples/src/armory/examples/object_detection/datasets/visdrone.py +++ b/examples/src/armory/examples/object_detection/datasets/visdrone.py @@ -84,6 +84,7 @@ def transform(sample): TRAIN_URL = "https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-train.zip" VAL_URL = "https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-val.zip" +TEST_URL = "https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-dev.zip" def load_dataset() -> datasets.DatasetDict: @@ -95,21 +96,27 @@ def load_dataset() -> datasets.DatasetDict: """ dl_manager = datasets.DownloadManager(dataset_name="VisDrone2019") ds_features = features() - paths = dl_manager.download({"train": TRAIN_URL, "val": VAL_URL}) + paths = dl_manager.download({"train": TRAIN_URL, "val": VAL_URL, "test": TEST_URL}) train_files = dl_manager.iter_archive(paths["train"]) val_files = dl_manager.iter_archive(paths["val"]) + test_files = dl_manager.iter_archive(paths["test"]) return datasets.DatasetDict( { - "train": datasets.Dataset.from_generator( + datasets.Split.TRAIN: datasets.Dataset.from_generator( generate_samples, gen_kwargs={"files": train_files}, features=ds_features, ), - "val": datasets.Dataset.from_generator( + datasets.Split.VALIDATION: datasets.Dataset.from_generator( generate_samples, gen_kwargs={"files": val_files}, features=ds_features, ), + datasets.Split.TEST: datasets.Dataset.from_generator( + generate_samples, + gen_kwargs={"files": test_files}, + features=ds_features, + ), } ) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5.py b/examples/src/armory/examples/object_detection/visdrone_yolov5.py index 0906dbc9..65d919db 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5.py @@ -61,7 +61,7 @@ def load_dataset( batch_size: int, shuffle: bool, seed: Optional[int] = None, - split: Union[Literal["val"], Literal["train"]] = "val", + split: Union[Literal["validation"], Literal["train"]] = "validation", ): """Load VisDrone dataset""" with evaluation.autotrack(): From 7e802027db98df2c44653f93a473ee5e66b1c3fb Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Mon, 10 Jun 2024 17:03:34 -0400 Subject: [PATCH 27/28] Remove initial_patch from module --- .../visdrone_yolov5_robustdpatch.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index cb2720a3..4354fb8c 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -29,9 +29,8 @@ def __init__(self, model: armory.model.object_detection.YoloV5ObjectDetector): # TODO non-zero min value self.patch_min = 0 self.patch_max = self.model.inputs_spec.scale.max - self.patch = torch.randint(0, 255, self.patch_shape) / 255 * self.patch_max + self.patch = torch.randint(0, 256, self.patch_shape) / 255 * self.patch_max self.patch.requires_grad = True - self.initial_patch = self.patch.clone() self.targeted = False self.learning_rate = 0.1 self.augmentation = kornia.augmentation.container.ImageSequential( @@ -139,15 +138,16 @@ def load_dataset(batch_size: int, shuffle: bool, seed: Optional[int] = None): model = load_model() module = RobustDPatchModule(model) - trainer = Trainer(limit_train_batches=10, max_epochs=20) - trainer.fit(module, dataloader) - patch_np = ( - module.initial_patch.detach().cpu().numpy().transpose(1, 2, 0) * 255 - ).astype("uint8") + patch_np = (module.patch.detach().cpu().numpy().transpose(1, 2, 0) * 255).astype( + "uint8" + ) patch = PIL.Image.fromarray(patch_np) patch.save("initial_patch.png") + trainer = Trainer(limit_train_batches=10, max_epochs=20) + trainer.fit(module, dataloader) + patch_np = (module.patch.detach().cpu().numpy().transpose(1, 2, 0) * 255).astype( "uint8" ) From 062e2856203b882634e9463fa144dfb2ecad4774 Mon Sep 17 00:00:00 2001 From: Kyle Treubig Date: Tue, 11 Jun 2024 09:07:49 -0400 Subject: [PATCH 28/28] Negate the loss so optimizing it increases the loss --- .../examples/object_detection/visdrone_yolov5_robustdpatch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py index 4354fb8c..277535ef 100644 --- a/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py +++ b/examples/src/armory/examples/object_detection/visdrone_yolov5_robustdpatch.py @@ -79,7 +79,8 @@ def training_step(self, batch: armory.data.Batch, batch_idx: int): # Get loss from model outputs self.model.train() loss_components = self.model(inputs_with_augmentations, yolo_targets) - loss = loss_components["loss_total"] + # Negate the loss because the optimizer will be minimizing it + loss = -loss_components["loss_total"] self.log("loss", loss)