From d44903679997cff612ee6bd5ce71c97706b822d6 Mon Sep 17 00:00:00 2001 From: William Patton Date: Tue, 2 Jan 2024 09:46:44 -0800 Subject: [PATCH] Fix bug in rasterize graph we were using `graph.data.items()` to iterate over nodes instead of `graph.nodes` Squashed commit of the following: commit d027f5a260a1e2a9cf851efca85b7318434675d6 Author: William Patton Date: Tue Jan 2 09:44:21 2024 -0800 refactor rasterize_points test to use pytest commit eadb0476d8475b55120486df6cf30f95b6df86f4 Author: William Patton Date: Tue Jan 2 09:25:11 2024 -0800 remove extra roi handling The node only needs to request the data it needs for its own operations. If you request a mask for a set of points that extend outside the bounds of your mask you will get an error commit 29507f1f21d69cf76e34e7b0f05cd780100fd68b Author: William Patton Date: Tue Jan 2 09:22:21 2024 -0800 remove type cast we do a bitwise during the `__rasterize` call which results fails if you change the dtype commit 96e93e53ce0bc8240357259dab92f1ca64a08199 Author: William Patton Date: Tue Jan 2 09:21:16 2024 -0800 remove matplotlib commit eb2977a187a1cad95da54a515c84ce44d73b8315 Author: Samia Mohinta <44754434+Mohinta2892@users.noreply.github.com> Date: Thu Dec 14 15:27:41 2023 +0000 fix mask intersection with request outputs must match request rois when a mask is provided commit 682189dac2ef6b94876bd30df813717da6530060 Author: Samia Mohinta <44754434+Mohinta2892@users.noreply.github.com> Date: Thu Dec 14 15:25:12 2023 +0000 Update rasterize_graph.py commit e36dcf179ccd1aec6a5cafd31e7a9a858352faa1 Author: Mohinta2892 Date: Thu Nov 2 19:18:00 2023 +0000 reformat rasterize_graph and rasterize_points commit 42da2702e746f702d3d07144ab9fc1d4352b0c0d Author: Samia Mohinta <44754434+Mohinta2892@users.noreply.github.com> Date: Thu Nov 2 14:23:01 2023 +0000 Test for issue #193 Test added to pass mask to `RasterizeGraph()` via `RasterizationSettings`. commit b17cfad413f5ad7f48045a2167ec20d89674d939 Author: Samia Mohinta <44754434+Mohinta2892@users.noreply.github.com> Date: Thu Nov 2 14:19:42 2023 +0000 fix for issue #193 lines 224-226: replace graph.data.items() with graph.nodes lines 255-257: explicitly cast the boolean mask data to the original dtype of mask_array --- gunpowder/nodes/rasterize_graph.py | 9 +- tests/cases/rasterize_points.py | 412 +++++++++++++++-------------- 2 files changed, 217 insertions(+), 204 deletions(-) diff --git a/gunpowder/nodes/rasterize_graph.py b/gunpowder/nodes/rasterize_graph.py index 1a12335f..b660dd57 100644 --- a/gunpowder/nodes/rasterize_graph.py +++ b/gunpowder/nodes/rasterize_graph.py @@ -221,7 +221,8 @@ def process(self, batch, request): mask_array = batch.arrays[mask].crop(enlarged_vol_roi) # get those component labels in the mask, that contain graph labels = [] - for i, point in graph.data.items(): + # for i, point in graph.data.items(): + for i, point in enumerate(graph.nodes): v = Coordinate(point.location / voxel_size) v -= data_roi.begin labels.append(mask_array.data[v]) @@ -250,11 +251,15 @@ def process(self, batch, request): voxel_size, self.spec[self.array].dtype, self.settings, - Array(data=mask_array.data == label, spec=mask_array.spec), + Array( + data=(mask_array.data == label), + spec=mask_array.spec, + ), ) for label in labels ], axis=0, + dtype=self.spec[self.array].dtype, ) else: diff --git a/tests/cases/rasterize_points.py b/tests/cases/rasterize_points.py index 16c11ae1..a57906f8 100644 --- a/tests/cases/rasterize_points.py +++ b/tests/cases/rasterize_points.py @@ -1,31 +1,30 @@ -from .provider_test import ProviderTest +from .helper_sources import ArraySource, GraphSource from gunpowder import ( - BatchProvider, BatchRequest, - Batch, Roi, Coordinate, GraphSpec, Array, ArrayKey, - ArrayKeys, ArraySpec, RasterizeGraph, + MergeProvider, RasterizationSettings, build, ) -from gunpowder.graph import GraphKeys, GraphKey, Graph, Node, Edge +from gunpowder.graph import GraphKey, Graph, Node, Edge import numpy as np -import math -from random import randint -class GraphTestSource3D(BatchProvider): - def __init__(self): - self.voxel_size = Coordinate((40, 4, 4)) +def test_3d(): + graph_key = GraphKey("TEST_GRAPH") + array_key = ArrayKey("TEST_ARRAY") + rasterized_key = ArrayKey("RASTERIZED_ARRAY") + voxel_size = Coordinate((40, 4, 4)) - self.nodes = [ + graph = Graph( + [ # corners Node(id=1, location=np.array((-200, -200, -200))), Node(id=2, location=np.array((-200, -200, 199))), @@ -38,249 +37,258 @@ def __init__(self): # center Node(id=9, location=np.array((0, 0, 0))), Node(id=10, location=np.array((-1, -1, -1))), - ] - - self.graph_spec = GraphSpec(roi=Roi((-100, -100, -100), (300, 300, 300))) - self.array_spec = ArraySpec( - roi=Roi((-200, -200, -200), (400, 400, 400)), voxel_size=self.voxel_size - ) - - self.graph = Graph(self.nodes, [], self.graph_spec) - - def setup(self): - self.provides( - GraphKeys.TEST_GRAPH, - self.graph_spec, - ) - - self.provides( - ArrayKeys.GT_LABELS, - self.array_spec, - ) - - def provide(self, request): - batch = Batch() - - graph_roi = request[GraphKeys.TEST_GRAPH].roi - - batch.graphs[GraphKeys.TEST_GRAPH] = self.graph.crop(graph_roi).trim(graph_roi) - - roi_array = request[ArrayKeys.GT_LABELS].roi - - image = np.ones(roi_array.shape / self.voxel_size, dtype=np.uint64) - # label half of GT_LABELS differently - depth = image.shape[0] - image[0 : depth // 2] = 2 - - spec = self.spec[ArrayKeys.GT_LABELS].copy() - spec.roi = roi_array - batch.arrays[ArrayKeys.GT_LABELS] = Array(image, spec=spec) - - return batch - - -class GraphTestSourceWithEdge(BatchProvider): - def __init__(self): - self.voxel_size = Coordinate((1, 1, 1)) - - self.nodes = [ - # corners - Node(id=1, location=np.array((0, 4, 4))), - Node(id=2, location=np.array((9, 4, 4))), - ] - self.edges = [Edge(1, 2)] - - self.graph_spec = GraphSpec(roi=Roi((0, 0, 0), (10, 10, 10))) - self.graph = Graph(self.nodes, self.edges, self.graph_spec) - - def setup(self): - self.provides( - GraphKeys.TEST_GRAPH_WITH_EDGE, - self.graph_spec, - ) - - def provide(self, request): - batch = Batch() - - graph_roi = request[GraphKeys.TEST_GRAPH_WITH_EDGE].roi - - batch.graphs[GraphKeys.TEST_GRAPH_WITH_EDGE] = self.graph.crop(graph_roi).trim( - graph_roi - ) - - return batch - - -class TestRasterizePoints(ProviderTest): - def test_3d(self): - GraphKey("TEST_GRAPH") - ArrayKey("RASTERIZED") - - pipeline = GraphTestSource3D() + RasterizeGraph( - GraphKeys.TEST_GRAPH, - ArrayKeys.RASTERIZED, + ], + [], + GraphSpec(roi=Roi((-100, -100, -100), (300, 300, 300))), + ) + + array = Array( + np.ones((10, 100, 100)), + ArraySpec( + roi=Roi((-200, -200, -200), (400, 400, 400)), + voxel_size=voxel_size, + ), + ) + + pipeline = ( + (GraphSource(graph_key, graph), ArraySource(array_key, array)) + + MergeProvider() + + RasterizeGraph( + graph_key, + rasterized_key, ArraySpec(voxel_size=(40, 4, 4)), ) + ) - with build(pipeline): - request = BatchRequest() - roi = Roi((0, 0, 0), (200, 200, 200)) + with build(pipeline): + request = BatchRequest() + roi = Roi((0, 0, 0), (200, 200, 200)) - request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=roi) - request[ArrayKeys.GT_LABELS] = ArraySpec(roi=roi) - request[ArrayKeys.RASTERIZED] = ArraySpec(roi=roi) + request[graph_key] = GraphSpec(roi=roi) + request[array_key] = ArraySpec(roi=roi) + request[rasterized_key] = ArraySpec(roi=roi) - batch = pipeline.request_batch(request) + batch = pipeline.request_batch(request) - rasterized = batch.arrays[ArrayKeys.RASTERIZED].data - self.assertEqual(rasterized[0, 0, 0], 1) - self.assertEqual(rasterized[2, 20, 20], 0) - self.assertEqual(rasterized[4, 49, 49], 1) + rasterized = batch.arrays[rasterized_key].data + assert rasterized[0, 0, 0] == 1 + assert rasterized[2, 20, 20] == 0 + assert rasterized[4, 49, 49] == 1 - # same with different foreground/background labels + # same with different foreground/background labels - pipeline = GraphTestSource3D() + RasterizeGraph( - GraphKeys.TEST_GRAPH, - ArrayKeys.RASTERIZED, + pipeline = ( + (GraphSource(graph_key, graph), ArraySource(array_key, array)) + + MergeProvider() + + RasterizeGraph( + graph_key, + rasterized_key, ArraySpec(voxel_size=(40, 4, 4)), RasterizationSettings(radius=1, fg_value=0, bg_value=1), ) + ) - with build(pipeline): - request = BatchRequest() - roi = Roi((0, 0, 0), (200, 200, 200)) + with build(pipeline): + request = BatchRequest() + roi = Roi((0, 0, 0), (200, 200, 200)) - request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=roi) - request[ArrayKeys.GT_LABELS] = ArraySpec(roi=roi) - request[ArrayKeys.RASTERIZED] = ArraySpec(roi=roi) + request[graph_key] = GraphSpec(roi=roi) + request[array_key] = ArraySpec(roi=roi) + request[rasterized_key] = ArraySpec(roi=roi) - batch = pipeline.request_batch(request) + batch = pipeline.request_batch(request) - rasterized = batch.arrays[ArrayKeys.RASTERIZED].data - self.assertEqual(rasterized[0, 0, 0], 0) - self.assertEqual(rasterized[2, 20, 20], 1) - self.assertEqual(rasterized[4, 49, 49], 0) + rasterized = batch.arrays[rasterized_key].data + assert rasterized[0, 0, 0] == 0 + assert rasterized[2, 20, 20] == 1 + assert rasterized[4, 49, 49] == 0 - # same with different radius and inner radius + # same with different radius and inner radius - pipeline = GraphTestSource3D() + RasterizeGraph( - GraphKeys.TEST_GRAPH, - ArrayKeys.RASTERIZED, + pipeline = ( + (GraphSource(graph_key, graph), ArraySource(array_key, array)) + + MergeProvider() + + RasterizeGraph( + graph_key, + rasterized_key, ArraySpec(voxel_size=(40, 4, 4)), RasterizationSettings( radius=40, inner_radius_fraction=0.25, fg_value=1, bg_value=0 ), ) + ) - with build(pipeline): - request = BatchRequest() - roi = Roi((0, 0, 0), (200, 200, 200)) + with build(pipeline): + request = BatchRequest() + roi = Roi((0, 0, 0), (200, 200, 200)) - request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=roi) - request[ArrayKeys.GT_LABELS] = ArraySpec(roi=roi) - request[ArrayKeys.RASTERIZED] = ArraySpec(roi=roi) + request[graph_key] = GraphSpec(roi=roi) + request[array_key] = ArraySpec(roi=roi) + request[rasterized_key] = ArraySpec(roi=roi) - batch = pipeline.request_batch(request) + batch = pipeline.request_batch(request) - rasterized = batch.arrays[ArrayKeys.RASTERIZED].data + rasterized = batch.arrays[rasterized_key].data - # in the middle of the ball, there should be 0 (since inner radius is set) - self.assertEqual(rasterized[0, 0, 0], 0) - # check larger radius: rasterized point (0, 0, 0) should extend in - # x,y by 10; z, by 1 - self.assertEqual(rasterized[0, 10, 0], 1) - self.assertEqual(rasterized[0, 0, 10], 1) - self.assertEqual(rasterized[1, 0, 0], 1) + # in the middle of the ball, there should be 0 (since inner radius is set) + assert rasterized[0, 0, 0] == 0 + # check larger radius: rasterized point (0, 0, 0) should extend in + # x,y by 10; z, by 1 + assert rasterized[0, 10, 0] == 1 + assert rasterized[0, 0, 10] == 1 + assert rasterized[1, 0, 0] == 1 - self.assertEqual(rasterized[2, 20, 20], 0) - self.assertEqual(rasterized[4, 49, 49], 0) + assert rasterized[2, 20, 20] == 0 + assert rasterized[4, 49, 49] == 0 - # same with anisotropic radius + # same with different foreground/background labels + # and GT_LABELS as mask of type np.uint64. Issue #193 - pipeline = GraphTestSource3D() + RasterizeGraph( - GraphKeys.TEST_GRAPH, - ArrayKeys.RASTERIZED, + pipeline = ( + (GraphSource(graph_key, graph), ArraySource(array_key, array)) + + MergeProvider() + + RasterizeGraph( + graph_key, + rasterized_key, ArraySpec(voxel_size=(40, 4, 4)), - RasterizationSettings(radius=(40, 40, 20), fg_value=1, bg_value=0), + RasterizationSettings(radius=1, fg_value=0, bg_value=1, mask=array_key), ) + ) - with build(pipeline): - request = BatchRequest() - roi = Roi((0, 0, 0), (120, 80, 80)) + with build(pipeline): + request = BatchRequest() + roi = Roi((0, 0, 0), (200, 200, 200)) - request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=roi) - request[ArrayKeys.GT_LABELS] = ArraySpec(roi=roi) - request[ArrayKeys.RASTERIZED] = ArraySpec(roi=roi) + request[graph_key] = GraphSpec(roi=roi) + request[array_key] = ArraySpec(roi=roi) + request[rasterized_key] = ArraySpec(roi=roi) - batch = pipeline.request_batch(request) + batch = pipeline.request_batch(request) - rasterized = batch.arrays[ArrayKeys.RASTERIZED].data + rasterized = batch.arrays[rasterized_key].data + assert rasterized[0, 0, 0] == 0 + assert rasterized[2, 20, 20] == 1 + assert rasterized[4, 49, 49] == 0 - # check larger radius: rasterized point (0, 0, 0) should extend in - # x,y by 10; z, by 1 - self.assertEqual(rasterized[0, 10, 0], 1) - self.assertEqual(rasterized[0, 11, 0], 0) - self.assertEqual(rasterized[0, 0, 5], 1) - self.assertEqual(rasterized[0, 0, 6], 0) - self.assertEqual(rasterized[1, 0, 0], 1) - self.assertEqual(rasterized[2, 0, 0], 0) + # same with anisotropic radius - # same with anisotropic radius and inner radius - - pipeline = GraphTestSource3D() + RasterizeGraph( - GraphKeys.TEST_GRAPH, - ArrayKeys.RASTERIZED, + pipeline = ( + (GraphSource(graph_key, graph), ArraySource(array_key, array)) + + MergeProvider() + + RasterizeGraph( + graph_key, + rasterized_key, ArraySpec(voxel_size=(40, 4, 4)), - RasterizationSettings( - radius=(40, 40, 20), inner_radius_fraction=0.75, fg_value=1, bg_value=0 - ), + RasterizationSettings(radius=(40, 40, 20), fg_value=1, bg_value=0), ) + ) - with build(pipeline): - request = BatchRequest() - roi = Roi((0, 0, 0), (120, 80, 80)) + with build(pipeline): + request = BatchRequest() + roi = Roi((0, 0, 0), (120, 80, 80)) - request[GraphKeys.TEST_GRAPH] = GraphSpec(roi=roi) - request[ArrayKeys.GT_LABELS] = ArraySpec(roi=roi) - request[ArrayKeys.RASTERIZED] = ArraySpec(roi=roi) + request[graph_key] = GraphSpec(roi=roi) + request[array_key] = ArraySpec(roi=roi) + request[rasterized_key] = ArraySpec(roi=roi) - batch = pipeline.request_batch(request) + batch = pipeline.request_batch(request) - rasterized = batch.arrays[ArrayKeys.RASTERIZED].data + rasterized = batch.arrays[rasterized_key].data - # in the middle of the ball, there should be 0 (since inner radius is set) - self.assertEqual(rasterized[0, 0, 0], 0) - # check larger radius: rasterized point (0, 0, 0) should extend in - # x,y by 10; z, by 1 - self.assertEqual(rasterized[0, 10, 0], 1) - self.assertEqual(rasterized[0, 11, 0], 0) - self.assertEqual(rasterized[0, 0, 5], 1) - self.assertEqual(rasterized[0, 0, 6], 0) - self.assertEqual(rasterized[1, 0, 0], 1) - self.assertEqual(rasterized[2, 0, 0], 0) + # check larger radius: rasterized point (0, 0, 0) should extend in + # x,y by 10; z, by 1 + assert rasterized[0, 10, 0] == 1 + assert rasterized[0, 11, 0] == 0 + assert rasterized[0, 0, 5] == 1 + assert rasterized[0, 0, 6] == 0 + assert rasterized[1, 0, 0] == 1 + assert rasterized[2, 0, 0] == 0 - def test_with_edge(self): - graph_with_edge = GraphKey("TEST_GRAPH_WITH_EDGE") - array_with_edge = ArrayKey("RASTERIZED_EDGE") + # same with anisotropic radius and inner radius - pipeline = GraphTestSourceWithEdge() + RasterizeGraph( - GraphKeys.TEST_GRAPH_WITH_EDGE, - ArrayKeys.RASTERIZED_EDGE, + pipeline = ( + (GraphSource(graph_key, graph), ArraySource(array_key, array)) + + MergeProvider() + + RasterizeGraph( + graph_key, + rasterized_key, + ArraySpec(voxel_size=(40, 4, 4)), + RasterizationSettings( + radius=(40, 40, 20), inner_radius_fraction=0.75, fg_value=1, bg_value=0 + ), + ) + ) + + with build(pipeline): + request = BatchRequest() + roi = Roi((0, 0, 0), (120, 80, 80)) + + request[graph_key] = GraphSpec(roi=roi) + request[array_key] = ArraySpec(roi=roi) + request[rasterized_key] = ArraySpec(roi=roi) + + batch = pipeline.request_batch(request) + + rasterized = batch.arrays[rasterized_key].data + + # in the middle of the ball, there should be 0 (since inner radius is set) + assert rasterized[0, 0, 0] == 0 + # check larger radius: rasterized point (0, 0, 0) should extend in + # x,y by 10; z, by 1 + assert rasterized[0, 10, 0] == 1 + assert rasterized[0, 11, 0] == 0 + assert rasterized[0, 0, 5] == 1 + assert rasterized[0, 0, 6] == 0 + assert rasterized[1, 0, 0] == 1 + assert rasterized[2, 0, 0] == 0 + + +def test_with_edge(): + graph_key = GraphKey("TEST_GRAPH") + array_key = ArrayKey("TEST_ARRAY") + rasterized_key = ArrayKey("RASTERIZED_ARRAY") + voxel_size = Coordinate((40, 4, 4)) + + array = Array( + np.ones((10, 100, 100)), + ArraySpec( + roi=Roi((-200, -200, -200), (400, 400, 400)), + voxel_size=voxel_size, + ), + ) + + graph = Graph( + [ + # corners + Node(id=1, location=np.array((0, 4, 4))), + Node(id=2, location=np.array((9, 4, 4))), + ], + [Edge(1, 2)], + GraphSpec(roi=Roi((0, 0, 0), (10, 10, 10))), + ) + + pipeline = ( + (GraphSource(graph_key, graph), ArraySource(array_key, array)) + + MergeProvider() + + RasterizeGraph( + graph_key, + rasterized_key, ArraySpec(voxel_size=(1, 1, 1)), settings=RasterizationSettings(0.5), ) + ) - with build(pipeline): - request = BatchRequest() - roi = Roi((0, 0, 0), (10, 10, 10)) + with build(pipeline): + request = BatchRequest() + roi = Roi((0, 0, 0), (10, 10, 10)) - request[GraphKeys.TEST_GRAPH_WITH_EDGE] = GraphSpec(roi=roi) - request[ArrayKeys.RASTERIZED_EDGE] = ArraySpec(roi=roi) + request[graph_key] = GraphSpec(roi=roi) + request[rasterized_key] = ArraySpec(roi=roi) - batch = pipeline.request_batch(request) + batch = pipeline.request_batch(request) - rasterized = batch.arrays[ArrayKeys.RASTERIZED_EDGE].data + rasterized = batch.arrays[rasterized_key].data - assert ( - rasterized.sum() == 10 - ), f"rasterized has ones at: {np.where(rasterized==1)}" + assert ( + rasterized.sum() == 10 + ), f"rasterized has ones at: {np.where(rasterized==1)}"