From 47e9b910cf5e1024fce0c437653238e37bfd2f4c Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Mon, 29 Jul 2024 14:25:45 -0400 Subject: [PATCH 1/5] Fix path issue, reduce redundant overlaps --- .../findFeaturesSegment.py | 67 +++++++++++++------ 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index b4fb65fb52..39b9bdc099 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -134,7 +134,7 @@ def segment(img_path : Path, nlines : int = MAX_LEN): def generate_cnet(params, images): - match_segment_n = images["match"]["Segment"] + match_segment_n = images["match"][0]["Segment"] from_segment_n = images["from"][0]["Segment"] new_params = deepcopy(params) @@ -145,7 +145,7 @@ def generate_cnet(params, images): # make sure none of these keys are still in the params new_params.pop("FROMLIST", None) new_params.pop("FROM", None) - new_params["MATCH"] = images["match"]["Path"] + new_params["MATCH"] = images["match"][0]["Path"] og_onet = Path(params["ONET"]) og_tolist = Path(params["TOLIST"]) @@ -175,7 +175,7 @@ def generate_cnet(params, images): # check for overlaps is_overlapping = [] - for image in from_images: + for image in from_images: with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) overlapfromlist = tmpdir / "fromlist.lis" @@ -197,7 +197,7 @@ def generate_cnet(params, images): log.debug(f"overlap stats: {ret.stdout}") # first, throw it out if there is no overlap whatsoever - is_pair_overlapping = not any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) + is_pair_overlapping = any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) if is_pair_overlapping: is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0)) @@ -207,6 +207,9 @@ def generate_cnet(params, images): else: # not overlapping is_overlapping.append(False) + if not any(is_overlapping): + log.info("No overlaps were found.") + # mask images from_images = list(compress(from_images, is_overlapping)) log.debug(f"From images overlapping Match: {from_images}") @@ -318,12 +321,19 @@ def findFeaturesSegment(ui): pool.close() pool.join() output = output.get() - match_segments = segment(ui.GetCubeName("match"), ui.GetInteger("NL")) - from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output) + if len(img_list) > 1: + from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output) + else: + # img_list = 1 + from_segments = output[0] + for seg, value in from_segments.items(): + og_value = value + from_segments[seg] = [og_value] + segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist] segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())] - + # re-generate footprints pool = ThreadPool(ceil(nthreads/len(segment_paths))) output = pool.map_async(footprintinit, segment_paths) @@ -332,28 +342,47 @@ def findFeaturesSegment(ui): output = output.get() log.debug(f"{output}") - image_sets = list(itertools.product(match_segments.values(), from_segments.values())) - + image_sets = list(itertools.product(match_segments.values(), from_segments.values())) + x = match_segments.values() + y = from_segments.values() + x,y = (x,y) if len(x) > len(y) else (y,x) + image_cartesian = list(itertools.product(x, y)) + image_sets = [] + for i in image_cartesian: + if i[0][0]["Segment"] >= i[1]["Segment"]: + image_sets.append(i) + # restructure things to be easier to manage job_dicts = [] for im in image_sets: match = im[0] from_images = im[1] + if not isinstance(from_images, list): + # from_images must be list type + from_images_list = [] + from_images_list.append(from_images) + from_images = from_images_list job_dicts.append({ "match" : match, "from" : from_images - }) - + }) + # get params as a dictionary params = ui.GetParams() - - pool = ThreadPool(ceil(nthreads/len(job_dicts))) - starmap_args = list(zip([params]*len(job_dicts), job_dicts)) - output = pool.starmap_async(generate_cnet, starmap_args) - pool.close() - pool.join() - output = output.get() - log.debug(f"output: {output}") + try: + pool = ThreadPool(ceil(nthreads/len(job_dicts))) + starmap_args = list(zip([params]*len(job_dicts), job_dicts)) + output = pool.starmap_async(generate_cnet, starmap_args) + pool.close() + pool.join() + output = output.get() + log.debug(f"output: {output}") + except Exception as err: + log.debug('generate_cnet error') + log.debug(' '.join(err.cmd)) + log.debug(err.stdout) + log.debug(err.stderr) + return err # merge the networks onets = [o["onet"] for o in output if isinstance(o, dict)] From 1e3b7a2d1442898e63089048406db616fbc3010c Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Mon, 29 Jul 2024 14:54:58 -0400 Subject: [PATCH 2/5] Added comment --- .../apps/findFeaturesSegment/findFeaturesSegment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 39b9bdc099..21aca7a753 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -322,6 +322,7 @@ def findFeaturesSegment(ui): pool.join() output = output.get() match_segments = segment(ui.GetCubeName("match"), ui.GetInteger("NL")) + if len(img_list) > 1: from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output) else: @@ -331,6 +332,7 @@ def findFeaturesSegment(ui): og_value = value from_segments[seg] = [og_value] + print("FROM_SEGMENTS = " + str(from_segments)) segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist] segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())] @@ -342,6 +344,7 @@ def findFeaturesSegment(ui): output = output.get() log.debug(f"{output}") + # Remove redundant overlapping pairs image_sets = list(itertools.product(match_segments.values(), from_segments.values())) x = match_segments.values() y = from_segments.values() From 5d7776bb28865f178b9bacf5bff21f85d90671ce Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Tue, 30 Jul 2024 13:45:32 -0400 Subject: [PATCH 3/5] Remove old image_sets --- .../apps/findFeaturesSegment/findFeaturesSegment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 21aca7a753..f6d0025f20 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -345,7 +345,6 @@ def findFeaturesSegment(ui): log.debug(f"{output}") # Remove redundant overlapping pairs - image_sets = list(itertools.product(match_segments.values(), from_segments.values())) x = match_segments.values() y = from_segments.values() x,y = (x,y) if len(x) > len(y) else (y,x) From a31e6693f5e877171082f689e7fdb9a03a506875 Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Tue, 30 Jul 2024 15:26:10 -0400 Subject: [PATCH 4/5] Add check for match_segment_n --- .../apps/findFeaturesSegment/findFeaturesSegment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index f6d0025f20..50c00b685d 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -134,7 +134,10 @@ def segment(img_path : Path, nlines : int = MAX_LEN): def generate_cnet(params, images): - match_segment_n = images["match"][0]["Segment"] + if isinstance(images["match"], list): + match_segment_n = images["match"][0]["Segment"] + else: + match_segment_n = images["match"]["Segment"] from_segment_n = images["from"][0]["Segment"] new_params = deepcopy(params) @@ -332,7 +335,6 @@ def findFeaturesSegment(ui): og_value = value from_segments[seg] = [og_value] - print("FROM_SEGMENTS = " + str(from_segments)) segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist] segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())] From 698afc6fe7cd5db87324c7942a259bfb619edaf1 Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Fri, 2 Aug 2024 16:47:39 -0400 Subject: [PATCH 5/5] Revert overlap changes --- .../findFeaturesSegment.py | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 50c00b685d..d87b1f85e7 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -135,9 +135,11 @@ def segment(img_path : Path, nlines : int = MAX_LEN): def generate_cnet(params, images): if isinstance(images["match"], list): - match_segment_n = images["match"][0]["Segment"] + images_match_n = images["match"][0] else: - match_segment_n = images["match"]["Segment"] + + images_match_n = images["match"] + match_segment_n = images_match_n["Segment"] from_segment_n = images["from"][0]["Segment"] new_params = deepcopy(params) @@ -148,7 +150,7 @@ def generate_cnet(params, images): # make sure none of these keys are still in the params new_params.pop("FROMLIST", None) new_params.pop("FROM", None) - new_params["MATCH"] = images["match"][0]["Path"] + new_params["MATCH"] = images_match_n["Path"] og_onet = Path(params["ONET"]) og_tolist = Path(params["TOLIST"]) @@ -200,7 +202,7 @@ def generate_cnet(params, images): log.debug(f"overlap stats: {ret.stdout}") # first, throw it out if there is no overlap whatsoever - is_pair_overlapping = any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) + is_pair_overlapping = not any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) if is_pair_overlapping: is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0)) @@ -210,9 +212,6 @@ def generate_cnet(params, images): else: # not overlapping is_overlapping.append(False) - if not any(is_overlapping): - log.info("No overlaps were found.") - # mask images from_images = list(compress(from_images, is_overlapping)) log.debug(f"From images overlapping Match: {from_images}") @@ -236,12 +235,11 @@ def generate_cnet(params, images): log.debug(' '.join(err.cmd)) log.debug(err.stdout) log.debug(err.stderr) - return "ERROR" segmented_net = cnet.from_isis(new_params["ONET"]) # starting sample in inclusive, so subtract 1 - segmented_net.loc[segmented_net.serialnumber == images["match"]["SN"], "line"] += images["match"]["StartingLine"]-1 + segmented_net.loc[segmented_net.serialnumber == images_match_n["SN"], "line"] += images_match_n["StartingLine"]-1 # offset the images for k, image in enumerate(images["from"]): @@ -373,20 +371,13 @@ def findFeaturesSegment(ui): # get params as a dictionary params = ui.GetParams() - try: - pool = ThreadPool(ceil(nthreads/len(job_dicts))) - starmap_args = list(zip([params]*len(job_dicts), job_dicts)) - output = pool.starmap_async(generate_cnet, starmap_args) - pool.close() - pool.join() - output = output.get() - log.debug(f"output: {output}") - except Exception as err: - log.debug('generate_cnet error') - log.debug(' '.join(err.cmd)) - log.debug(err.stdout) - log.debug(err.stderr) - return err + pool = ThreadPool(ceil(nthreads/len(job_dicts))) + starmap_args = list(zip([params]*len(job_dicts), job_dicts)) + output = pool.starmap_async(generate_cnet, starmap_args) + pool.close() + pool.join() + output = output.get() + log.debug(f"output: {output}") # merge the networks onets = [o["onet"] for o in output if isinstance(o, dict)]