From 47e9b910cf5e1024fce0c437653238e37bfd2f4c Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Mon, 29 Jul 2024 14:25:45 -0400 Subject: [PATCH 01/11] Fix path issue, reduce redundant overlaps --- .../findFeaturesSegment.py | 67 +++++++++++++------ 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index b4fb65fb52..39b9bdc099 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -134,7 +134,7 @@ def segment(img_path : Path, nlines : int = MAX_LEN): def generate_cnet(params, images): - match_segment_n = images["match"]["Segment"] + match_segment_n = images["match"][0]["Segment"] from_segment_n = images["from"][0]["Segment"] new_params = deepcopy(params) @@ -145,7 +145,7 @@ def generate_cnet(params, images): # make sure none of these keys are still in the params new_params.pop("FROMLIST", None) new_params.pop("FROM", None) - new_params["MATCH"] = images["match"]["Path"] + new_params["MATCH"] = images["match"][0]["Path"] og_onet = Path(params["ONET"]) og_tolist = Path(params["TOLIST"]) @@ -175,7 +175,7 @@ def generate_cnet(params, images): # check for overlaps is_overlapping = [] - for image in from_images: + for image in from_images: with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) overlapfromlist = tmpdir / "fromlist.lis" @@ -197,7 +197,7 @@ def generate_cnet(params, images): log.debug(f"overlap stats: {ret.stdout}") # first, throw it out if there is no overlap whatsoever - is_pair_overlapping = not any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) + is_pair_overlapping = any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) if is_pair_overlapping: is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0)) @@ -207,6 +207,9 @@ def generate_cnet(params, images): else: # not overlapping is_overlapping.append(False) + if not any(is_overlapping): + log.info("No overlaps were found.") + # mask images from_images = list(compress(from_images, is_overlapping)) log.debug(f"From images overlapping Match: {from_images}") @@ -318,12 +321,19 @@ def findFeaturesSegment(ui): pool.close() pool.join() output = output.get() - match_segments = segment(ui.GetCubeName("match"), ui.GetInteger("NL")) - from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output) + if len(img_list) > 1: + from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output) + else: + # img_list = 1 + from_segments = output[0] + for seg, value in from_segments.items(): + og_value = value + from_segments[seg] = [og_value] + segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist] segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())] - + # re-generate footprints pool = ThreadPool(ceil(nthreads/len(segment_paths))) output = pool.map_async(footprintinit, segment_paths) @@ -332,28 +342,47 @@ def findFeaturesSegment(ui): output = output.get() log.debug(f"{output}") - image_sets = list(itertools.product(match_segments.values(), from_segments.values())) - + image_sets = list(itertools.product(match_segments.values(), from_segments.values())) + x = match_segments.values() + y = from_segments.values() + x,y = (x,y) if len(x) > len(y) else (y,x) + image_cartesian = list(itertools.product(x, y)) + image_sets = [] + for i in image_cartesian: + if i[0][0]["Segment"] >= i[1]["Segment"]: + image_sets.append(i) + # restructure things to be easier to manage job_dicts = [] for im in image_sets: match = im[0] from_images = im[1] + if not isinstance(from_images, list): + # from_images must be list type + from_images_list = [] + from_images_list.append(from_images) + from_images = from_images_list job_dicts.append({ "match" : match, "from" : from_images - }) - + }) + # get params as a dictionary params = ui.GetParams() - - pool = ThreadPool(ceil(nthreads/len(job_dicts))) - starmap_args = list(zip([params]*len(job_dicts), job_dicts)) - output = pool.starmap_async(generate_cnet, starmap_args) - pool.close() - pool.join() - output = output.get() - log.debug(f"output: {output}") + try: + pool = ThreadPool(ceil(nthreads/len(job_dicts))) + starmap_args = list(zip([params]*len(job_dicts), job_dicts)) + output = pool.starmap_async(generate_cnet, starmap_args) + pool.close() + pool.join() + output = output.get() + log.debug(f"output: {output}") + except Exception as err: + log.debug('generate_cnet error') + log.debug(' '.join(err.cmd)) + log.debug(err.stdout) + log.debug(err.stderr) + return err # merge the networks onets = [o["onet"] for o in output if isinstance(o, dict)] From 1e3b7a2d1442898e63089048406db616fbc3010c Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Mon, 29 Jul 2024 14:54:58 -0400 Subject: [PATCH 02/11] Added comment --- .../apps/findFeaturesSegment/findFeaturesSegment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 39b9bdc099..21aca7a753 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -322,6 +322,7 @@ def findFeaturesSegment(ui): pool.join() output = output.get() match_segments = segment(ui.GetCubeName("match"), ui.GetInteger("NL")) + if len(img_list) > 1: from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output) else: @@ -331,6 +332,7 @@ def findFeaturesSegment(ui): og_value = value from_segments[seg] = [og_value] + print("FROM_SEGMENTS = " + str(from_segments)) segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist] segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())] @@ -342,6 +344,7 @@ def findFeaturesSegment(ui): output = output.get() log.debug(f"{output}") + # Remove redundant overlapping pairs image_sets = list(itertools.product(match_segments.values(), from_segments.values())) x = match_segments.values() y = from_segments.values() From 5d7776bb28865f178b9bacf5bff21f85d90671ce Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Tue, 30 Jul 2024 13:45:32 -0400 Subject: [PATCH 03/11] Remove old image_sets --- .../apps/findFeaturesSegment/findFeaturesSegment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 21aca7a753..f6d0025f20 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -345,7 +345,6 @@ def findFeaturesSegment(ui): log.debug(f"{output}") # Remove redundant overlapping pairs - image_sets = list(itertools.product(match_segments.values(), from_segments.values())) x = match_segments.values() y = from_segments.values() x,y = (x,y) if len(x) > len(y) else (y,x) From a31e6693f5e877171082f689e7fdb9a03a506875 Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Tue, 30 Jul 2024 15:26:10 -0400 Subject: [PATCH 04/11] Add check for match_segment_n --- .../apps/findFeaturesSegment/findFeaturesSegment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index f6d0025f20..50c00b685d 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -134,7 +134,10 @@ def segment(img_path : Path, nlines : int = MAX_LEN): def generate_cnet(params, images): - match_segment_n = images["match"][0]["Segment"] + if isinstance(images["match"], list): + match_segment_n = images["match"][0]["Segment"] + else: + match_segment_n = images["match"]["Segment"] from_segment_n = images["from"][0]["Segment"] new_params = deepcopy(params) @@ -332,7 +335,6 @@ def findFeaturesSegment(ui): og_value = value from_segments[seg] = [og_value] - print("FROM_SEGMENTS = " + str(from_segments)) segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist] segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())] From 698afc6fe7cd5db87324c7942a259bfb619edaf1 Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Fri, 2 Aug 2024 16:47:39 -0400 Subject: [PATCH 05/11] Revert overlap changes --- .../findFeaturesSegment.py | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 50c00b685d..d87b1f85e7 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -135,9 +135,11 @@ def segment(img_path : Path, nlines : int = MAX_LEN): def generate_cnet(params, images): if isinstance(images["match"], list): - match_segment_n = images["match"][0]["Segment"] + images_match_n = images["match"][0] else: - match_segment_n = images["match"]["Segment"] + + images_match_n = images["match"] + match_segment_n = images_match_n["Segment"] from_segment_n = images["from"][0]["Segment"] new_params = deepcopy(params) @@ -148,7 +150,7 @@ def generate_cnet(params, images): # make sure none of these keys are still in the params new_params.pop("FROMLIST", None) new_params.pop("FROM", None) - new_params["MATCH"] = images["match"][0]["Path"] + new_params["MATCH"] = images_match_n["Path"] og_onet = Path(params["ONET"]) og_tolist = Path(params["TOLIST"]) @@ -200,7 +202,7 @@ def generate_cnet(params, images): log.debug(f"overlap stats: {ret.stdout}") # first, throw it out if there is no overlap whatsoever - is_pair_overlapping = any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) + is_pair_overlapping = not any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) if is_pair_overlapping: is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0)) @@ -210,9 +212,6 @@ def generate_cnet(params, images): else: # not overlapping is_overlapping.append(False) - if not any(is_overlapping): - log.info("No overlaps were found.") - # mask images from_images = list(compress(from_images, is_overlapping)) log.debug(f"From images overlapping Match: {from_images}") @@ -236,12 +235,11 @@ def generate_cnet(params, images): log.debug(' '.join(err.cmd)) log.debug(err.stdout) log.debug(err.stderr) - return "ERROR" segmented_net = cnet.from_isis(new_params["ONET"]) # starting sample in inclusive, so subtract 1 - segmented_net.loc[segmented_net.serialnumber == images["match"]["SN"], "line"] += images["match"]["StartingLine"]-1 + segmented_net.loc[segmented_net.serialnumber == images_match_n["SN"], "line"] += images_match_n["StartingLine"]-1 # offset the images for k, image in enumerate(images["from"]): @@ -373,20 +371,13 @@ def findFeaturesSegment(ui): # get params as a dictionary params = ui.GetParams() - try: - pool = ThreadPool(ceil(nthreads/len(job_dicts))) - starmap_args = list(zip([params]*len(job_dicts), job_dicts)) - output = pool.starmap_async(generate_cnet, starmap_args) - pool.close() - pool.join() - output = output.get() - log.debug(f"output: {output}") - except Exception as err: - log.debug('generate_cnet error') - log.debug(' '.join(err.cmd)) - log.debug(err.stdout) - log.debug(err.stderr) - return err + pool = ThreadPool(ceil(nthreads/len(job_dicts))) + starmap_args = list(zip([params]*len(job_dicts), job_dicts)) + output = pool.starmap_async(generate_cnet, starmap_args) + pool.close() + pool.join() + output = output.get() + log.debug(f"output: {output}") # merge the networks onets = [o["onet"] for o in output if isinstance(o, dict)] From 238cdb411d1f51af9402f07714237aa970ff6608 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 19 Aug 2024 08:45:40 -0700 Subject: [PATCH 06/11] updated script to improve times and fix bugs --- .../findFeaturesSegment.py | 179 ++++++++++-------- 1 file changed, 105 insertions(+), 74 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index d87b1f85e7..12f020aeb2 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -67,14 +67,21 @@ def read_cubelist(cube_list : Path): list list of files """ + files = [] + with open(cube_list) as c: content = c.read() content = content.strip() files = content.split("\n") - return files + + for i in range(len(files)): + files[i] = os.path.abspath(files[i]) + return files + -def segment(img_path : Path, nlines : int = MAX_LEN): + +def segment(img_path : Path, workdir : Path, nlines : int = MAX_LEN): """ Segments an image into multiple parts. @@ -97,19 +104,27 @@ def segment(img_path : Path, nlines : int = MAX_LEN): segment_metadata = {} try: - ret = kisis.segment(img_path, nl=nlines, overlap=0, pref__="$ISISROOT/IsisPreferences") + # change workdir so output goes there + oldpwd = os.getcwd() + + # create if it doesn't exist + workdir.mkdir(parents=True, exist_ok=True) + work_img = workdir / img_path.name + shutil.copyfile(img_path, work_img) + ret = kisis.segment(work_img, nl=nlines, overlap=0, pref__="$ISISROOT/IsisPreferences") + log.debug(f"{ret}") segment_metadata = pvl.loads(filter_progress(ret.stdout)) # comes nested in a "results" group, trim it off segment_metadata = [s[1] for s in segment_metadata] - glob_str = str(img_path.parent / img_path.stem) + ".segment*" + glob_str = str(workdir / img_path.stem) + ".segment*" log.debug(f"globbing segments: glob({glob_str})") segments = sorted(glob(glob_str)) - + log.debug(f"segments: {segments}") - + i = 0 for s, meta in zip(segments, segment_metadata): i += 1 @@ -118,7 +133,7 @@ def segment(img_path : Path, nlines : int = MAX_LEN): log.debug(f"sn: {serial}") meta["SN"] = serial meta["Segment"] = i - meta["Original"] = str(img_path) + meta["Original"] = str(work_img) seg_dict_keys = [f"seg{n}" for n in range(1,len(segments)+1)] segment_dict = dict(zip(seg_dict_keys, segment_metadata)) @@ -133,20 +148,22 @@ def segment(img_path : Path, nlines : int = MAX_LEN): def generate_cnet(params, images): - if isinstance(images["match"], list): images_match_n = images["match"][0] else: - images_match_n = images["match"] match_segment_n = images_match_n["Segment"] from_segment_n = images["from"][0]["Segment"] + print(images) + workdir = params["workdir"] + new_params = deepcopy(params) new_params.pop("NL") new_params.pop("MINAREA") new_params.pop("MINTHICKNESS") - + new_params.pop("workdir") + # make sure none of these keys are still in the params new_params.pop("FROMLIST", None) new_params.pop("FROM", None) @@ -163,7 +180,7 @@ def generate_cnet(params, images): match_stem = Path(new_params["MATCH"]).stem - fromlist_path = Path(og_tolist.parent / f"from_images_segment{from_segment_n}.lis") + fromlist_path = Path(workdir / f"from_images_segment{from_segment_n}.lis") from_stem = fromlist_path.stem if "debuglog" in new_params: @@ -173,49 +190,48 @@ def generate_cnet(params, images): new_params["NETWORKID"] = og_networkid + f"_{match_segment_n}_{from_stem}" new_params["POINTID"] = og_pointid + f"_{match_segment_n}_{from_stem}" - new_params["ONET"] = f"{og_onet.parent/og_onet.stem}_{match_segment_n}_{from_stem}.net" - new_params["TOLIST"] = f"{og_tolist.parent/og_tolist.stem}_{match_segment_n}_{from_stem}.lis" - + new_params["ONET"] = f"{workdir/og_onet.stem}_{match_segment_n}_{from_stem}.net" + new_params["TOLIST"] = f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}.lis" + log.debug(new_params) + overlapfromlist = workdir / f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}_overlap_fromlist.lis" + overlaptolist = workdir / f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}.overlaps" + # check for overlaps is_overlapping = [] - for image in from_images: - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = Path(tmpdir) - overlapfromlist = tmpdir / "fromlist.lis" - overlaptolist = tmpdir / "tolist.lis" - - kisis.fromlist.make([*from_images, new_params["MATCH"]], overlapfromlist) - - try: - kisis.findimageoverlaps(fromlist=overlapfromlist, overlaplist=overlaptolist) - except subprocess.CalledProcessError as err: - print('Had an ISIS error:') - print(' '.join(err.cmd)) - print(err.stdout) - print(err.stderr) - raise err - - ret = kisis.overlapstats(fromlist=overlapfromlist, overlaplist=overlaptolist, pref__="$ISISROOT/IsisPreferences") - stats = pvl.loads(filter_progress(ret.stdout)) - log.debug(f"overlap stats: {ret.stdout}") - - # first, throw it out if there is no overlap whatsoever - is_pair_overlapping = not any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats]) - - if is_pair_overlapping: - is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0)) - is_area_large_enough = stats["Results"]["AreaMinimum"] > float(params.get("MINAREA", 0)) - is_pair_overlapping = all([is_thick_enough, is_area_large_enough]) - is_overlapping.append(is_pair_overlapping) - else: # not overlapping - is_overlapping.append(False) - - # mask images - from_images = list(compress(from_images, is_overlapping)) - log.debug(f"From images overlapping Match: {from_images}") + all_images = [*from_images, new_params["MATCH"]] + print("ALL IMAGES: ", " ".join(all_images)) + kisis.fromlist.make(all_images, overlapfromlist) + try: + kisis.findimageoverlaps(fromlist=overlapfromlist, overlaplist=overlaptolist) + except subprocess.CalledProcessError as err: + print('Had an ISIS error:') + print(' '.join(err.cmd)) + print(err.stdout) + print(err.stderr) + return "No Overlaps From FindImageOverlaps" + + ret = kisis.overlapstats(fromlist=overlapfromlist, overlaplist=overlaptolist, pref__="$ISISROOT/IsisPreferences") + stats = pvl.loads(filter_progress(ret.stdout)) + log.debug(f"overlap stats: {ret.stdout}") + + # # first, throw it out if there is no overlap whatsoever + # has_overlap = not any([len(k[1].get("NoOverlap", "")) == new_params["MATCH"] for k in stats]) + + # if has_overlap: + # is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0)) + # is_area_large_enough = stats["Results"]["AreaMinimum"] > float(params.get("MINAREA", 0)) + # is_pair_overlapping = all([is_thick_enough, is_area_large_enough]) + # is_overlapping.append(has_overlap) + # else: # not overlapping + # is_overlapping.append(False) + + # # mask images + # from_images = list(compress(from_images, is_overlapping)) + log.debug(f"From images overlapping Match: {from_images}") + log.debug(f"Has overlaps list: {is_overlapping}") if from_images: log.debug(f"FROMLIST: {from_images}") @@ -236,21 +252,23 @@ def generate_cnet(params, images): log.debug(err.stdout) log.debug(err.stderr) - segmented_net = cnet.from_isis(new_params["ONET"]) + if os.path.exists(new_params["ONET"]): + segmented_net = cnet.from_isis(new_params["ONET"]) - # starting sample in inclusive, so subtract 1 - segmented_net.loc[segmented_net.serialnumber == images_match_n["SN"], "line"] += images_match_n["StartingLine"]-1 + # starting sample in inclusive, so subtract 1 + segmented_net.loc[segmented_net.serialnumber == images_match_n["SN"], "line"] += images_match_n["StartingLine"]-1 - # offset the images - for k, image in enumerate(images["from"]): - image_sn = image["SN"] + # offset the images + for k, image in enumerate(images["from"]): + image_sn = image["SN"] - # starting sample is inclusive, so subtract 1 - segmented_net.loc[segmented_net.serialnumber == image_sn, "line"] += starting_lines[k]-1 - cnet.to_isis(segmented_net, new_params["ONET"], targetname="moon") - - from_originals = [image["Original"] for image in images["from"]] - return {"onet": new_params["ONET"], "original_images": from_originals} + # starting sample is inclusive, so subtract 1 + segmented_net.loc[segmented_net.serialnumber == image_sn, "line"] += starting_lines[k]-1 + cnet.to_isis(segmented_net, new_params["ONET"], targetname="moon") + + from_originals = [image["Original"] for image in images["from"]] + return {"onet": new_params["ONET"], "original_images": from_originals} + return "No Overlap" else: return "No Overlap" @@ -303,6 +321,11 @@ def findFeaturesSegment(ui): else: log.basicConfig(level=log.INFO) + workdir = Path(tempfile.TemporaryDirectory().name) + + if ui.WasEntered("Workdir"): + workdir = Path(ui.GetFileName("Workdir")) + img_list = [] if ui.WasEntered("From"): img_list = [ui.GetFileName("From")] @@ -317,12 +340,16 @@ def findFeaturesSegment(ui): else: nthreads = int(multiprocessing.cpu_count()) - pool = ThreadPool(ceil(nthreads/len(img_list))) - output = pool.map_async(segment, img_list) + pool = ThreadPool(nthreads) + starmap_args = [] + for image in img_list: + print(image) + starmap_args.append([image, workdir, ui.GetInteger("NL")]) + output = pool.starmap_async(segment, starmap_args) pool.close() pool.join() output = output.get() - match_segments = segment(ui.GetCubeName("match"), ui.GetInteger("NL")) + match_segments = segment(os.path.abspath(ui.GetCubeName("match")), workdir, ui.GetInteger("NL")) if len(img_list) > 1: from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output) @@ -335,9 +362,9 @@ def findFeaturesSegment(ui): segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist] segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())] - + # re-generate footprints - pool = ThreadPool(ceil(nthreads/len(segment_paths))) + pool = ThreadPool(nthreads) output = pool.map_async(footprintinit, segment_paths) pool.close() pool.join() @@ -347,12 +374,12 @@ def findFeaturesSegment(ui): # Remove redundant overlapping pairs x = match_segments.values() y = from_segments.values() - x,y = (x,y) if len(x) > len(y) else (y,x) + # x,y = (x,y) if len(x) > len(y) else (y,x) image_cartesian = list(itertools.product(x, y)) - image_sets = [] - for i in image_cartesian: - if i[0][0]["Segment"] >= i[1]["Segment"]: - image_sets.append(i) + image_sets = image_cartesian + # for i in image_cartesian: + # if i[0][0]["Segment"] >= i[1]["Segment"]: + # image_sets.append(i) # restructure things to be easier to manage job_dicts = [] @@ -371,7 +398,11 @@ def findFeaturesSegment(ui): # get params as a dictionary params = ui.GetParams() - pool = ThreadPool(ceil(nthreads/len(job_dicts))) + params["workdir"] = workdir + + # findfeatures is already threaded, limit python threads by the maxthreads + # parameter, no maththreads_findfeatures * maxthreads_python <= maxthreads + pool = ThreadPool(nthreads/len(job_dicts)) starmap_args = list(zip([params]*len(job_dicts), job_dicts)) output = pool.starmap_async(generate_cnet, starmap_args) pool.close() @@ -410,9 +441,9 @@ def findFeaturesSegment(ui): # Dont merge shutil.copy(onets[0], ui.GetFileName("onet")) - log.info(f"COMPLETE, wrote { ui.GetFileName("onet")}") + log.info(f"COMPLETE, wrote: {ui.GetFileName("onet")}") + log.info(f"Intermediate files written to: {workdir}") if __name__ == "__main__": ui = astroset.init_application(sys.argv) findFeaturesSegment(ui) - From 3ef67ab2660a08699ae76160abb9cc93b16f2098 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 19 Aug 2024 08:48:17 -0700 Subject: [PATCH 07/11] added workdir param --- .../findFeaturesSegment/findFeaturesSegment.xml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml index 54ebc5f1cf..a35429c6c8 100644 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml @@ -139,6 +139,17 @@ *.lis + + filename + output + Directory to output intermediate files to + + This directory is where any intermediate and temp files are saved to. + If this is set to None (default), these files go into the temp directory. + + None + + integer output @@ -1432,4 +1443,4 @@ End - + \ No newline at end of file From 3ec2eb7857cfacdcf419fe28a5ed2cd911a3ea22 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 19 Aug 2024 17:41:28 -0700 Subject: [PATCH 08/11] made sure file are cleaned up when process fails --- .../findFeaturesSegment.py | 54 +++++++++++-------- .../findFeaturesSegment.xml | 4 +- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 12f020aeb2..38ba21ad08 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -156,13 +156,14 @@ def generate_cnet(params, images): from_segment_n = images["from"][0]["Segment"] print(images) - workdir = params["workdir"] + workdir = Path(params["WORKDIR"]) new_params = deepcopy(params) new_params.pop("NL") new_params.pop("MINAREA") new_params.pop("MINTHICKNESS") - new_params.pop("workdir") + new_params.pop("WORKDIR") + # make sure none of these keys are still in the params new_params.pop("FROMLIST", None) @@ -190,13 +191,13 @@ def generate_cnet(params, images): new_params["NETWORKID"] = og_networkid + f"_{match_segment_n}_{from_stem}" new_params["POINTID"] = og_pointid + f"_{match_segment_n}_{from_stem}" - new_params["ONET"] = f"{workdir/og_onet.stem}_{match_segment_n}_{from_stem}.net" - new_params["TOLIST"] = f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}.lis" + new_params["ONET"] = f"{og_onet.stem}_{match_segment_n}_{from_stem}.net" + new_params["TOLIST"] = f"{og_tolist.stem}_{match_segment_n}_{from_stem}.lis" log.debug(new_params) - overlapfromlist = workdir / f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}_overlap_fromlist.lis" - overlaptolist = workdir / f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}.overlaps" + overlapfromlist = workdir / f"{workdir / og_tolist.stem}_{match_segment_n}_{from_stem}_overlap_fromlist.lis" + overlaptolist = workdir / f"{og_tolist.stem}_{match_segment_n}_{from_stem}.overlaps" # check for overlaps is_overlapping = [] @@ -207,7 +208,7 @@ def generate_cnet(params, images): try: kisis.findimageoverlaps(fromlist=overlapfromlist, overlaplist=overlaptolist) except subprocess.CalledProcessError as err: - print('Had an ISIS error:') + print('Find Image Overlaps Had an ISIS error:') print(' '.join(err.cmd)) print(err.stdout) print(err.stderr) @@ -242,12 +243,14 @@ def generate_cnet(params, images): else: log.debug(f"{fromlist_path} already exists") new_params["FROMLIST"] = str(fromlist_path) + + log.debug(f"Running FindFeatures with Params: {new_params}") try: ret = kisis.findfeatures(**new_params) log.debug(f"returned: {ret}") except subprocess.CalledProcessError as err: - log.debug('Had an ISIS error:') + log.debug('Find Features Had an ISIS error:') log.debug(' '.join(err.cmd)) log.debug(err.stdout) log.debug(err.stderr) @@ -296,7 +299,7 @@ def merge(d1, d2, k): return v1+v2 -def findFeaturesSegment(ui): +def findFeaturesSegment(ui, workdir): """ findFeaturesSegment Calls FindFeatures on segmented images @@ -321,11 +324,6 @@ def findFeaturesSegment(ui): else: log.basicConfig(level=log.INFO) - workdir = Path(tempfile.TemporaryDirectory().name) - - if ui.WasEntered("Workdir"): - workdir = Path(ui.GetFileName("Workdir")) - img_list = [] if ui.WasEntered("From"): img_list = [ui.GetFileName("From")] @@ -398,11 +396,12 @@ def findFeaturesSegment(ui): # get params as a dictionary params = ui.GetParams() - params["workdir"] = workdir + if is_workdir_temp: + params["WORKDIR"] = workdir # findfeatures is already threaded, limit python threads by the maxthreads # parameter, no maththreads_findfeatures * maxthreads_python <= maxthreads - pool = ThreadPool(nthreads/len(job_dicts)) + pool = ThreadPool(int(nthreads/len(job_dicts))) starmap_args = list(zip([params]*len(job_dicts), job_dicts)) output = pool.starmap_async(generate_cnet, starmap_args) pool.close() @@ -441,9 +440,22 @@ def findFeaturesSegment(ui): # Dont merge shutil.copy(onets[0], ui.GetFileName("onet")) - log.info(f"COMPLETE, wrote: {ui.GetFileName("onet")}") - log.info(f"Intermediate files written to: {workdir}") - -if __name__ == "__main__": +if __name__ == "__main__": ui = astroset.init_application(sys.argv) - findFeaturesSegment(ui) + is_workdir_temp = not ui.WasEntered("WorkDir") + workdir = Path(tempfile.TemporaryDirectory().name) + if ui.WasEntered("Workdir"): + workdir = Path(ui.GetFileName("Workdir")) + + try: + findFeaturesSegment(ui, workdir) + except Exception as e: + if is_workdir_temp: + shutil.rmtree(workdir) + raise e + + log.info(f"COMPLETE, wrote: {ui.GetFileName("onet")}") + if is_workdir_temp: + shutil.rmtree(workdir) + else: + log.info(f"Intermediate files written to: {workdir}") \ No newline at end of file diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml index a35429c6c8..4cead8407d 100644 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml @@ -145,7 +145,9 @@ Directory to output intermediate files to This directory is where any intermediate and temp files are saved to. - If this is set to None (default), these files go into the temp directory. + If this is set to None (default), these files go into the temp directory + which is deleted when the program is terminated. Set this if you want to debug + a network. None From df3af20110112381faca16d138a5a17ca2af3e95 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Tue, 20 Aug 2024 14:57:20 -0700 Subject: [PATCH 09/11] removed bug --- .../apps/findFeaturesSegment/findFeaturesSegment.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 38ba21ad08..66b296dbab 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -196,7 +196,7 @@ def generate_cnet(params, images): log.debug(new_params) - overlapfromlist = workdir / f"{workdir / og_tolist.stem}_{match_segment_n}_{from_stem}_overlap_fromlist.lis" + overlapfromlist = workdir / f"{og_tolist.stem}_{match_segment_n}_{from_stem}_overlap_fromlist.lis" overlaptolist = workdir / f"{og_tolist.stem}_{match_segment_n}_{from_stem}.overlaps" # check for overlaps @@ -341,7 +341,6 @@ def findFeaturesSegment(ui, workdir): pool = ThreadPool(nthreads) starmap_args = [] for image in img_list: - print(image) starmap_args.append([image, workdir, ui.GetInteger("NL")]) output = pool.starmap_async(segment, starmap_args) pool.close() @@ -427,7 +426,7 @@ def findFeaturesSegment(ui, workdir): log.debug(f"merged images: {final_images}") kisis.fromlist.make(final_images, Path(ui.GetFileName("tolist"))) - + if len(onets) > 1: try: kisis.cnetmerge(clist = onet_list, onet=ui.GetFileName("onet"), networkid=ui.GetAsString("networkid"), description=f"{ui.GetString('description')}") @@ -454,8 +453,8 @@ def findFeaturesSegment(ui, workdir): shutil.rmtree(workdir) raise e - log.info(f"COMPLETE, wrote: {ui.GetFileName("onet")}") + # log.info(f"COMPLETE, wrote: {ui.GetFileName("onet")}") if is_workdir_temp: shutil.rmtree(workdir) else: - log.info(f"Intermediate files written to: {workdir}") \ No newline at end of file + log.info("Intermediate files written to") From 906ee48a8e4fd8918afc06110ef556fd6b90ccab Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 12 Sep 2024 12:55:55 -0700 Subject: [PATCH 10/11] fixes to tolist --- .../findFeaturesSegment/findFeaturesSegment.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py index 66b296dbab..f74147d490 100755 --- a/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py +++ b/isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py @@ -112,6 +112,7 @@ def segment(img_path : Path, workdir : Path, nlines : int = MAX_LEN): work_img = workdir / img_path.name shutil.copyfile(img_path, work_img) ret = kisis.segment(work_img, nl=nlines, overlap=0, pref__="$ISISROOT/IsisPreferences") + os.remove(work_img) log.debug(f"{ret}") segment_metadata = pvl.loads(filter_progress(ret.stdout)) @@ -133,7 +134,7 @@ def segment(img_path : Path, workdir : Path, nlines : int = MAX_LEN): log.debug(f"sn: {serial}") meta["SN"] = serial meta["Segment"] = i - meta["Original"] = str(work_img) + meta["Original"] = str(img_path) seg_dict_keys = [f"seg{n}" for n in range(1,len(segments)+1)] segment_dict = dict(zip(seg_dict_keys, segment_metadata)) @@ -221,14 +222,6 @@ def generate_cnet(params, images): # # first, throw it out if there is no overlap whatsoever # has_overlap = not any([len(k[1].get("NoOverlap", "")) == new_params["MATCH"] for k in stats]) - # if has_overlap: - # is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0)) - # is_area_large_enough = stats["Results"]["AreaMinimum"] > float(params.get("MINAREA", 0)) - # is_pair_overlapping = all([is_thick_enough, is_area_large_enough]) - # is_overlapping.append(has_overlap) - # else: # not overlapping - # is_overlapping.append(False) - # # mask images # from_images = list(compress(from_images, is_overlapping)) log.debug(f"From images overlapping Match: {from_images}") @@ -422,7 +415,7 @@ def findFeaturesSegment(ui, workdir): tolists = [set(o["original_images"]) for o in output if isinstance(o, dict)] final_images = set.union(*tolists) - final_images.add(ui.GetCubeName("match")) + final_images.add(os.path.abspath(ui.GetCubeName("match"))) log.debug(f"merged images: {final_images}") kisis.fromlist.make(final_images, Path(ui.GetFileName("tolist"))) @@ -456,5 +449,6 @@ def findFeaturesSegment(ui, workdir): # log.info(f"COMPLETE, wrote: {ui.GetFileName("onet")}") if is_workdir_temp: shutil.rmtree(workdir) + log.info("Complete") else: - log.info("Intermediate files written to") + log.info("Intermediate files written to %s", workdir) From bb6576976d49d79d13cf8ddec9d90c9abefefc9c Mon Sep 17 00:00:00 2001 From: Christine Kim Date: Mon, 4 Nov 2024 13:47:47 -0500 Subject: [PATCH 11/11] Trigger build