Skip to content

Commit

Permalink
improved times and fixed bugs
Browse files Browse the repository at this point in the history
  • Loading branch information
@Kelvinrr
Kelvinrr committed Aug 19, 2024
1 parent fa78c06 commit 03bd425
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 73 deletions.
178 changes: 105 additions & 73 deletions isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,21 @@ def read_cubelist(cube_list : Path):
list
list of files
"""
files = []

with open(cube_list) as c:
content = c.read()
content = content.strip()
files = content.split("\n")
return files

for i in range(len(files)):
files[i] = os.path.abspath(files[i])
return files



def segment(img_path : Path, nlines : int = MAX_LEN):

def segment(img_path : Path, workdir : Path, nlines : int = MAX_LEN):
"""
Segments an image into multiple parts.
Expand All @@ -97,19 +104,27 @@ def segment(img_path : Path, nlines : int = MAX_LEN):

segment_metadata = {}
try:
ret = kisis.segment(img_path, nl=nlines, overlap=0, pref__="$ISISROOT/IsisPreferences")
# change workdir so output goes there
oldpwd = os.getcwd()

# create if it doesn't exist
workdir.mkdir(parents=True, exist_ok=True)
work_img = workdir / img_path.name
shutil.copyfile(img_path, work_img)
ret = kisis.segment(work_img, nl=nlines, overlap=0, pref__="$ISISROOT/IsisPreferences")

log.debug(f"{ret}")
segment_metadata = pvl.loads(filter_progress(ret.stdout))

# comes nested in a "results" group, trim it off
segment_metadata = [s[1] for s in segment_metadata]

glob_str = str(img_path.parent / img_path.stem) + ".segment*"
glob_str = str(workdir / img_path.stem) + ".segment*"
log.debug(f"globbing segments: glob({glob_str})")
segments = sorted(glob(glob_str))

log.debug(f"segments: {segments}")

i = 0
for s, meta in zip(segments, segment_metadata):
i += 1
Expand All @@ -118,7 +133,7 @@ def segment(img_path : Path, nlines : int = MAX_LEN):
log.debug(f"sn: {serial}")
meta["SN"] = serial
meta["Segment"] = i
meta["Original"] = str(img_path)
meta["Original"] = str(work_img)

seg_dict_keys = [f"seg{n}" for n in range(1,len(segments)+1)]
segment_dict = dict(zip(seg_dict_keys, segment_metadata))
Expand All @@ -133,20 +148,22 @@ def segment(img_path : Path, nlines : int = MAX_LEN):


def generate_cnet(params, images):

if isinstance(images["match"], list):
images_match_n = images["match"][0]
else:

images_match_n = images["match"]
match_segment_n = images_match_n["Segment"]
from_segment_n = images["from"][0]["Segment"]

print(images)
workdir = params["workdir"]

new_params = deepcopy(params)
new_params.pop("NL")
new_params.pop("MINAREA")
new_params.pop("MINTHICKNESS")

new_params.pop("workdir")

# make sure none of these keys are still in the params
new_params.pop("FROMLIST", None)
new_params.pop("FROM", None)
Expand All @@ -163,7 +180,7 @@ def generate_cnet(params, images):

match_stem = Path(new_params["MATCH"]).stem

fromlist_path = Path(og_tolist.parent / f"from_images_segment{from_segment_n}.lis")
fromlist_path = Path(workdir / f"from_images_segment{from_segment_n}.lis")
from_stem = fromlist_path.stem

if "debuglog" in new_params:
Expand All @@ -173,49 +190,48 @@ def generate_cnet(params, images):

new_params["NETWORKID"] = og_networkid + f"_{match_segment_n}_{from_stem}"
new_params["POINTID"] = og_pointid + f"_{match_segment_n}_{from_stem}"
new_params["ONET"] = f"{og_onet.parent/og_onet.stem}_{match_segment_n}_{from_stem}.net"
new_params["TOLIST"] = f"{og_tolist.parent/og_tolist.stem}_{match_segment_n}_{from_stem}.lis"

new_params["ONET"] = f"{workdir/og_onet.stem}_{match_segment_n}_{from_stem}.net"
new_params["TOLIST"] = f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}.lis"
log.debug(new_params)

overlapfromlist = workdir / f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}_overlap_fromlist.lis"
overlaptolist = workdir / f"{workdir/og_tolist.stem}_{match_segment_n}_{from_stem}.overlaps"

# check for overlaps
is_overlapping = []
for image in from_images:
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
overlapfromlist = tmpdir / "fromlist.lis"
overlaptolist = tmpdir / "tolist.lis"

kisis.fromlist.make([*from_images, new_params["MATCH"]], overlapfromlist)

try:
kisis.findimageoverlaps(fromlist=overlapfromlist, overlaplist=overlaptolist)
except subprocess.CalledProcessError as err:
print('Had an ISIS error:')
print(' '.join(err.cmd))
print(err.stdout)
print(err.stderr)
raise err

ret = kisis.overlapstats(fromlist=overlapfromlist, overlaplist=overlaptolist, pref__="$ISISROOT/IsisPreferences")
stats = pvl.loads(filter_progress(ret.stdout))
log.debug(f"overlap stats: {ret.stdout}")

# first, throw it out if there is no overlap whatsoever
is_pair_overlapping = not any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats])

if is_pair_overlapping:
is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0))
is_area_large_enough = stats["Results"]["AreaMinimum"] > float(params.get("MINAREA", 0))
is_pair_overlapping = all([is_thick_enough, is_area_large_enough])
is_overlapping.append(is_pair_overlapping)
else: # not overlapping
is_overlapping.append(False)

# mask images
from_images = list(compress(from_images, is_overlapping))
log.debug(f"From images overlapping Match: {from_images}")
all_images = [*from_images, new_params["MATCH"]]
print("ALL IMAGES: ", " ".join(all_images))
kisis.fromlist.make(all_images, overlapfromlist)

try:
kisis.findimageoverlaps(fromlist=overlapfromlist, overlaplist=overlaptolist)
except subprocess.CalledProcessError as err:
print('Had an ISIS error:')
print(' '.join(err.cmd))
print(err.stdout)
print(err.stderr)
return "No Overlaps From FindImageOverlaps"

ret = kisis.overlapstats(fromlist=overlapfromlist, overlaplist=overlaptolist, pref__="$ISISROOT/IsisPreferences")
stats = pvl.loads(filter_progress(ret.stdout))
log.debug(f"overlap stats: {ret.stdout}")

# # first, throw it out if there is no overlap whatsoever
# has_overlap = not any([len(k[1].get("NoOverlap", "")) == new_params["MATCH"] for k in stats])

# if has_overlap:
# is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0))
# is_area_large_enough = stats["Results"]["AreaMinimum"] > float(params.get("MINAREA", 0))
# is_pair_overlapping = all([is_thick_enough, is_area_large_enough])
# is_overlapping.append(has_overlap)
# else: # not overlapping
# is_overlapping.append(False)

# # mask images
# from_images = list(compress(from_images, is_overlapping))
log.debug(f"From images overlapping Match: {from_images}")
log.debug(f"Has overlaps list: {is_overlapping}")

if from_images:
log.debug(f"FROMLIST: {from_images}")
Expand All @@ -236,21 +252,23 @@ def generate_cnet(params, images):
log.debug(err.stdout)
log.debug(err.stderr)

segmented_net = cnet.from_isis(new_params["ONET"])
if os.path.exists(new_params["ONET"]):
segmented_net = cnet.from_isis(new_params["ONET"])

# starting sample in inclusive, so subtract 1
segmented_net.loc[segmented_net.serialnumber == images_match_n["SN"], "line"] += images_match_n["StartingLine"]-1
# starting sample in inclusive, so subtract 1
segmented_net.loc[segmented_net.serialnumber == images_match_n["SN"], "line"] += images_match_n["StartingLine"]-1

# offset the images
for k, image in enumerate(images["from"]):
image_sn = image["SN"]
# offset the images
for k, image in enumerate(images["from"]):
image_sn = image["SN"]

# starting sample is inclusive, so subtract 1
segmented_net.loc[segmented_net.serialnumber == image_sn, "line"] += starting_lines[k]-1
cnet.to_isis(segmented_net, new_params["ONET"], targetname="moon")

from_originals = [image["Original"] for image in images["from"]]
return {"onet": new_params["ONET"], "original_images": from_originals}
# starting sample is inclusive, so subtract 1
segmented_net.loc[segmented_net.serialnumber == image_sn, "line"] += starting_lines[k]-1
cnet.to_isis(segmented_net, new_params["ONET"], targetname="moon")

from_originals = [image["Original"] for image in images["from"]]
return {"onet": new_params["ONET"], "original_images": from_originals}
return "No Overlap"
else:
return "No Overlap"

Expand Down Expand Up @@ -303,6 +321,11 @@ def findFeaturesSegment(ui):
else:
log.basicConfig(level=log.INFO)

workdir = Path(tempfile.TemporaryDirectory().name)

if ui.WasEntered("Workdir"):
workdir = Path(ui.GetFileName("Workdir"))

img_list = []
if ui.WasEntered("From"):
img_list = [ui.GetFileName("From")]
Expand All @@ -317,12 +340,16 @@ def findFeaturesSegment(ui):
else:
nthreads = int(multiprocessing.cpu_count())

pool = ThreadPool(ceil(nthreads/len(img_list)))
output = pool.map_async(segment, img_list)
pool = ThreadPool(nthreads)
starmap_args = []
for image in img_list:
print(image)
starmap_args.append([image, workdir, ui.GetInteger("NL")])
output = pool.starmap_async(segment, starmap_args)
pool.close()
pool.join()
output = output.get()
match_segments = segment(ui.GetCubeName("match"), ui.GetInteger("NL"))
match_segments = segment(os.path.abspath(ui.GetCubeName("match")), workdir, ui.GetInteger("NL"))

if len(img_list) > 1:
from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output)
Expand All @@ -335,9 +362,9 @@ def findFeaturesSegment(ui):

segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist]
segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())]

# re-generate footprints
pool = ThreadPool(ceil(nthreads/len(segment_paths)))
pool = ThreadPool(nthreads)
output = pool.map_async(footprintinit, segment_paths)
pool.close()
pool.join()
Expand All @@ -347,12 +374,12 @@ def findFeaturesSegment(ui):
# Remove redundant overlapping pairs
x = match_segments.values()
y = from_segments.values()
x,y = (x,y) if len(x) > len(y) else (y,x)
# x,y = (x,y) if len(x) > len(y) else (y,x)
image_cartesian = list(itertools.product(x, y))
image_sets = []
for i in image_cartesian:
if i[0][0]["Segment"] >= i[1]["Segment"]:
image_sets.append(i)
image_sets = image_cartesian
# for i in image_cartesian:
# if i[0][0]["Segment"] >= i[1]["Segment"]:
# image_sets.append(i)

# restructure things to be easier to manage
job_dicts = []
Expand All @@ -371,7 +398,11 @@ def findFeaturesSegment(ui):

# get params as a dictionary
params = ui.GetParams()
pool = ThreadPool(ceil(nthreads/len(job_dicts)))
params["workdir"] = workdir

# findfeatures is already threaded, limit python threads by the maxthreads
# parameter, no maththreads_findfeatures * maxthreads_python <= maxthreads
pool = ThreadPool(nthreads/len(job_dicts))
starmap_args = list(zip([params]*len(job_dicts), job_dicts))
output = pool.starmap_async(generate_cnet, starmap_args)
pool.close()
Expand Down Expand Up @@ -410,7 +441,8 @@ def findFeaturesSegment(ui):
# Dont merge
shutil.copy(onets[0], ui.GetFileName("onet"))

log.info(f"COMPLETE, wrote { ui.GetFileName("onet")}")
log.info(f"COMPLETE, wrote: {ui.GetFileName("onet")}")
log.info(f"Intermediate files written to: {workdir}")

if __name__ == "__main__":
ui = astroset.init_application(sys.argv)
Expand Down
11 changes: 11 additions & 0 deletions isis/python_bindings/apps/findFeaturesSegment/findFeaturesSegment.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,17 @@
<filter>*.lis</filter>
</parameter>

<parameter name="WORKDIR">
<type>filename</type>
<fileMode>output</fileMode>
<brief>Directory to output intermediate files to</brief>
<description>
This directory is where any intermediate and temp files are saved to.
If this is set to None (default), these files go into the temp directory.
</description>
<internalDefault>None</internalDefault>
</parameter>

<parameter name="NL">
<type>integer</type>
<fileMode>output</fileMode>
Expand Down

0 comments on commit 03bd425

Please sign in to comment.