Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

findFeaturesSegment.py script fixes #5565

Merged
merged 11 commits into from
Nov 4, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ def segment(img_path : Path, nlines : int = MAX_LEN):

def generate_cnet(params, images):

match_segment_n = images["match"]["Segment"]
match_segment_n = images["match"][0]["Segment"]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this needed every time? Or only sometimes? I find it odd that it seemed to be working in some cases. Sometimes you have to check both cases.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm very true, I'll have to add some sort of check then. Good catch.

from_segment_n = images["from"][0]["Segment"]

new_params = deepcopy(params)
Expand All @@ -145,7 +145,7 @@ def generate_cnet(params, images):
# make sure none of these keys are still in the params
new_params.pop("FROMLIST", None)
new_params.pop("FROM", None)
new_params["MATCH"] = images["match"]["Path"]
new_params["MATCH"] = images["match"][0]["Path"]

og_onet = Path(params["ONET"])
og_tolist = Path(params["TOLIST"])
Expand Down Expand Up @@ -175,7 +175,7 @@ def generate_cnet(params, images):

# check for overlaps
is_overlapping = []
for image in from_images:
for image in from_images:
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
overlapfromlist = tmpdir / "fromlist.lis"
Expand All @@ -197,7 +197,7 @@ def generate_cnet(params, images):
log.debug(f"overlap stats: {ret.stdout}")

# first, throw it out if there is no overlap whatsoever
is_pair_overlapping = not any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats])
is_pair_overlapping = any([k[1].get("NoOverlap", "") == new_params["MATCH"] for k in stats])
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bruh that's a pretty bad bug you caught there


if is_pair_overlapping:
is_thick_enough = stats["Results"]["ThicknessMinimum"] > float(params.get("MINTHICKNESS", 0))
Expand All @@ -207,6 +207,9 @@ def generate_cnet(params, images):
else: # not overlapping
is_overlapping.append(False)

if not any(is_overlapping):
log.info("No overlaps were found.")

# mask images
from_images = list(compress(from_images, is_overlapping))
log.debug(f"From images overlapping Match: {from_images}")
Expand Down Expand Up @@ -318,12 +321,21 @@ def findFeaturesSegment(ui):
pool.close()
pool.join()
output = output.get()

match_segments = segment(ui.GetCubeName("match"), ui.GetInteger("NL"))
from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output)

if len(img_list) > 1:
from_segments = reduce(lambda d1,d2: {k: merge(d1, d2, k) for k in set(d1)|set(d2)}, output)
else:
# img_list = 1
from_segments = output[0]
for seg, value in from_segments.items():
og_value = value
from_segments[seg] = [og_value]

print("FROM_SEGMENTS = " + str(from_segments))
segment_paths = [s["Path"] for sublist in list(from_segments.values()) for s in sublist]
segment_paths = segment_paths + [s["Path"] for s in list(match_segments.values())]

# re-generate footprints
pool = ThreadPool(ceil(nthreads/len(segment_paths)))
output = pool.map_async(footprintinit, segment_paths)
Expand All @@ -332,28 +344,48 @@ def findFeaturesSegment(ui):
output = output.get()
log.debug(f"{output}")

image_sets = list(itertools.product(match_segments.values(), from_segments.values()))

# Remove redundant overlapping pairs
image_sets = list(itertools.product(match_segments.values(), from_segments.values()))
x = match_segments.values()
y = from_segments.values()
x,y = (x,y) if len(x) > len(y) else (y,x)
image_cartesian = list(itertools.product(x, y))
image_sets = []
for i in image_cartesian:
if i[0][0]["Segment"] >= i[1]["Segment"]:
image_sets.append(i)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How was this tested? There is some difference between the example I gave in the issue and how it would be implemented here. Just want to make sure it is working.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So I added some print statements with the old image_sets and image_sets after this for loop for the two areas Lynn mentioned in here (I can point you to the data areas too) and the results are attached as image_set_sort_before.txt and image_set_sort_after.txt. It looks like both examples show the removal of a redundant pair, going from (1,1), (1,2), (2,1), (2,2) to (1,1), (2,1), (2,2).


# restructure things to be easier to manage
job_dicts = []
for im in image_sets:
match = im[0]
from_images = im[1]
if not isinstance(from_images, list):
# from_images must be list type
from_images_list = []
from_images_list.append(from_images)
from_images = from_images_list
job_dicts.append({
"match" : match,
"from" : from_images
})
})

# get params as a dictionary
params = ui.GetParams()

pool = ThreadPool(ceil(nthreads/len(job_dicts)))
starmap_args = list(zip([params]*len(job_dicts), job_dicts))
output = pool.starmap_async(generate_cnet, starmap_args)
pool.close()
pool.join()
output = output.get()
log.debug(f"output: {output}")
try:
pool = ThreadPool(ceil(nthreads/len(job_dicts)))
starmap_args = list(zip([params]*len(job_dicts), job_dicts))
output = pool.starmap_async(generate_cnet, starmap_args)
pool.close()
pool.join()
output = output.get()
log.debug(f"output: {output}")
except Exception as err:
log.debug('generate_cnet error')
log.debug(' '.join(err.cmd))
log.debug(err.stdout)
log.debug(err.stderr)
return err

# merge the networks
onets = [o["onet"] for o in output if isinstance(o, dict)]
Expand Down