Skip to content

Commit

Permalink
further simplification of map_catalogs
Browse files Browse the repository at this point in the history
  • Loading branch information
ntessore committed Dec 10, 2023
1 parent c9bb224 commit 90c9f57
Showing 1 changed file with 10 additions and 22 deletions.
32 changes: 10 additions & 22 deletions heracles/maps.py
Original file line number Diff line number Diff line change
Expand Up @@ -790,26 +790,15 @@ def map_catalogs(
out = TocDict()

# collect groups of items to go through
# groups are tuples of (name, items)
# items are tuples of (key, mapper, catalog, name)
# items are tuples of (key, mapper, catalog)
groups = [
[((k, i), mapper, catalog) for k, mapper in maps.items()]
for i, catalog in catalogs.items()
]

# flatten groups for parallel processing
if parallel:
# one group of all items
groups = [
(
None,
[
((k, i), mapper, catalog)
for i, catalog in catalogs.items()
for k, mapper in maps.items()
],
),
]
else:
# one group for each catalogue
groups = [
(i, [((k, i), mapper, catalog) for k, mapper in maps.items()])
for i, catalog in catalogs.items()
]
groups = [sum(groups, [])]

# display a progress bar if asked to
if progress:
Expand All @@ -818,12 +807,11 @@ def map_catalogs(
# create the progress bar
# add the main task -- this must be the first task
_progress = Progress()
total_items = sum(len(items) for _, items in groups)
_progress.add_task("mapping", total=total_items)
_progress.add_task("mapping", total=sum(map(len, groups)))
_progress.start()

# process all groups of mappers and catalogues
for name, items in groups:
for items in groups:
# mappers return coroutines, which are ran concurrently
keys, coros = [], []
for key, mapper, catalog in items:
Expand Down

0 comments on commit 90c9f57

Please sign in to comment.