Skip to content

Commit

Permalink
Fill gaps with -1 so that the rank doesnt change when loading RR
Browse files Browse the repository at this point in the history
  • Loading branch information
ferdonline committed Oct 29, 2024
1 parent e8bdb88 commit 9d2431e
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions tools/rebalance-corenrn-data.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,16 +95,16 @@ def write_dat_file(buckets, infos: dict, output_file="rebalanced-files.dat"):
logging.info("Writing out data from %d buckets to file: %s", len(buckets), output_file)

# CoreNeuron does RoundRobin - we need to transpose the entries
zipped_entries = itertools.zip_longest(*buckets)
# When a sequence finishes use "-1" (to keep in sync)
zipped_entries = itertools.zip_longest(*buckets, fillvalue="-1\n")

with open(output_file, "w") as out:
print(infos["version"], file=out)
print(infos["n_files"], file=out)

for entries in zipped_entries:
for entry in entries:
if entry is not None:
out.write(entry)
out.write(entry)


def get_entry_size(base_dir, dat_entry):
Expand Down

0 comments on commit 9d2431e

Please sign in to comment.