Skip to content

Commit

Permalink
Adds files to files entry
Browse files Browse the repository at this point in the history
  • Loading branch information
tainagdcoleman committed Sep 12, 2024
1 parent c576b8e commit 3e761e7
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 6 deletions.
9 changes: 9 additions & 0 deletions wfcommons/common/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,18 @@ def write_json(self, json_file_path: Optional[pathlib.Path] = None) -> None:
machines_list.append(machine.name)
workflow_machines.append(machine.as_dict())

# add files to the workflow json object (input and output)
for file in task.input_files:
files.append(file.as_dict())
for file in task.output_files:
files.append(file.as_dict())

if workflow_machines:
workflow_json["workflow"]["execution"]["machines"] = workflow_machines

if files and len(files) > 0:
workflow_json["workflow"]["specification"]["files"] = files

# write to file
if not json_file_path:
json_file_path = pathlib.Path(f"{self.name.lower()}.json")
Expand Down
11 changes: 6 additions & 5 deletions wfcommons/wfgen/abstract_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,6 @@ def _generate_task_files(self, task: Task) -> List[File]:
output_files_list = self._generate_files(task.task_id, task_recipe['output'], FileLink.OUTPUT)
task.output_files = self.tasks_files[task.task_id]



# obtain input files from parents
input_files = []

Expand All @@ -221,13 +219,14 @@ def _generate_task_files(self, task: Task) -> List[File]:
if input_file not in self.tasks_files_names[task.task_id]:
self.tasks_files[task.task_id].append(File(name=input_file,
link=FileLink.INPUT,
size=input_file.size))
size=input_file.size))
self.tasks_files_names[task.task_id].append(input_file)

# generate additional input files
self._generate_files(task.task_id, task_recipe['input'], FileLink.INPUT)
task.input_files = [ifile for ifile in self.tasks_files[task.task_id] if ifile.link == FileLink.INPUT]


return output_files_list

def _generate_files(self, task_id: str, recipe: Dict[str, Any], link: FileLink) -> List[File]:
Expand All @@ -248,9 +247,9 @@ def _generate_files(self, task_id: str, recipe: Dict[str, Any], link: FileLink)
extension_list: List[str] = []
for f in self.tasks_files[task_id]:
if f.link == link:
extension_list.append(path.splitext(f.file_id)[1] if '.' in f.file_id else f.file_id)
files_list.append(f)
extension_list.append(path.splitext(f.name)[1] if '.' in f.name else f.name)


for extension in recipe:
if extension not in extension_list:
file = self._generate_file(extension, recipe, link)
Expand Down Expand Up @@ -278,10 +277,12 @@ def _generate_file(self, extension: str, recipe: Dict[str, Any], link: FileLink)
else self.output_file_size_factor) * generate_rvs(recipe[extension]['distribution'],
recipe[extension]['min'],
recipe[extension]['max']))

return File(file_id=str(uuid.uuid4()) + extension,
link=link,
size=size)


def _get_files_by_task_and_link(self, task_id: str, link: FileLink) -> List[File]:
"""
Get the list of files for a task ID and link type.
Expand Down
2 changes: 1 addition & 1 deletion wfcommons/wfinstances/instance_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def build_summary(self,
# For each input_file and output_file, append the file size to the dictionary
for infile in task.input_files:
extension: str = path.splitext(infile.file_id)[1] if '.' in infile.file_id else infile.file_id
# print(f"file {infile.file_id} extension: {extension}")

if extension[1:].isnumeric():
extension = path.splitext(infile.file_id.replace(extension, ''))[1]

Expand Down

0 comments on commit 3e761e7

Please sign in to comment.