forked from nextflow-io/nextflow
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add inspect command (nextflow-io#4069)
This commit introduces a new nextflow command named `inspect`. The inspect command allows resolving a pipeline script or project reporting all container images used by the pipeline execution. The main advantage of this command over the existing `config` command is that it's able to resolve container names defined "dynamically" or Wave containers that are only determined at execution time. The command option `-concretise` when used along with the Wave freeze option allows building ahead all the container images required by the pipeline execution. Signed-off-by: Ben Sherman <bentshermann@gmail.com> Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
- Loading branch information
1 parent
ccfc769
commit 94143d6
Showing
17 changed files
with
571 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
modules/nextflow/src/main/groovy/nextflow/cli/CmdInspect.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| /* | ||
| * Copyright 2013-2023, Seqera Labs | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package nextflow.cli | ||
|
|
||
| import com.beust.jcommander.DynamicParameter | ||
| import com.beust.jcommander.Parameter | ||
| import com.beust.jcommander.Parameters | ||
| import groovy.transform.CompileStatic | ||
| import groovy.util.logging.Slf4j | ||
| import nextflow.Session | ||
| import nextflow.container.inspect.ContainersInspector | ||
| import nextflow.util.LoggerHelper | ||
| /** | ||
| * Implement `inspect` command | ||
| * | ||
| * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com> | ||
| */ | ||
| @Slf4j | ||
| @CompileStatic | ||
| @Parameters(commandDescription = "Inspect process settings in a pipeline project") | ||
| class CmdInspect extends CmdBase { | ||
|
|
||
| @Override | ||
| String getName() { | ||
| return 'inspect' | ||
| } | ||
|
|
||
| @Parameter(names=['-concretize'], description = "Build the container images resolved by the inspect command") | ||
| boolean concretize | ||
|
|
||
| @Parameter(names=['-c','-config'], hidden = true) | ||
| List<String> runConfig | ||
|
|
||
| @Parameter(names=['-format'], description = "Inspect output format. Can be 'json' or 'config'") | ||
| String format = 'json' | ||
|
|
||
| @Parameter(names=['-i','-ignore-errors'], description = 'Ignore errors while inspecting the pipeline') | ||
| boolean ignoreErrors | ||
|
|
||
| @DynamicParameter(names = '--', hidden = true) | ||
| Map<String,String> params = new LinkedHashMap<>() | ||
|
|
||
| @Parameter(names='-params-file', description = 'Load script parameters from a JSON/YAML file') | ||
| String paramsFile | ||
|
|
||
| @Parameter(names=['-profile'], description = 'Use the given configuration profile(s)') | ||
| String profile | ||
|
|
||
| @Parameter(names=['-r','-revision'], description = 'Revision of the project to inspect (either a git branch, tag or commit SHA number)') | ||
| String revision | ||
|
|
||
| @Parameter(description = 'Project name or repository url') | ||
| List<String> args | ||
|
|
||
| @Override | ||
| void run() { | ||
| // configure quiet mode | ||
| LoggerHelper.setQuiet(true) | ||
| // setup the target run command | ||
| final target = new CmdRun() | ||
| target.launcher = this.launcher | ||
| target.args = args | ||
| target.profile = this.profile | ||
| target.revision = this.revision | ||
| target.runConfig = this.runConfig | ||
| target.params = this.params | ||
| target.paramsFile = this.paramsFile | ||
| target.preview = true | ||
| target.previewAction = this.&applyInspect | ||
| target.ansiLog = false | ||
| // run it | ||
| target.run() | ||
| } | ||
|
|
||
| protected void applyInspect(Session session) { | ||
| // disable wave await mode when running | ||
| if( session.config.wave instanceof Map ) | ||
| checkWaveConfig(session.config.wave as Map) | ||
| // run the inspector | ||
| new ContainersInspector(session.dag) | ||
| .withFormat(format) | ||
| .withIgnoreErrors(ignoreErrors) | ||
| .printContainers() | ||
| } | ||
|
|
||
| protected void checkWaveConfig(Map wave) { | ||
| if( wave.enabled && wave.freeze ) | ||
| wave.dryRun = !concretize | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
113 changes: 113 additions & 0 deletions
113
modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainersInspector.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| /* | ||
| * Copyright 2013-2023, Seqera Labs | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package nextflow.container.inspect | ||
|
|
||
| import groovy.json.JsonBuilder | ||
| import groovy.json.JsonOutput | ||
| import groovy.transform.CompileStatic | ||
| import groovy.util.logging.Slf4j | ||
| import nextflow.dag.DAG | ||
| import nextflow.exception.AbortOperationException | ||
| import org.codehaus.groovy.util.ListHashMap | ||
| /** | ||
| * Preview the list of containers used by a pipeline. | ||
| * | ||
| * @author Ben Sherman <bentshermann@gmail.com> | ||
| */ | ||
| @Slf4j | ||
| @CompileStatic | ||
| class ContainersInspector { | ||
|
|
||
| private DAG dag | ||
|
|
||
| private String format | ||
|
|
||
| private boolean ignoreErrors | ||
|
|
||
| ContainersInspector(DAG dag) { | ||
| this.dag = dag | ||
| } | ||
|
|
||
| ContainersInspector withFormat(String format) { | ||
| if( format !in ['config', 'json'] ) | ||
| throw new AbortOperationException("Invalid format for containers inspect '${format}' -- should be 'config' or 'json'") | ||
| this.format = format | ||
| return this | ||
| } | ||
|
|
||
| ContainersInspector withIgnoreErrors(boolean ignore) { | ||
| this.ignoreErrors = ignore | ||
| return this | ||
| } | ||
|
|
||
| String renderContainers() { | ||
| log.debug "Rendering container preview" | ||
| final containers = getContainers() | ||
| if( format == 'config' ) | ||
| return renderConfig(containers) | ||
| if( format == 'json' ) | ||
| return renderJson(containers) | ||
| else | ||
| throw new IllegalStateException("Unknown containers preview format: $format") | ||
| } | ||
|
|
||
| void printContainers() { | ||
| final result = renderContainers() | ||
| if( result ) | ||
| print result | ||
| } | ||
|
|
||
| protected Map<String,String> getContainers() { | ||
| final containers = new ListHashMap<String,String>() | ||
|
|
||
| for( def vertex : dag.vertices ) { | ||
| // skip nodes that are not processes | ||
| final process = vertex.process | ||
| if( !process ) | ||
| continue | ||
|
|
||
| try { | ||
| // get container preview | ||
| containers[process.name] = process.createTaskPreview().getContainer() | ||
| } | ||
| catch( Exception e ) { | ||
| if( ignoreErrors ) | ||
| log.warn "Unable to inspect container for task `$process.name` - cause: ${e.message}" | ||
| else | ||
| throw e | ||
| } | ||
| } | ||
|
|
||
| return containers | ||
| } | ||
|
|
||
| protected String renderConfig(Map<String,String> containers) { | ||
| final result = new StringBuilder() | ||
| for( Map.Entry<String,String> entry : containers ) { | ||
| result.append("process { withName: '${entry.key}' { container = '${entry.value}' } }\n") | ||
| } | ||
| return result.toString() | ||
| } | ||
|
|
||
| protected String renderJson(Map<String,String> containers) { | ||
| final list = containers.collect( (k, v) -> [name: k, container: v] ) | ||
| final result = Map.of("processes", list) | ||
| return JsonOutput.prettyPrint(new JsonBuilder(result).toString()) + '\n' | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.