From 853be62744e27599a30e91827c98f6ce1d376d3c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 13 Nov 2024 12:30:40 +0100 Subject: [PATCH 1/3] Use Java native functions to read the trace file to avoid problems where Nextflow got stuck reading the trace File. Signed-off-by: Lehmann_Fabian --- .../groovy/nextflow/trace/TraceRecord.groovy | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy index 33c4a5d510..acf9c6f8c4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy @@ -418,48 +418,47 @@ class TraceRecord implements Serializable { */ TraceRecord parseTraceFile( Path file ) { + try (BufferedReader reader = new BufferedReader(new FileReader(file.toFile()))) { + + if( reader.readLine() != 'nextflow.trace/v2' ) + return parseLegacy(file, file.readLines()) + + String line + while(( line = reader.readLine() ) != null ) { + final pair = line.tokenize('=') + final name = pair[0] + final value = pair[1] + if( value == null ) + continue + + switch (name) { + case '%cpu': + case '%mem': + // fields '%cpu' and '%mem' are expressed as percent value + this.put(name, parseInt(value, file, name) / 10F) + break + + case 'rss': + case 'vmem': + case 'peak_rss': + case 'peak_vmem': + // these fields are provided in KB, so they are normalized to bytes + def val = parseLong(value, file, name) * 1024 + this.put(name, val) + break - final text = file.text - - final lines = text.readLines() - if( !lines ) - return this - if( lines[0] != 'nextflow.trace/v2' ) - return parseLegacy(file, lines) - - for( int i=0; i Date: Wed, 13 Nov 2024 16:59:11 +0100 Subject: [PATCH 2/3] Fix test cases to allow toFile method on temporary files Signed-off-by: Lehmann_Fabian --- .../groovy/nextflow/processor/TaskHandlerTest.groovy | 7 ++++++- .../nextflow/trace/TraceFileObserverTest.groovy | 12 ++++++++++-- .../groovy/nextflow/trace/TraceRecordTest.groovy | 11 +++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskHandlerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskHandlerTest.groovy index 6ec29faf44..0b0f3a6226 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskHandlerTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskHandlerTest.groovy @@ -16,6 +16,9 @@ package nextflow.processor +import spock.lang.TempDir + +import java.nio.file.Path import java.util.concurrent.atomic.LongAdder import nextflow.Session @@ -33,6 +36,9 @@ class TaskHandlerTest extends Specification { def static final long KB = 1024 + @TempDir + Path folder + def 'test get trace record'() { given: @@ -42,7 +48,6 @@ class TaskHandlerTest extends Specification { ''' .leftTrim() - def folder = TestHelper.createInMemTempDir() folder.resolve( TaskRun.CMD_TRACE ).text = traceText def config = [ diff --git a/modules/nextflow/src/test/groovy/nextflow/trace/TraceFileObserverTest.groovy b/modules/nextflow/src/test/groovy/nextflow/trace/TraceFileObserverTest.groovy index af7c092094..559a973834 100644 --- a/modules/nextflow/src/test/groovy/nextflow/trace/TraceFileObserverTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/trace/TraceFileObserverTest.groovy @@ -15,6 +15,9 @@ */ package nextflow.trace + +import spock.lang.TempDir + import java.nio.file.Files import nextflow.Session @@ -28,7 +31,9 @@ import nextflow.processor.TaskStatus import nextflow.util.CacheHelper import nextflow.util.Duration import spock.lang.Specification -import test.TestHelper + +import java.nio.file.Path + /** * * @author Paolo Di Tommaso @@ -37,6 +42,9 @@ class TraceFileObserverTest extends Specification { final static long MB = 1024 * 1024 + @TempDir + Path tempDir + def setupSpec() { TraceRecord.TIMEZONE = TimeZone.getTimeZone('UTC') // note: set the timezone to be sure the time string does not change on CI test servers } @@ -235,7 +243,7 @@ class TraceFileObserverTest extends Specification { given: final KB = 1024L - final file = TestHelper.createInMemTempFile('trace') + final file = Files.createTempFile(tempDir,'trace', '.txt') file.text = ''' pid state %cpu %mem vmem rss peak_vmem peak_rss rchar wchar syscr syscw read_bytes write_bytes 18 0 7999 46 7868980 6694900 7876620 6702144 2147483647 2147483647 44001533 148401890 2147483647 2147483647 diff --git a/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy b/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy index f30d2c7d8f..af156ce0a1 100644 --- a/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy @@ -18,9 +18,13 @@ package nextflow.trace import groovy.json.JsonSlurper import spock.lang.Specification +import spock.lang.TempDir import spock.lang.Unroll import test.TestHelper +import java.nio.file.Files +import java.nio.file.Path + /** * * @author Paolo Di Tommaso @@ -33,6 +37,9 @@ class TraceRecordTest extends Specification { TraceRecord.TIMEZONE = TimeZone.getTimeZone('UTC') // note: set the timezone to be sure the time string does not change on CI test servers } + @TempDir + Path tempDir + def 'test get field'() { given: @@ -153,7 +160,7 @@ class TraceRecordTest extends Specification { def 'should parse a trace file and return a TraceRecord object'() { given: - def file = TestHelper.createInMemTempFile('trace') + def file = Files.createTempFile(tempDir,'trace', '.txt') file.text = '''\ nextflow.trace/v2 realtime=12021 @@ -200,7 +207,7 @@ class TraceRecordTest extends Specification { def 'should parse a legacy trace file and return a TraceRecord object'() { given: - def file = TestHelper.createInMemTempFile('trace') + def file = Files.createTempFile(tempDir,'trace', '.txt') file.text = ''' pid state %cpu %mem vmem rss peak_vmem peak_rss rchar wchar syscr syscw read_bytes write_bytes 1 0 10 20 11084 1220 21084 2220 4790 12 11 1 20 30 From c2faac1644b29b4f9c8ed05396fbcdc40caafd27 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 13 Nov 2024 16:59:59 +0100 Subject: [PATCH 3/3] Recover functionality for empty trace file Signed-off-by: Lehmann_Fabian --- .../src/main/groovy/nextflow/trace/TraceRecord.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy index acf9c6f8c4..fad84b9625 100644 --- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy @@ -420,10 +420,13 @@ class TraceRecord implements Serializable { TraceRecord parseTraceFile( Path file ) { try (BufferedReader reader = new BufferedReader(new FileReader(file.toFile()))) { - if( reader.readLine() != 'nextflow.trace/v2' ) + String line + if( ( line = reader.readLine() ) == null ) { + return this + } else if ( line != 'nextflow.trace/v2' ) { return parseLegacy(file, file.readLines()) + } - String line while(( line = reader.readLine() ) != null ) { final pair = line.tokenize('=') final name = pair[0]