From a6354cd7e2106249567424700392531e5c8cc2ae Mon Sep 17 00:00:00 2001 From: mingji Date: Thu, 14 Nov 2024 15:23:55 +0800 Subject: [PATCH] [CELEBORN-1718] Fix memory storage file won't hard split when memory file is full and worker has no disks --- .../celeborn/service/deploy/worker/PushDataHandler.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala index 592e93f093..b6ddff2613 100644 --- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala +++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala @@ -1251,8 +1251,13 @@ class PushDataHandler(val workerSource: WorkerSource) extends BaseMessageHandler |fileName:${fileWriter.getCurrentFileInfo.getFilePath} |""".stripMargin) if (fileWriter.needHardSplitForMemoryShuffleStorage()) { + workerSource.incCounter(WorkerSource.WRITE_DATA_HARD_SPLIT_COUNT) + callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue))) + logInfo( + s"Do hardSplit for memory shuffle file fileLength:${fileWriter.getMemoryFileInfo.getFileLength}") return true } + val diskFileInfo = fileWriter.getDiskFileInfo if (diskFileInfo != null) { if (workerPartitionSplitEnabled && ((diskFull && diskFileInfo.getFileLength > partitionSplitMinimumSize) ||