From af157d1676b7dec009b840f45eaa879311ef5b68 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 4 May 2026 14:14:13 -0700 Subject: [PATCH 1/2] Fix async profiler output copy to s3 Signed-off-by: Jihoon Son --- .../com/nvidia/spark/rapids/asyncProfiler.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala index f706672c0b6..b161b90d84f 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -355,16 +355,21 @@ object AsyncProfilerOnExecutor extends Logging { // Compress the temp file first val compressedTempPath = Files.createTempFile("compressed-jfr", ".gz") if (compressJfrFile(tempFilePath, compressedTempPath)) { - fs.copyFromLocalFile(new Path(compressedTempPath.toString), outPath) + withResource(fs.create(outPath, false)) { out => + Files.copy(compressedTempPath, out) + } compressedTempPath.toFile.delete() // delete compressed temp file } else { // If compression fails, copy the original file log.warn("JFR compression failed, copying uncompressed file") - fs.copyFromLocalFile(new Path(tempFilePath.toString), - new Path(asyncProfilerPrefix.get, baseFileName)) + withResource(fs.create(new Path(asyncProfilerPrefix.get, baseFileName), false)) { out => + Files.copy(tempFilePath, out) + } } } else { - fs.copyFromLocalFile(new Path(tempFilePath.toString), outPath) + withResource(fs.create(outPath, false)) { out => + Files.copy(tempFilePath, out) + } } tempFilePath.toFile.delete() // delete the original temp file after moving } else { From 8319439408beca4173a36248dcd51dc2c7aceb28 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 4 May 2026 14:47:18 -0700 Subject: [PATCH 2/2] fix line limit --- .../src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala index b161b90d84f..1cd6f625909 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/asyncProfiler.scala @@ -362,7 +362,8 @@ object AsyncProfilerOnExecutor extends Logging { } else { // If compression fails, copy the original file log.warn("JFR compression failed, copying uncompressed file") - withResource(fs.create(new Path(asyncProfilerPrefix.get, baseFileName), false)) { out => + withResource( + fs.create(new Path(asyncProfilerPrefix.get, baseFileName), false)) { out => Files.copy(tempFilePath, out) } }