From cb7f5783c891689f25767ebc0f62ac30877a2b4b Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Thu, 4 Dec 2025 17:03:00 -0500 Subject: [PATCH 1/5] copy fix over from PR #7826 --- CHANGELOG.md | 1 + .../scala/cromwell/backend/impl/aws/AwsBatchJob.scala | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 249775fb31..a4d9823313 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 92 Release Notes ### AWS Batch +* Fixed an issue where job failures before all outputs were written would cause delocalization to fail, preventing the upload of return code, stdout, and stderr files needed for debugging. * Moved the option to tag job resources from runtime attributes to backend config. * Appended the custom labels to the list of resource tags to propagate diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala index 1372cb9c02..2434f8621d 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala @@ -577,7 +577,6 @@ final case class AwsBatchJob( replaced.patch(insertionPoint, preamble, 0) + s""" |{ - |set -e |# (re-)add tags to include added volumes: |if [[ "${doTagging}" == "true" ]]; then | echo "*** TAGGING RESOURCES ***" @@ -588,10 +587,16 @@ final case class AwsBatchJob( |DELOCALIZATION_FAILED=0 |$outputCopyCommand |echo "DELOCALIZATION RESULT: $$DELOCALIZATION_FAILED" + |rc=$$(head -n 1 $workDir/${jobPaths.returnCodeFilename} 2>/dev/null) || rc=1 |if [[ $$DELOCALIZATION_FAILED -eq 1 ]]; then | echo '*** DELOCALIZATION FAILED ***' - | echo '*** EXITING WITH RETURN CODE 1***' - | exit 1 + | if [[ "$$rc" -eq 0 ]]; then + | echo '*** EXITING WITH RETURN CODE 1 ***' + | exit 1 + | else + | echo "*** EXITING WITH ORIGINAL RETURN CODE $$rc ***" + | exit $$rc + | fi |else | echo '*** COMPLETED DELOCALIZATION ***' |fi From 77d1cfc57a581a70330b3d81fe21ec613b2583c5 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 5 Dec 2025 09:30:09 -0500 Subject: [PATCH 2/5] fix unit test --- .../cromwell/backend/impl/aws/AwsBatchJobSpec.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala index 9c1a9c7706..16fec6627a 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala @@ -471,7 +471,6 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi val job = generateJobWithS3InOut val postscript = s""" - |set -e |# (re-)add tags to include added volumes: |if [[ "false" == "true" ]]; then | echo "*** TAGGING RESOURCES ***" @@ -487,10 +486,16 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi |if [ -f "/tmp/scratch/hello-stdout.log" ]; then _s3_delocalize_with_retry "/tmp/scratch/hello-stdout.log" "${job.jobPaths.standardPaths.output}"; fi | |echo "DELOCALIZATION RESULT: $$DELOCALIZATION_FAILED" + |rc=$$(head -n 1 /tmp/scratch/hello-rc.txt 2>/dev/null) || rc=1 |if [[ $$DELOCALIZATION_FAILED -eq 1 ]]; then | echo '*** DELOCALIZATION FAILED ***' - | echo '*** EXITING WITH RETURN CODE 1***' - | exit 1 + | if [[ "$$rc" -eq 0 ]]; then + | echo '*** EXITING WITH RETURN CODE 1 ***' + | exit 1 + | else + | echo "*** EXITING WITH ORIGINAL RETURN CODE $$rc ***" + | exit $$rc + | fi |else | echo '*** COMPLETED DELOCALIZATION ***' |fi From 9990f9ebf9905dfd52e0e4608bfb84e1691e7a01 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 9 Dec 2025 13:31:11 -0500 Subject: [PATCH 3/5] delocalize debugging files first --- .../backend/impl/aws/AwsBatchJob.scala | 215 +++++++++--------- .../backend/impl/aws/AwsBatchJobSpec.scala | 11 +- 2 files changed, 112 insertions(+), 114 deletions(-) diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala index 2434f8621d..8dfd59c503 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala @@ -464,139 +464,142 @@ final case class AwsBatchJob( val stdErr = dockerStderr.replace("/cromwell_root", workDir) // generate a series of s3 commands to delocalize artifacts from the container to storage at the end of the task - val outputCopyCommand = outputs - .map { - // local is relative path, no mountpoint disk in front. - case output: AwsBatchFileOutput if output.local.pathAsString.contains("*") => "" // filter out globs - case output: AwsBatchFileOutput if output.s3key.endsWith(".list") && output.s3key.contains("glob-") => - Log.debug("Globbing : check for EFS settings.") - val s3GlobOutDirectory = output.s3key.replace(".list", "") - // glob paths are not generated with 127 char limit, using generateGlobPaths(). name can be used safely - val globDirectory = output.name.replace(".list", "") - /* - * Need to process this list and de-localize each file if the list file actually exists - * if it doesn't exist then 'touch' it so that it can be copied otherwise later steps will get upset - * about the missing file - */ - if (efsMntPoint.isDefined && output.mount.mountPoint.pathAsString == efsMntPoint.get) { - Log.debug( - "EFS glob output file detected: " + output.s3key + s" / ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" - ) - val test_cmd = if (efsDelocalize.isDefined && efsDelocalize.getOrElse(false)) { - Log.debug("delocalization on EFS is enabled") - Log.debug(s"Delocalizing $globDirectory to $s3GlobOutDirectory\n") + val outputCopyCommand = { + val coreDelocalize = + s""" + |if [ -f "$workDir/${jobPaths.returnCodeFilename}" ]; then _s3_delocalize_with_retry "$workDir/${jobPaths.returnCodeFilename}" "${jobPaths.callRoot.pathAsString}/${jobPaths.returnCodeFilename}" ; fi + |if [ -f "$stdErr" ]; then _s3_delocalize_with_retry "$stdErr" "${jobPaths.standardPaths.error.pathAsString}"; fi + |if [ -f "$stdOut" ]; then _s3_delocalize_with_retry "$stdOut" "${jobPaths.standardPaths.output.pathAsString}"; fi + |""".stripMargin + + val otherOutputs = outputs + .map { + // local is relative path, no mountpoint disk in front. + case output: AwsBatchFileOutput if output.local.pathAsString.contains("*") => "" // filter out globs + case output: AwsBatchFileOutput if output.s3key.endsWith(".list") && output.s3key.contains("glob-") => + Log.debug("Globbing : check for EFS settings.") + val s3GlobOutDirectory = output.s3key.replace(".list", "") + // glob paths are not generated with 127 char limit, using generateGlobPaths(). name can be used safely + val globDirectory = output.name.replace(".list", "") + /* + * Need to process this list and de-localize each file if the list file actually exists + * if it doesn't exist then 'touch' it so that it can be copied otherwise later steps will get upset + * about the missing file + */ + if (efsMntPoint.isDefined && output.mount.mountPoint.pathAsString == efsMntPoint.get) { + Log.debug( + "EFS glob output file detected: " + output.s3key + s" / ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" + ) + val test_cmd = if (efsDelocalize.isDefined && efsDelocalize.getOrElse(false)) { + Log.debug("delocalization on EFS is enabled") + Log.debug(s"Delocalizing $globDirectory to $s3GlobOutDirectory\n") + s""" + |touch "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" + |_s3_delocalize_with_retry "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${output.s3key}" + |if [ -e $globDirectory ]; then _s3_delocalize_with_retry "$globDirectory" "$s3GlobOutDirectory" ; fi + |""".stripMargin + } else { + + // check file for existence + s""" + |# test the glob list + |test -e "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" || (echo 'output file: ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString} does not exist' && DELOCALIZATION_FAILED=1) + |# test individual files. + |SAVEIFS="$$IFS" + |IFS=$$'\n' + |for F in $$(cat "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}"); do + | test -e "${globDirectory}/$$F" || (echo 'globbed file: "${globDirectory}/$$F" does not exist' && DELOCALIZATION_FAILED=1 && break) + |done + |IFS="$$SAVEIFS" + |""" + } + // need to make md5sum? + val md5_cmd = if (efsMakeMD5.isDefined && efsMakeMD5.getOrElse(false)) { + Log.debug("Add cmd to create MD5 sibling.") + // generate MD5 if missing or if local file is newer than sibling md5 + s""" + |if [[ ! -f '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' || '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}' -nt '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' ]]; then + | # the glob list + | md5sum '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}' > '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' || (echo 'Could not generate ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' && DELOCALIZATION_FAILED=1 ); + | # globbed files, using specified number of cpus for parallel processing. + | SAVEIFS="$$IFS" + | IFS=$$'\n' + | cat "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" | xargs -I% -P${runtimeAttributes.cpu.##.toString} bash -c "md5sum ${globDirectory}/% > ${globDirectory}/%.md5" + | IFS="$$SAVEIFS" + |fi + |""".stripMargin + } + // return combined result s""" - |touch "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" - |_s3_delocalize_with_retry "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${output.s3key}" - |if [ -e $globDirectory ]; then _s3_delocalize_with_retry "$globDirectory" "$s3GlobOutDirectory" ; fi - |""".stripMargin + |${test_cmd} + |${md5_cmd} + | """.stripMargin } else { - - // check file for existence + // default delocalization command. + Log.debug(s"Delocalize from ${output.name} to ${output.s3key}\n") s""" - |# test the glob list - |test -e "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" || (echo 'output file: ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString} does not exist' && DELOCALIZATION_FAILED=1) - |# test individual files. - |SAVEIFS="$$IFS" - |IFS=$$'\n' - |for F in $$(cat "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}"); do - | test -e "${globDirectory}/$$F" || (echo 'globbed file: "${globDirectory}/$$F" does not exist' && DELOCALIZATION_FAILED=1 && break) - |done - |IFS="$$SAVEIFS" - |""" + |touch "${output.name}" + |_s3_delocalize_with_retry "${output.name}" "${output.s3key}" + |if [ -e "$globDirectory" ]; then _s3_delocalize_with_retry "$globDirectory" "$s3GlobOutDirectory" ; fi""".stripMargin } - // need to make md5sum? - val md5_cmd = if (efsMakeMD5.isDefined && efsMakeMD5.getOrElse(false)) { - Log.debug("Add cmd to create MD5 sibling.") - // generate MD5 if missing or if local file is newer than sibling md5 - s""" - |if [[ ! -f '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' || '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}' -nt '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' ]]; then - | # the glob list - | md5sum '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}' > '${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' || (echo 'Could not generate ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}.md5' && DELOCALIZATION_FAILED=1 ); - | # globbed files, using specified number of cpus for parallel processing. - | SAVEIFS="$$IFS" - | IFS=$$'\n' - | cat "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" | xargs -I% -P${runtimeAttributes.cpu.##.toString} bash -c "md5sum ${globDirectory}/% > ${globDirectory}/%.md5" - | IFS="$$SAVEIFS" - |fi - |""".stripMargin - } - // return combined result - s""" - |${test_cmd} - |${md5_cmd} - | """.stripMargin - } else { - // default delocalization command. - Log.debug(s"Delocalize from ${output.name} to ${output.s3key}\n") - s""" - |touch "${output.name}" - |_s3_delocalize_with_retry "${output.name}" "${output.s3key}" - |if [ -e "$globDirectory" ]; then _s3_delocalize_with_retry "$globDirectory" "$s3GlobOutDirectory" ; fi""".stripMargin - } - // files on /cromwell/ working dir must be delocalized - case output: AwsBatchFileOutput + // files on /cromwell/ working dir must be delocalized + case output: AwsBatchFileOutput if output.s3key.startsWith( "s3://" ) && output.mount.mountPoint.pathAsString == AwsBatchWorkingDisk.MountPoint.pathAsString => - // output is on working disk mount - s"""_s3_delocalize_with_retry "$workDir/${output.local.pathAsString}" "${output.s3key}" "${output.optional}" """.stripMargin + // output is on working disk mount + s"""_s3_delocalize_with_retry "$workDir/${output.local.pathAsString}" "${output.s3key}" "${output.optional}" """.stripMargin - // files on EFS mounts are optionally delocalized. - case output: AwsBatchFileOutput + // files on EFS mounts are optionally delocalized. + case output: AwsBatchFileOutput if efsMntPoint.isDefined && output.mount.mountPoint.pathAsString == efsMntPoint.get => - Log.debug( - "EFS output file detected: " + output.s3key + s" / ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" - ) - // EFS located file : test existence or delocalize. - if (efsDelocalize.isDefined && efsDelocalize.getOrElse(false)) { - Log.debug("efs-delocalization enabled") - s"""_s3_delocalize_with_retry "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${output.s3key}" "${output.optional}" """.stripMargin - } else { - Log.debug("efs-delocalization disabled") - s"""_check_efs_outfile "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${efsMakeMD5 - .getOrElse(false)}" "${output.optional}" """.stripMargin - } + Log.debug( + "EFS output file detected: " + output.s3key + s" / ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" + ) + // EFS located file : test existence or delocalize. + if (efsDelocalize.isDefined && efsDelocalize.getOrElse(false)) { + Log.debug("efs-delocalization enabled") + s"""_s3_delocalize_with_retry "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${output.s3key}" "${output.optional}" """.stripMargin + } else { + Log.debug("efs-delocalization disabled") + s"""_check_efs_outfile "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${ + efsMakeMD5 + .getOrElse(false) + }" "${output.optional}" """.stripMargin + } - case output: AwsBatchFileOutput => - // output on a different mount - Log.debug("output data on other mount") - Log.debug(output.toString) - s"""_s3_delocalize_with_retry "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${output.s3key}" """.stripMargin - case _ => "" - } - .mkString("\n") + "\n" + - s""" - |if [ -f "$workDir/${jobPaths.returnCodeFilename}" ]; then _s3_delocalize_with_retry "$workDir/${jobPaths.returnCodeFilename}" "${jobPaths.callRoot.pathAsString}/${jobPaths.returnCodeFilename}" ; fi - |if [ -f "$stdErr" ]; then _s3_delocalize_with_retry "$stdErr" "${jobPaths.standardPaths.error.pathAsString}"; fi - |if [ -f "$stdOut" ]; then _s3_delocalize_with_retry "$stdOut" "${jobPaths.standardPaths.output.pathAsString}"; fi - |""".stripMargin + case output: AwsBatchFileOutput => + // output on a different mount + Log.debug("output data on other mount") + Log.debug(output.toString) + s"""_s3_delocalize_with_retry "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${output.s3key}" """.stripMargin + case _ => "" + } + .mkString("\n") + + // Prepend core delocalize lines so rc, stderr, stdout are always delocalized first. + coreDelocalize + "\n" + otherOutputs + "\n" + } // insert the preamble at the insertion point and the postscript copy command at the end replaced.patch(insertionPoint, preamble, 0) + s""" |{ + |set -e |# (re-)add tags to include added volumes: |if [[ "${doTagging}" == "true" ]]; then | echo "*** TAGGING RESOURCES ***" | _add_tags |fi - | |echo '*** DELOCALIZING OUTPUTS ***' |DELOCALIZATION_FAILED=0 |$outputCopyCommand |echo "DELOCALIZATION RESULT: $$DELOCALIZATION_FAILED" - |rc=$$(head -n 1 $workDir/${jobPaths.returnCodeFilename} 2>/dev/null) || rc=1 |if [[ $$DELOCALIZATION_FAILED -eq 1 ]]; then | echo '*** DELOCALIZATION FAILED ***' - | if [[ "$$rc" -eq 0 ]]; then - | echo '*** EXITING WITH RETURN CODE 1 ***' - | exit 1 - | else - | echo "*** EXITING WITH ORIGINAL RETURN CODE $$rc ***" - | exit $$rc - | fi + | echo '*** EXITING WITH RETURN CODE 1***' + | exit 1 |else | echo '*** COMPLETED DELOCALIZATION ***' |fi diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala index 16fec6627a..9c1a9c7706 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala @@ -471,6 +471,7 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi val job = generateJobWithS3InOut val postscript = s""" + |set -e |# (re-)add tags to include added volumes: |if [[ "false" == "true" ]]; then | echo "*** TAGGING RESOURCES ***" @@ -486,16 +487,10 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi |if [ -f "/tmp/scratch/hello-stdout.log" ]; then _s3_delocalize_with_retry "/tmp/scratch/hello-stdout.log" "${job.jobPaths.standardPaths.output}"; fi | |echo "DELOCALIZATION RESULT: $$DELOCALIZATION_FAILED" - |rc=$$(head -n 1 /tmp/scratch/hello-rc.txt 2>/dev/null) || rc=1 |if [[ $$DELOCALIZATION_FAILED -eq 1 ]]; then | echo '*** DELOCALIZATION FAILED ***' - | if [[ "$$rc" -eq 0 ]]; then - | echo '*** EXITING WITH RETURN CODE 1 ***' - | exit 1 - | else - | echo "*** EXITING WITH ORIGINAL RETURN CODE $$rc ***" - | exit $$rc - | fi + | echo '*** EXITING WITH RETURN CODE 1***' + | exit 1 |else | echo '*** COMPLETED DELOCALIZATION ***' |fi From 4d5a7f571b8cf836789a14de0d271da640ee0e85 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 9 Dec 2025 13:38:09 -0500 Subject: [PATCH 4/5] format fix --- .../cromwell/backend/impl/aws/AwsBatchJob.scala | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala index 8dfd59c503..a9796fdf04 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchJob.scala @@ -545,15 +545,15 @@ final case class AwsBatchJob( // files on /cromwell/ working dir must be delocalized case output: AwsBatchFileOutput - if output.s3key.startsWith( - "s3://" - ) && output.mount.mountPoint.pathAsString == AwsBatchWorkingDisk.MountPoint.pathAsString => + if output.s3key.startsWith( + "s3://" + ) && output.mount.mountPoint.pathAsString == AwsBatchWorkingDisk.MountPoint.pathAsString => // output is on working disk mount s"""_s3_delocalize_with_retry "$workDir/${output.local.pathAsString}" "${output.s3key}" "${output.optional}" """.stripMargin // files on EFS mounts are optionally delocalized. case output: AwsBatchFileOutput - if efsMntPoint.isDefined && output.mount.mountPoint.pathAsString == efsMntPoint.get => + if efsMntPoint.isDefined && output.mount.mountPoint.pathAsString == efsMntPoint.get => Log.debug( "EFS output file detected: " + output.s3key + s" / ${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" ) @@ -563,10 +563,8 @@ final case class AwsBatchJob( s"""_s3_delocalize_with_retry "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${output.s3key}" "${output.optional}" """.stripMargin } else { Log.debug("efs-delocalization disabled") - s"""_check_efs_outfile "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${ - efsMakeMD5 - .getOrElse(false) - }" "${output.optional}" """.stripMargin + s"""_check_efs_outfile "${output.mount.mountPoint.pathAsString}/${output.local.pathAsString}" "${efsMakeMD5 + .getOrElse(false)}" "${output.optional}" """.stripMargin } case output: AwsBatchFileOutput => @@ -592,6 +590,7 @@ final case class AwsBatchJob( | echo "*** TAGGING RESOURCES ***" | _add_tags |fi + | |echo '*** DELOCALIZING OUTPUTS ***' |DELOCALIZATION_FAILED=0 |$outputCopyCommand From befc10933a07bd65ff0e39eff73080a49f0ae1fb Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 9 Dec 2025 15:18:55 -0500 Subject: [PATCH 5/5] fix unit test --- .../scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala index 9c1a9c7706..15373cc46f 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala @@ -471,6 +471,7 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi val job = generateJobWithS3InOut val postscript = s""" + |{ |set -e |# (re-)add tags to include added volumes: |if [[ "false" == "true" ]]; then @@ -480,12 +481,13 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi | |echo '*** DELOCALIZING OUTPUTS ***' |DELOCALIZATION_FAILED=0 - |_s3_delocalize_with_retry "/tmp/scratch/baa" "s3://bucket/somewhere/baa" "false" | |if [ -f "/tmp/scratch/hello-rc.txt" ]; then _s3_delocalize_with_retry "/tmp/scratch/hello-rc.txt" "${job.jobPaths.returnCode}" ; fi |if [ -f "/tmp/scratch/hello-stderr.log" ]; then _s3_delocalize_with_retry "/tmp/scratch/hello-stderr.log" "${job.jobPaths.standardPaths.error}"; fi |if [ -f "/tmp/scratch/hello-stdout.log" ]; then _s3_delocalize_with_retry "/tmp/scratch/hello-stdout.log" "${job.jobPaths.standardPaths.output}"; fi | + |_s3_delocalize_with_retry "/tmp/scratch/baa" "s3://bucket/somewhere/baa" "false" + | |echo "DELOCALIZATION RESULT: $$DELOCALIZATION_FAILED" |if [[ $$DELOCALIZATION_FAILED -eq 1 ]]; then | echo '*** DELOCALIZATION FAILED ***' @@ -500,7 +502,9 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi |exit $$rc |} |""".stripMargin - job.reconfiguredScript should include(postscript) + + val normalize: String => String = s => s.replaceAll("\\s+", " ").trim + normalize(job.reconfiguredScript) should include(normalize(postscript)) } it should "generate preamble with input copy command in reconfigured script" in {