Skip to content

Commit 51a1006

Browse files
boegelboegel
authored
Merge pull request #844 from smoors/alt_artefacts
add support for alternative artefacts checks in check-build.sh
2 parents 0460342 + e5990d2 commit 51a1006

File tree

1 file changed

+158
-137
lines changed

1 file changed

+158
-137
lines changed

‎bot/check-build.sh‎

Lines changed: 158 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
# https://github.com/EESSI/software-layer.git
99
#
1010
# author: Thomas Roeblitz (@trz42)
11+
# author: Samuel Moors (@smoors)
1112
#
1213
# license: GPLv2
1314
#
@@ -59,10 +60,12 @@ display_help() {
5960
echo " OPTIONS:"
6061
echo " -h | --help - display this usage information [default: false]"
6162
echo " -v | --verbose - display more information [default: false]"
63+
echo " --use-check-build-artefacts-script - alternative build artefacts check (sources file check-build-artefacts.sh if exists) [default: false]"
6264
}
6365

6466
# set defaults for command line arguments
6567
VERBOSE=0
68+
USE_CHECK_BUILD_ARTEFACTS_SCRIPT=0
6669

6770
POSITIONAL_ARGS=()
6871

@@ -76,6 +79,10 @@ while [[ $# -gt 0 ]]; do
7679
VERBOSE=1
7780
shift 1
7881
;;
82+
--use-check-build-artefacts-script)
83+
USE_CHECK_BUILD_ARTEFACTS_SCRIPT=1
84+
shift 1
85+
;;
7986
--)
8087
shift
8188
POSITIONAL_ARGS+=("$@") # save positional args
@@ -157,20 +164,22 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
157164
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
158165
fi
159166

160-
TGZ=-1
161-
TARBALL=
162-
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
163-
GP_tgz_created="\.tar\.gz created!"
164-
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
165-
if [[ $? -eq 0 ]]; then
166-
TGZ=1
167-
TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
168-
else
169-
TGZ=0
170-
fi
171-
# have to be careful to not add searched for pattern into slurm out file
172-
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
173-
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
167+
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
168+
TGZ=-1
169+
TARBALL=
170+
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
171+
GP_tgz_created="\.tar\.gz created!"
172+
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
173+
if [[ $? -eq 0 ]]; then
174+
TGZ=1
175+
TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
176+
else
177+
TGZ=0
178+
fi
179+
# have to be careful to not add searched for pattern into slurm out file
180+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
181+
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
182+
fi
174183
fi
175184

176185
[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}"
@@ -180,7 +189,9 @@ fi
180189
[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)"
181190
[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
182191
[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
183-
[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
192+
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
193+
[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
194+
fi
184195

185196
# Here, we try to do some additional analysis on the output file
186197
# to see if we can print a more clear 'reason' for the failure
@@ -208,8 +219,8 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
208219
[[ ${FAILED} -eq 0 ]] && \
209220
[[ ${MISSING} -eq 0 ]] && \
210221
[[ ${NO_MISSING} -eq 1 ]] && \
211-
[[ ${TGZ} -eq 1 ]] && \
212-
[[ ! -z ${TARBALL} ]]; then
222+
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \
223+
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then
213224
# SUCCESS
214225
status="SUCCESS"
215226
reason=""
@@ -417,140 +428,150 @@ success_msg="found message(s) matching <code>${GP_no_missing}</code>"
417428
failure_msg="no message matching <code>${GP_no_missing}</code>"
418429
comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
419430

420-
success_msg="found message matching <code>${GP_tgz_created}</code>"
421-
failure_msg="no message matching <code>${GP_tgz_created}</code>"
422-
comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
431+
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
432+
success_msg="found message matching <code>${GP_tgz_created}</code>"
433+
failure_msg="no message matching <code>${GP_tgz_created}</code>"
434+
comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
435+
fi
423436

424437
# Now, do the actual replacement of __DETAILS_FMT__
425438
comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
426439
comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
427440
comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
428441

429-
# first construct comment_artefacts_list
430-
# then use it to set comment_artefacts
431-
comment_artifacts_list=""
432-
433-
# TARBALL should only contain a single tarball
434-
if [[ ! -z ${TARBALL} ]]; then
435-
# Example of the detailed information for a tarball. The actual result MUST be a
436-
# single line (no '\n') or it would break the structure of the markdown table
437-
# that holds status updates of a bot job.
438-
#
439-
# <dd>
440-
# <details>
441-
# <summary><code>eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz</code></summary>
442-
# size: 234 MiB (245366784 bytes)<br/>
443-
# entries: 1234<br/>
444-
# modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_<br/>
445-
# <pre>
446-
# GCC/9.3.0.lua<br/>
447-
# GCC/10.3.0.lua<br/>
448-
# OpenSSL/1.1.lua
449-
# </pre>
450-
# software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_
451-
# <pre>
452-
# GCC/9.3.0/<br/>
453-
# CMake/3.20.1-GCCcore-10.3.0/<br/>
454-
# OpenMPI/4.1.1-GCC-10.3.0/
455-
# </pre>
456-
# other under _2023.06/software/linux/x86_64/intel/cascadelake/_
457-
# <pre>
458-
# .lmod/cache/spiderT.lua<br/>
459-
# .lmod/cache/spiderT.luac_5.1<br/>
460-
# .lmod/cache/timestamp
461-
# </pre>
462-
# </details>
463-
# </dd>
464-
size="$(stat --dereference --printf=%s ${TARBALL})"
465-
size_mib=$((${size} >> 20))
466-
tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
467-
tar tf ${TARBALL} > ${tmpfile}
468-
entries=$(cat ${tmpfile} | wc -l)
469-
# determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
470-
# e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
471-
# cfg/job.cfg contains (only the attributes to be used are shown below):
472-
# [repository]
473-
# repo_version = 2023.06
474-
# [architecture]
475-
# os_type = linux
476-
# software_subdir = x86_64/intel/skylake_avx512
477-
repo_version=$(cfg_get_value "repository" "repo_version")
478-
os_type=$(cfg_get_value "architecture" "os_type")
479-
software_subdir=$(cfg_get_value "architecture" "software_subdir")
480-
accelerator=$(cfg_get_value "architecture" "accelerator")
481-
prefix="${repo_version}/software/${os_type}/${software_subdir}"
482-
483-
# if we build for an accelerator, the prefix is different
484-
if [[ ! -z ${accelerator} ]]; then
485-
prefix="${prefix}/accel/${accelerator}"
486-
fi
487-
488-
# extract directories/entries from tarball content
489-
modules_entries=$(grep "${prefix}/modules" ${tmpfile})
490-
software_entries=$(grep "${prefix}/software" ${tmpfile})
491-
other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
492-
other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
493-
modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
494-
software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
495-
496-
artefact_summary="<summary>$(print_code_item '__ITEM__' ${TARBALL})</summary>"
442+
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
443+
# first construct comment_artefacts_list
444+
# then use it to set comment_artefacts
497445
comment_artifacts_list=""
498-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
499-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})"
500-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
501-
comment_artifacts_list="${comment_artifacts_list}<pre>"
502-
if [[ ! -z ${modules} ]]; then
503-
while IFS= read -r mod ; do
504-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${mod})"
505-
done <<< "${modules}"
506-
else
507-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
508-
fi
509-
comment_artifacts_list="${comment_artifacts_list}</pre>"
510-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
511-
comment_artifacts_list="${comment_artifacts_list}<pre>"
512-
if [[ ! -z ${software_pkgs} ]]; then
513-
while IFS= read -r sw_pkg ; do
514-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${sw_pkg})"
515-
done <<< "${software_pkgs}"
516-
else
517-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
518-
fi
519-
comment_artifacts_list="${comment_artifacts_list}</pre>"
520-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
521-
comment_artifacts_list="${comment_artifacts_list}<pre>"
522-
if [[ ! -z ${other_shortened} ]]; then
523-
while IFS= read -r other ; do
524-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${other})"
525-
done <<< "${other_shortened}"
446+
447+
# TARBALL should only contain a single tarball
448+
if [[ ! -z ${TARBALL} ]]; then
449+
# Example of the detailed information for a tarball. The actual result MUST be a
450+
# single line (no '\n') or it would break the structure of the markdown table
451+
# that holds status updates of a bot job.
452+
#
453+
# <dd>
454+
# <details>
455+
# <summary><code>eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz</code></summary>
456+
# size: 234 MiB (245366784 bytes)<br/>
457+
# entries: 1234<br/>
458+
# modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_<br/>
459+
# <pre>
460+
# GCC/9.3.0.lua<br/>
461+
# GCC/10.3.0.lua<br/>
462+
# OpenSSL/1.1.lua
463+
# </pre>
464+
# software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_
465+
# <pre>
466+
# GCC/9.3.0/<br/>
467+
# CMake/3.20.1-GCCcore-10.3.0/<br/>
468+
# OpenMPI/4.1.1-GCC-10.3.0/
469+
# </pre>
470+
# other under _2023.06/software/linux/x86_64/intel/cascadelake/_
471+
# <pre>
472+
# .lmod/cache/spiderT.lua<br/>
473+
# .lmod/cache/spiderT.luac_5.1<br/>
474+
# .lmod/cache/timestamp
475+
# </pre>
476+
# </details>
477+
# </dd>
478+
size="$(stat --dereference --printf=%s ${TARBALL})"
479+
size_mib=$((${size} >> 20))
480+
tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
481+
tar tf ${TARBALL} > ${tmpfile}
482+
entries=$(cat ${tmpfile} | wc -l)
483+
# determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
484+
# e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
485+
# cfg/job.cfg contains (only the attributes to be used are shown below):
486+
# [repository]
487+
# repo_version = 2023.06
488+
# [architecture]
489+
# os_type = linux
490+
# software_subdir = x86_64/intel/skylake_avx512
491+
repo_version=$(cfg_get_value "repository" "repo_version")
492+
os_type=$(cfg_get_value "architecture" "os_type")
493+
software_subdir=$(cfg_get_value "architecture" "software_subdir")
494+
accelerator=$(cfg_get_value "architecture" "accelerator")
495+
prefix="${repo_version}/software/${os_type}/${software_subdir}"
496+
497+
# if we build for an accelerator, the prefix is different
498+
if [[ ! -z ${accelerator} ]]; then
499+
prefix="${prefix}/accel/${accelerator}"
500+
fi
501+
502+
# extract directories/entries from tarball content
503+
modules_entries=$(grep "${prefix}/modules" ${tmpfile})
504+
software_entries=$(grep "${prefix}/software" ${tmpfile})
505+
other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
506+
other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
507+
modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
508+
software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
509+
510+
artefact_summary="<summary>$(print_code_item '__ITEM__' ${TARBALL})</summary>"
511+
comment_artifacts_list=""
512+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
513+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})"
514+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
515+
comment_artifacts_list="${comment_artifacts_list}<pre>"
516+
if [[ ! -z ${modules} ]]; then
517+
while IFS= read -r mod ; do
518+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${mod})"
519+
done <<< "${modules}"
520+
else
521+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
522+
fi
523+
comment_artifacts_list="${comment_artifacts_list}</pre>"
524+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
525+
comment_artifacts_list="${comment_artifacts_list}<pre>"
526+
if [[ ! -z ${software_pkgs} ]]; then
527+
while IFS= read -r sw_pkg ; do
528+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${sw_pkg})"
529+
done <<< "${software_pkgs}"
530+
else
531+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
532+
fi
533+
comment_artifacts_list="${comment_artifacts_list}</pre>"
534+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
535+
comment_artifacts_list="${comment_artifacts_list}<pre>"
536+
if [[ ! -z ${other_shortened} ]]; then
537+
while IFS= read -r other ; do
538+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${other})"
539+
done <<< "${other_shortened}"
540+
else
541+
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
542+
fi
543+
comment_artifacts_list="${comment_artifacts_list}</pre>"
526544
else
527-
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
545+
comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')"
528546
fi
529-
comment_artifacts_list="${comment_artifacts_list}</pre>"
530-
else
531-
comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')"
532-
fi
533547

534-
comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
535-
comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}"
536-
comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}"
548+
comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
549+
comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}"
550+
comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}"
537551

538-
comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
539-
comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}"
540-
comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}}
552+
comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
553+
comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}"
554+
comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}}
541555

542-
echo "${comment_description}" >> ${job_result_file}
556+
echo "${comment_description}" >> ${job_result_file}
543557

544-
# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts
545-
# - this should make use of subsequent steps such as deploying a tarball more
546-
# efficient
547-
echo "status = ${status}" >> ${job_result_file}
548-
echo "artefacts = " >> ${job_result_file}
549-
echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file}
558+
# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts
559+
# - this should make use of subsequent steps such as deploying a tarball more
560+
# efficient
561+
echo "status = ${status}" >> ${job_result_file}
562+
echo "artefacts = " >> ${job_result_file}
563+
echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file}
550564

551-
# remove tmpfile
552-
if [[ -f ${tmpfile} ]]; then
553-
rm ${tmpfile}
565+
# remove tmpfile
566+
if [[ -f ${tmpfile} ]]; then
567+
rm ${tmpfile}
568+
fi
569+
570+
elif [[ -f "$TOPDIR/check-build-artefacts.sh" ]]; then
571+
source "$TOPDIR/check-build-artefacts.sh"
572+
else
573+
echo "ERROR: Required script $TOPDIR/check-build-artefacts.sh not found!" >&2
574+
exit 1
554575
fi
555576

556577
# exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1)

0 commit comments

Comments
 (0)