From 2e9458d435647a6dd8d95fcd92b83df12a3b5a17 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Thu, 24 Jul 2025 20:32:07 +0200 Subject: [PATCH 1/6] Initial changes for the ICPC World Finals in Baku * Install php-bcmath extension * Copy the pc2packages key to the Ubuntu archive keyring * Install openjdk 21 * Copy kotlin folder instead of using old apt package * Use correct repo URL * Do not use cgroupv1 --- .../roles/base_packages/defaults/main.yml | 1 + .../ansible/roles/judgedaemon/tasks/main.yml | 51 +++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/provision-contest/ansible/roles/base_packages/defaults/main.yml b/provision-contest/ansible/roles/base_packages/defaults/main.yml index 90f1ef2b..6a552323 100644 --- a/provision-contest/ansible/roles/base_packages/defaults/main.yml +++ b/provision-contest/ansible/roles/base_packages/defaults/main.yml @@ -42,6 +42,7 @@ INSTALLED_PACKAGES: - php-mysql - php-xml - php-zip + - php-bcmath - pv - python3-sphinx - python3-sphinx-rtd-theme diff --git a/provision-contest/ansible/roles/judgedaemon/tasks/main.yml b/provision-contest/ansible/roles/judgedaemon/tasks/main.yml index e585b7e2..fb56e1eb 100644 --- a/provision-contest/ansible/roles/judgedaemon/tasks/main.yml +++ b/provision-contest/ansible/roles/judgedaemon/tasks/main.yml @@ -33,26 +33,69 @@ src: chroot-list dest: /tmp/dj_ansible/ +- name: Check if pc2packages key is already in ubuntu-archive-keyring.gpg + ansible.builtin.shell: | + gpg --no-default-keyring --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg --list-keys --with-colons | \ + grep -q "$(gpg --show-keys --with-colons /etc/apt/trusted.gpg.d/pc2packages.asc | grep '^fpr:' | head -1 | cut -d: -f10)" + register: key_exists + failed_when: false + changed_when: false + check_mode: false + when: ICPC_IMAGE + +- name: Add pc2packages key to ubuntu-archive-keyring.gpg + ansible.builtin.shell: | + gpg --no-default-keyring --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg --import /etc/apt/trusted.gpg.d/pc2packages.asc + when: + - ICPC_IMAGE + - key_exists.rc != 0 + - name: Create chroot shell: "set -o pipefail && {{ DJ_DIR }}/misc-tools/dj_make_chroot -y -H - -i icpc-kotlinc,openjdk-17-jdk-headless + -i openjdk-21-jdk-headless -l \"$(ls /tmp/dj_ansible/install-chroot/*.deb 2>/dev/null | tr '\n' ',')\" -s \"$(ls /tmp/dj_ansible/chroot-list/*.list 2>/dev/null | tr '\n' ',')\" 2>&1 | tee /tmp/dj_make_chroot.log; grep '^Done building chroot in' /tmp/dj_make_chroot.log" environment: - DEBMIRROR: "{%- if WF_RESTRICTED_NETWORK and ICPC_IMAGE -%}https://packages/ubuntu - {%- elif ICPC_IMAGE -%}https://sysopspackages.icpc.global/ubuntu + DEBMIRROR: "{%- if WF_RESTRICTED_NETWORK and ICPC_IMAGE -%}https://packages/ubuntu-noble + {%- elif ICPC_IMAGE -%}https://sysopspackages.icpc.global/ubuntu-noble {%- else -%} {%- endif -%}" args: executable: /bin/bash creates: "/chroot/domjudge" +- name: Create kotlinc directory in chroot + ansible.builtin.file: + path: /chroot/domjudge/usr/lib/kotlinc + state: directory + mode: '0755' + when: ICPC_IMAGE + +- name: Sync kotlinc folder to chroot + ansible.posix.synchronize: + src: /opt/kotlinc/ + dest: /chroot/domjudge/usr/lib/kotlinc/ + delete: false + recursive: true + delegate_to: "{{ inventory_hostname }}" + when: ICPC_IMAGE + +- name: Create symlinks for kotlin binaries in chroot + ansible.builtin.file: + src: ../../lib/kotlinc/bin/{{ item }} + dest: /chroot/domjudge/usr/local/bin/{{ item }} + state: link + loop: + - kotlin + - kotlinc + when: ICPC_IMAGE + - name: Pre-generate the kernel flags for ansible usage set_fact: - procline: "apparmor=0 systemd.unified_cgroup_hierarchy=0 cgroup_enable=memory swapaccount=1 isolcpus={{ cpucore | join(',') }}" + procline: "apparmor=0 cgroup_enable=memory swapaccount=1 isolcpus={{ cpucore | join(',') }}" - name: Add cgroup kernel parameters lineinfile: From 5320220241ba5f092a77b8691e8ac36a9f78cf97 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Wed, 30 Jul 2025 18:45:26 +0200 Subject: [PATCH 2/6] Store disabled CPUs We will use those later for a report and to not duplicate the method of disabling. --- provision-contest/disable-turboboost_ht | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/provision-contest/disable-turboboost_ht b/provision-contest/disable-turboboost_ht index ce41e815..07f6d792 100755 --- a/provision-contest/disable-turboboost_ht +++ b/provision-contest/disable-turboboost_ht @@ -4,6 +4,13 @@ set -eu -o pipefail shopt -s extglob declare -A core_ids +declare -a disabled_cores + +disable_cpu () { + cpu="$1" + echo 0 > $cpu/online + disabled_cores+=("${cpu##*/}") +} # shellcheck disable=SC2012 for cpu in $(ls -1d /sys/devices/system/cpu/cpu* | sort --version-sort) ; do @@ -41,7 +48,7 @@ for cpu in $(ls -1d /sys/devices/system/cpu/cpu* | sort --version-sort) ; do # 'delimiter'. core_id=$(cat $cpu/topology/core_id | tr -d '\n')'-'$(cat $cpu/topology/physical_package_id | tr -d '\n') if [[ ${core_ids[$core_id]:-} ]]; then - echo 0 > $cpu/online + disable_cpu $cpu else core_ids[$core_id]=1 fi From 6dea67eae9f5d28ac4f9e0f395a03ecfa7c9de39 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Wed, 30 Jul 2025 19:13:44 +0200 Subject: [PATCH 3/6] Store the isolated cores for ansible This is defence in depth by disallowing the kernel to schedule work on the online cpu's before the script disables the CPU's. The script can fail when the CPU taken down is currently working on something. --- provision-contest/disable-turboboost_ht | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/provision-contest/disable-turboboost_ht b/provision-contest/disable-turboboost_ht index 07f6d792..1d6739b5 100755 --- a/provision-contest/disable-turboboost_ht +++ b/provision-contest/disable-turboboost_ht @@ -12,6 +12,8 @@ disable_cpu () { disabled_cores+=("${cpu##*/}") } +store_isolcpus_fact="/var/tmp/isolcpus" + # shellcheck disable=SC2012 for cpu in $(ls -1d /sys/devices/system/cpu/cpu* | sort --version-sort) ; do [[ $(basename $cpu) =~ ^cpu[0-9]+$ ]] || continue @@ -54,6 +56,11 @@ for cpu in $(ls -1d /sys/devices/system/cpu/cpu* | sort --version-sort) ; do fi done +if [ -n "${store_isolcpus_fact}" ]; then + csv_string=$(IFS=, ; echo "${disabled_cores[*]}") + echo "$csv_string" > "$store_isolcpus_fact" +fi + DIR_INTEL=/sys/devices/system/cpu/intel_pstate DIR_AMD=/sys/devices/system/cpu/cpufreq if [ -d $DIR_INTEL ]; then From ad7f7229a22df4b1901d22e50add5f4a7632c29f Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Wed, 30 Jul 2025 18:55:32 +0200 Subject: [PATCH 4/6] Clarify the package/socket usage Package is more correct but socket is better known. Also fixed a typo. --- provision-contest/disable-turboboost_ht | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/provision-contest/disable-turboboost_ht b/provision-contest/disable-turboboost_ht index 1d6739b5..f077ee5c 100755 --- a/provision-contest/disable-turboboost_ht +++ b/provision-contest/disable-turboboost_ht @@ -41,10 +41,10 @@ for cpu in $(ls -1d /sys/devices/system/cpu/cpu* | sort --version-sort) ; do chmod a-w $cpu/cpufreq/scaling_{max,min}_freq fi - # Disable all but one thread on each core. Both core_id and physical_package_id are - # numbers it must be ensured that for the following examples are seen as distinct: + # Disable all but one thread on each core per socket. Both core_id and physical_package_id + # are numbers so it must be ensured that for the following examples are seen as distinct: # - core_id=1, physical_package=11 - # - core_id=11, physycal_package=1 + # - core_id=11, physical_package=1 # Simple concatenation would result in the string '111' for both cores. Though `cat` # adds a newline after each file, we do not want to rely on `cat` to always add this # 'delimiter'. From b3a9d8d99bb8473647683b25b69ab4cec5543633 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Tue, 29 Jul 2025 22:14:33 +0200 Subject: [PATCH 5/6] Consider ARM processors The boost exists for asahi@M1 laptops but not on a raspberry with ARMv7. --- provision-contest/disable-turboboost_ht | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/provision-contest/disable-turboboost_ht b/provision-contest/disable-turboboost_ht index f077ee5c..ef535ee1 100755 --- a/provision-contest/disable-turboboost_ht +++ b/provision-contest/disable-turboboost_ht @@ -63,6 +63,8 @@ fi DIR_INTEL=/sys/devices/system/cpu/intel_pstate DIR_AMD=/sys/devices/system/cpu/cpufreq +# Same for some ARM CPUs +FILE_AMD=$DIR_AMD/boost if [ -d $DIR_INTEL ]; then # now disable turbo boost FILE=$DIR_INTEL/no_turbo @@ -75,11 +77,10 @@ if [ -d $DIR_INTEL ]; then # increase freq from powersaving to normal, but don't overclock echo 100 > $DIR_INTEL/min_perf_pct echo 100 > $DIR_INTEL/max_perf_pct -elif [ -d $DIR_AMD ]; then +elif [ -f $FILE_AMD ]; then # now disable boosting - FILE=$DIR_AMD/boost - echo -n 0 > $FILE || echo "Could not write to '$FILE', ignoring for now..." - if [ $(cat $FILE) -ne 0 ]; then + echo -n 0 > $FILE_AMD || echo "Could not write to '$FILE_AMD', ignoring for now..." + if [ $(cat $FILE_AMD) -ne 0 ]; then echo "Error: turboboost still enabled!" exit 1 fi From 4a523b88452008a5f8930c1485c075669ce98eaf Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Tue, 29 Jul 2025 21:40:06 +0200 Subject: [PATCH 6/6] Disable efficiency cores Also clarified the package/socket as package is more correct but socket is better known and fixed a typo. --- provision-contest/disable-turboboost_ht | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/provision-contest/disable-turboboost_ht b/provision-contest/disable-turboboost_ht index ef535ee1..bca9112c 100755 --- a/provision-contest/disable-turboboost_ht +++ b/provision-contest/disable-turboboost_ht @@ -14,6 +14,14 @@ disable_cpu () { store_isolcpus_fact="/var/tmp/isolcpus" +# Learn all efficency cores if they exist +cpu_list=() +if [ -f /sys/devices/cpu_atom/cpus ]; then + range="$(cat /sys/devices/cpu_atom/cpus)" + IFS='-' read -r start end <<< "$range" + cpu_list=($(eval echo {$start..$end})) +fi + # shellcheck disable=SC2012 for cpu in $(ls -1d /sys/devices/system/cpu/cpu* | sort --version-sort) ; do [[ $(basename $cpu) =~ ^cpu[0-9]+$ ]] || continue @@ -49,6 +57,20 @@ for cpu in $(ls -1d /sys/devices/system/cpu/cpu* | sort --version-sort) ; do # adds a newline after each file, we do not want to rely on `cat` to always add this # 'delimiter'. core_id=$(cat $cpu/topology/core_id | tr -d '\n')'-'$(cat $cpu/topology/physical_package_id | tr -d '\n') + + # Disable all efficiency cores + found=0 + for efficiency_core in "${cpu_list[@]}"; do + if [[ "$cpu" == "/sys/devices/system/cpu/cpu$efficiency_core" ]]; then + disable_cpu $cpu + found=1 + break + fi + done + if [ "$found" -eq "1" ]; then + continue + fi + if [[ ${core_ids[$core_id]:-} ]]; then disable_cpu $cpu else