From d3dd94582305ceb4bfd2a31f5a8894849788567f Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Thu, 27 Nov 2025 19:40:20 +0100 Subject: [PATCH 01/13] Set more grep-able passwords in CI jobs, drop domjudge user The domjudge mysql user should be created by our setup scripts, so that we test these and need to set the password only in one place. Rename mysql_root to mysql_log helper to clarify behaviour. --- .github/jobs/baseinstall.sh | 52 ++++++++++++-------------- .github/jobs/ci_settings.sh | 9 +---- .github/workflows/database-upgrade.yml | 14 +++---- .github/workflows/integration.yml | 8 ++-- .github/workflows/unit-tests.yml | 6 +-- .github/workflows/webstandard.yml | 6 +-- 6 files changed, 42 insertions(+), 53 deletions(-) mode change 100644 => 100755 .github/jobs/ci_settings.sh diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index 719f065608..ec5cb31d65 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -11,8 +11,6 @@ export APP_ENV="${4:-prod}" # In the test environment, we need to use a different database [ "$APP_ENV" = "prod" ] && DATABASE_NAME=domjudge || DATABASE_NAME=domjudge_test -MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root} - set -euxo pipefail if [ -z "$phpversion" ]; then @@ -74,25 +72,22 @@ EOF cat ~/.my.cnf # TODO: Remove after fixing https://github.com/DOMjudge/domjudge/issues/2848 -mysql_root "SET GLOBAL innodb_snapshot_isolation = OFF;" +mysql_log "SET GLOBAL innodb_snapshot_isolation = OFF;" -mysql_root "CREATE DATABASE IF NOT EXISTS \`$DATABASE_NAME\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" -mysql_root "CREATE USER IF NOT EXISTS \`domjudge\`@'%' IDENTIFIED BY 'domjudge';" -mysql_root "GRANT SELECT, INSERT, UPDATE, DELETE ON \`$DATABASE_NAME\`.* TO 'domjudge'@'%';" -mysql_root "FLUSH PRIVILEGES;" +mysql_log "CREATE DATABASE IF NOT EXISTS \`$DATABASE_NAME\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" +mysql_log "CREATE USER IF NOT EXISTS \`domjudge\`@'%' IDENTIFIED BY 'domjudge';" +mysql_log "GRANT SELECT, INSERT, UPDATE, DELETE ON \`$DATABASE_NAME\`.* TO 'domjudge'@'%';" +mysql_log "FLUSH PRIVILEGES;" +echo "unused:sqlserver:$DATABASE_NAME:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret # Show some MySQL debugging -mysql_root "show databases" -mysql_root "SELECT CURRENT_USER();" -mysql_root "SELECT USER();" -mysql_root "SELECT user,host FROM mysql.user" -mysql_root "SET GLOBAL max_allowed_packet=1073741824" -mysql_root "SHOW GLOBAL STATUS LIKE 'Connection_errors_%'" -mysql_root "SHOW VARIABLES LIKE 'innodb_snapshot_isolation'" -mysql_root "SHOW VARIABLES LIKE '%_timeout'" -echo "unused:sqlserver:$DATABASE_NAME:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret -mysql_user "SELECT CURRENT_USER();" -mysql_user "SELECT USER();" +mysql_log "show databases" +mysql_log "SELECT CURRENT_USER();" +mysql_log "SELECT USER();" +mysql_log "SELECT user,host FROM mysql.user" +mysql_log "SET GLOBAL max_allowed_packet=1073741824" +mysql_log "SHOW GLOBAL STATUS LIKE 'Connection_errors_%'" +mysql_log "SHOW VARIABLES LIKE '%_timeout'" section_end if [ "${db}" = "install" ]; then @@ -102,6 +97,7 @@ if [ "${db}" = "install" ]; then elif [ "${db}" = "upgrade" ]; then section_start "Upgrade DOMjudge database" /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} upgrade + /opt/domjudge/domserver/webapp/bin/console domjudge:reset-user-password admin "pass" section_end fi @@ -141,7 +137,7 @@ if [ "${db}" = "install" ]; then section_start "Install the example data" if [ "$version" = "unit" ]; then # Make sure admin has no team associated so we will not insert submissions during unit tests. - mysql_root "UPDATE user SET teamid=null WHERE userid=1;" $DATABASE_NAME + mysql_log "UPDATE user SET teamid=null WHERE userid=1;" $DATABASE_NAME fi /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples | tee -a "$ARTIFACTS/mysql.txt" section_end @@ -149,23 +145,23 @@ fi section_start "Setup user" # We're using the admin user in all possible roles -mysql_root "DELETE FROM userrole WHERE userid=1;" $DATABASE_NAME +mysql_log "DELETE FROM userrole WHERE userid=1;" $DATABASE_NAME if [ "$version" = "team" ]; then # Add team to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME - mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME + mysql_log "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME + mysql_log "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME elif [ "$version" = "jury" ]; then # Add jury to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" $DATABASE_NAME + mysql_log "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" $DATABASE_NAME elif [ "$version" = "balloon" ]; then # Add balloon to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" $DATABASE_NAME + mysql_log "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" $DATABASE_NAME elif [ "$version" = "admin" ]; then # Add admin to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME + mysql_log "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME elif [ "$version" = "all" ] || [ "$version" = "unit" ]; then - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME - mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME + mysql_log "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME + mysql_log "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME + mysql_log "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME fi section_end diff --git a/.github/jobs/ci_settings.sh b/.github/jobs/ci_settings.sh old mode 100644 new mode 100755 index 391996752b..5a49359f84 --- a/.github/jobs/ci_settings.sh +++ b/.github/jobs/ci_settings.sh @@ -24,14 +24,9 @@ section_end_internal () { trace_on } -mysql_root () { +mysql_log () { # shellcheck disable=SC2086 - echo "$1" | mysql -uroot -proot ${2:-} | tee -a "$ARTIFACTS"/mysql.txt -} - -mysql_user () { - # shellcheck disable=SC2086 - echo "$1" | mysql -udomjudge -pdomjudge ${2:-} | tee -a "$ARTIFACTS"/mysql.txt + echo "$1" | mysql -uroot -p${MYSQL_ROOT_PASSWORD} ${2:-} | tee -a "$ARTIFACTS"/mysql.txt } show_phpinfo() { diff --git a/.github/workflows/database-upgrade.yml b/.github/workflows/database-upgrade.yml index 251844a882..6c3cbf0cc9 100644 --- a/.github/workflows/database-upgrade.yml +++ b/.github/workflows/database-upgrade.yml @@ -9,6 +9,8 @@ on: jobs: upgrade_test: runs-on: ubuntu-latest + env: + MYSQL_ROOT_PASSWORD: mysql_root_password container: image: domjudge/gitlabci:24.04 services: @@ -17,9 +19,7 @@ jobs: ports: - 3306:3306 env: - MYSQL_ROOT_PASSWORD: root - MYSQL_USER: domjudge - MYSQL_PASSWORD: domjudge + MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }} options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - uses: actions/checkout@v4 @@ -38,18 +38,16 @@ jobs: restore-keys: | composer- - name: Import Database - run: mysql -hsqlserver -uroot -proot < .github/jobs/data/dj733.sql + run: mysql -hsqlserver -uroot -p${{ env.MYSQL_ROOT_PASSWORD }} < .github/jobs/data/dj733.sql - name: Upgrade DOMjudge run: .github/jobs/baseinstall.sh default upgrade - - name: Setting initial Admin Password - run: echo "pass" > /opt/domjudge/domserver/etc/initial_admin_password.secret - name: Check for Errors in the Upgrade - run: mysql -hsqlserver -uroot -proot -e "SHOW TABLES FROM domjudge;" + run: mysql -hsqlserver -uroot -p${{ env.MYSQL_ROOT_PASSWORD }} -e "SHOW TABLES FROM domjudge;" - name: Check for Errors in DOMjudge Webinterface run: .github/jobs/webstandard.sh none admin - name: dump the db if: ${{ !cancelled() }} - run: mysqldump -uroot -proot --quick --max_allowed_packet=1024M domjudge > /tmp/db.sql + run: mysqldump -hsqlserver -uroot -p${{ env.MYSQL_ROOT_PASSWORD }} --quick --max_allowed_packet=1024M domjudge > /tmp/db.sql - name: Upload database dump for debugging if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0fdadf5350..e8e4b97d6c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,6 +9,8 @@ on: jobs: integration: runs-on: ubuntu-24.04 + env: + MYSQL_ROOT_PASSWORD: mysql_root_password timeout-minutes: 30 container: image: domjudge/gitlabci:24.04 @@ -19,9 +21,7 @@ jobs: ports: - 3306:3306 env: - MYSQL_ROOT_PASSWORD: root - MYSQL_USER: domjudge - MYSQL_PASSWORD: domjudge + MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }} options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - uses: actions/checkout@v4 @@ -95,7 +95,7 @@ jobs: done - name: dump the db if: ${{ !cancelled() }} - run: mysqldump -uroot -proot --quick --max_allowed_packet=1024M domjudge > /tmp/db.sql + run: mysqldump -uroot -p${{ env.MYSQL_ROOT_PASSWORD }} --quick --max_allowed_packet=1024M domjudge > /tmp/db.sql - name: Upload database dump for debugging if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d6d53f8712..c65b35c374 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -28,6 +28,8 @@ jobs: permissions: checks: write runs-on: ubuntu-24.04 + env: + MYSQL_ROOT_PASSWORD: mysql_root_password timeout-minutes: 30 container: image: domjudge/gitlabci:24.04 @@ -37,9 +39,7 @@ jobs: ports: - 3306:3306 env: - MYSQL_ROOT_PASSWORD: root - MYSQL_USER: domjudge - MYSQL_PASSWORD: domjudge + MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }} options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 strategy: matrix: diff --git a/.github/workflows/webstandard.yml b/.github/workflows/webstandard.yml index 165a70596b..00b8e746bd 100644 --- a/.github/workflows/webstandard.yml +++ b/.github/workflows/webstandard.yml @@ -9,6 +9,8 @@ on: jobs: standards: runs-on: ubuntu-latest + env: + MYSQL_ROOT_PASSWORD: mysql_root_password container: image: domjudge/gitlabci:24.04 services: @@ -17,9 +19,7 @@ jobs: ports: - 3306:3306 env: - MYSQL_ROOT_PASSWORD: root - MYSQL_USER: domjudge - MYSQL_PASSWORD: domjudge + MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }} options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 strategy: matrix: From 88ba0bc1d83ba01b82016dc72325c711b3d56e77 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Thu, 27 Nov 2025 19:41:17 +0100 Subject: [PATCH 02/13] Set more greppable admin password --- .github/jobs/baseinstall.sh | 8 ++++---- .github/jobs/webstandard.sh | 5 ++--- .github/workflows/database-upgrade.yml | 1 + .github/workflows/integration.yml | 10 +++++----- .github/workflows/unit-tests.yml | 1 + .github/workflows/webstandard.yml | 1 + 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index ec5cb31d65..b1653e4ab8 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -32,9 +32,9 @@ yarnpkg install cd .. section_end -section_start "Set simple admin password" -echo "password" > ./etc/initial_admin_password.secret -echo "default login admin password password" > ~/.netrc +section_start "Set admin password" +echo "${ADMIN_PASSWORD}" > ./etc/initial_admin_password.secret +echo "default login admin password ${ADMIN_PASSWORD}" > ~/.netrc section_end section_start "Install domserver" @@ -97,7 +97,7 @@ if [ "${db}" = "install" ]; then elif [ "${db}" = "upgrade" ]; then section_start "Upgrade DOMjudge database" /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} upgrade - /opt/domjudge/domserver/webapp/bin/console domjudge:reset-user-password admin "pass" + /opt/domjudge/domserver/webapp/bin/console domjudge:reset-user-password admin "${ADMIN_PASSWORD}" section_end fi diff --git a/.github/jobs/webstandard.sh b/.github/jobs/webstandard.sh index a48fd70f77..4a5331398c 100755 --- a/.github/jobs/webstandard.sh +++ b/.github/jobs/webstandard.sh @@ -20,19 +20,18 @@ section_start "Setup pa11y" section_end section_start "Setup the test user" -ADMINPASS=$(cat etc/initial_admin_password.secret) export COOKIEJAR COOKIEJAR=$(mktemp --tmpdir) export CURLOPTS="--fail -sq -m 30 -b $COOKIEJAR" if [ "$ROLE" = "public" ]; then - ADMINPASS="failedlogin" + ADMIN_PASSWORD="failedlogin" fi # Make an initial request which will get us a session id, and grab the csrf token from it CSRFTOKEN=$(curl $CURLOPTS -c $COOKIEJAR "http://localhost/domjudge/login" 2>/dev/null | sed -n 's/.*_csrf_token.*value="\(.*\)".*/\1/p') # Make a second request with our session + csrf token to actually log in # shellcheck disable=SC2086 -curl $CURLOPTS -c "$COOKIEJAR" -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F "_password=$ADMINPASS" "http://localhost/domjudge/login" +curl $CURLOPTS -c "$COOKIEJAR" -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F "_password=$ADMIN_PASSWORD" "http://localhost/domjudge/login" # Move back to the default directory cd "$DIR" diff --git a/.github/workflows/database-upgrade.yml b/.github/workflows/database-upgrade.yml index 6c3cbf0cc9..57833acf39 100644 --- a/.github/workflows/database-upgrade.yml +++ b/.github/workflows/database-upgrade.yml @@ -10,6 +10,7 @@ jobs: upgrade_test: runs-on: ubuntu-latest env: + ADMIN_PASSWORD: admin_password MYSQL_ROOT_PASSWORD: mysql_root_password container: image: domjudge/gitlabci:24.04 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e8e4b97d6c..1f60a2c6f3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,6 +10,7 @@ jobs: integration: runs-on: ubuntu-24.04 env: + ADMIN_PASSWORD: admin_password MYSQL_ROOT_PASSWORD: mysql_root_password timeout-minutes: 30 container: @@ -57,11 +58,10 @@ jobs: - name: Check nginx run: curl -v https://localhost/domjudge/ - name: Configure print command - working-directory: submit run: | - curl --fail -u 'admin:password' -X 'GET' 'http://localhost/domjudge/api/v4/config?strict=false' \ + curl --fail -u "admin:${ADMIN_PASSWORD}" -X 'GET' 'http://localhost/domjudge/api/v4/config?strict=false' \ | jq '.print_command |= "cp [file] /tmp/dj-printfile"' \ - | curl --fail -u 'admin:password' -X 'PUT' -T - 'http://localhost/domjudge/api/v4/config?strict=false' \ + | curl --fail -u "admin:${ADMIN_PASSWORD}" -X 'PUT' -T - 'http://localhost/domjudge/api/v4/config?strict=false' \ - name: Testing submit client working-directory: submit run: make check-full @@ -127,7 +127,7 @@ jobs: # Make an initial request which will get us a session id, and grab the csrf token from it CSRFTOKEN=$(curl $CURLOPTS -c /tmp/cookiejar "http://localhost/domjudge/login" | sed -n 's/.*_csrf_token.*value="\(.*\)".*/\1/p') # Make a second request with our session + csrf token to actually log in - curl $CURLOPTS -c /tmp/cookiejar -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F "_password=password" "http://localhost/domjudge/login" + curl $CURLOPTS -c /tmp/cookiejar -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F "_password=${ADMIN_PASSWORD}" "http://localhost/domjudge/login" # Send a general clarification to later test if we see the event. curl $CURLOPTS -F "sendto=" -F "problem=1-" -F "bodytext=Testing" -F "submit=Send" \ "http://localhost/domjudge/jury/clarifications/send" -o /dev/null @@ -172,4 +172,4 @@ jobs: export CCS_SPECS_PINNED_SHA1='a68aff54c4e60fc2bff2fc5c36c119bffa4d30f1' ( cd ccs-specs && git reset --hard $CCS_SPECS_PINNED_SHA1 ) export CHECK_API="${HOME}/ccs-specs/check-api.sh -j ${HOME}/yajsv" - $CHECK_API -n -C -e -a 'strict=1' http://admin:password@localhost/domjudge/api + $CHECK_API -n -C -e -a 'strict=1' "http://admin:${ADMIN_PASSWORD}@localhost/domjudge/api" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c65b35c374..c966aaadf8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -29,6 +29,7 @@ jobs: checks: write runs-on: ubuntu-24.04 env: + ADMIN_PASSWORD: admin_password MYSQL_ROOT_PASSWORD: mysql_root_password timeout-minutes: 30 container: diff --git a/.github/workflows/webstandard.yml b/.github/workflows/webstandard.yml index 00b8e746bd..ab1343f2c0 100644 --- a/.github/workflows/webstandard.yml +++ b/.github/workflows/webstandard.yml @@ -10,6 +10,7 @@ jobs: standards: runs-on: ubuntu-latest env: + ADMIN_PASSWORD: admin_password MYSQL_ROOT_PASSWORD: mysql_root_password container: image: domjudge/gitlabci:24.04 From fe688f260d697af976c2fe9e7074bdd2ff9bef8e Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Fri, 28 Nov 2025 10:14:54 +0100 Subject: [PATCH 03/13] Use mysql credentials from `~/.my.cnf` --- .github/jobs/baseinstall.sh | 6 +++--- .github/jobs/ci_settings.sh | 2 +- .github/workflows/integration.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index b1653e4ab8..c1764bae53 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -92,11 +92,11 @@ section_end if [ "${db}" = "install" ]; then section_start "Install DOMjudge database" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} bare-install + /opt/domjudge/domserver/bin/dj_setup_database bare-install section_end elif [ "${db}" = "upgrade" ]; then section_start "Upgrade DOMjudge database" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} upgrade + /opt/domjudge/domserver/bin/dj_setup_database upgrade /opt/domjudge/domserver/webapp/bin/console domjudge:reset-user-password admin "${ADMIN_PASSWORD}" section_end fi @@ -139,7 +139,7 @@ if [ "${db}" = "install" ]; then # Make sure admin has no team associated so we will not insert submissions during unit tests. mysql_log "UPDATE user SET teamid=null WHERE userid=1;" $DATABASE_NAME fi - /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples | tee -a "$ARTIFACTS/mysql.txt" + /opt/domjudge/domserver/bin/dj_setup_database install-examples | tee -a "$ARTIFACTS/mysql.txt" section_end fi diff --git a/.github/jobs/ci_settings.sh b/.github/jobs/ci_settings.sh index 5a49359f84..b807178e67 100755 --- a/.github/jobs/ci_settings.sh +++ b/.github/jobs/ci_settings.sh @@ -26,7 +26,7 @@ section_end_internal () { mysql_log () { # shellcheck disable=SC2086 - echo "$1" | mysql -uroot -p${MYSQL_ROOT_PASSWORD} ${2:-} | tee -a "$ARTIFACTS"/mysql.txt + echo "$1" | mysql ${2:-} | tee -a "$ARTIFACTS"/mysql.txt } show_phpinfo() { diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1f60a2c6f3..ec5fc282d6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -95,7 +95,7 @@ jobs: done - name: dump the db if: ${{ !cancelled() }} - run: mysqldump -uroot -p${{ env.MYSQL_ROOT_PASSWORD }} --quick --max_allowed_packet=1024M domjudge > /tmp/db.sql + run: mysqldump --quick --max_allowed_packet=1024M domjudge > /tmp/db.sql - name: Upload database dump for debugging if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 From 98dc0e5c43ffc12a5e08e10560f4b0d54786f5a4 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 25 Aug 2024 15:44:23 +0200 Subject: [PATCH 04/13] URL encode variables that go into a MySQL credentials URI See for reference: - https://dev.mysql.com/doc/refman/8.0/en/connecting-using-uri-or-key-value-pairs.html#connecting-using-uri - https://symfony.com/doc/current/doctrine.html but note that we must use `rawurlencode` instead of `urlencode` which differ in how they encode a space (as tested). Fixes: #2651 Closes: #2502 as this is likely fixed but I couldn't reproduce it --- webapp/config/load_db_secrets.php | 6 +++++- webapp/config/packages/doctrine.yaml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/webapp/config/load_db_secrets.php b/webapp/config/load_db_secrets.php index ea4ca11b83..c3a8b3dfb5 100644 --- a/webapp/config/load_db_secrets.php +++ b/webapp/config/load_db_secrets.php @@ -36,7 +36,11 @@ function get_db_url(): string break; } - return sprintf('mysql://%s:%s@%s:%d/%s?serverVersion=5.7.0', $user, $pass, $host, $port ?? 3306, $db); + return sprintf( + 'mysql://%s:%s@%s:%d/%s?serverVersion=5.7.0', + rawurlencode($user), rawurlencode($pass), rawurlencode($host), + $port ?? 3306, rawurlencode($db) + ); } function get_app_secret(): string diff --git a/webapp/config/packages/doctrine.yaml b/webapp/config/packages/doctrine.yaml index 89bf0223b3..b919a7bd43 100644 --- a/webapp/config/packages/doctrine.yaml +++ b/webapp/config/packages/doctrine.yaml @@ -7,7 +7,7 @@ doctrine: charset: utf8mb4 collate: utf8mb4_unicode_ci - url: '%env(resolve:DATABASE_URL)%' + url: '%env(DATABASE_URL)%' profiling_collect_backtrace: '%kernel.debug%' types: tinyint: App\Doctrine\DBAL\Types\TinyIntType From b997b2b9d87946d0dd12aa53149c050b25d6d27b Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Fri, 22 Nov 2024 18:12:28 +0100 Subject: [PATCH 05/13] Fixes for supporting passwords with weird characters Use URL encoding in DATABASE_URL and return mysql_options as an array (via ugly global variable), so each element in it can be separately added to the command line using `@` for expansion. Because of changing the script to bash, also reverts some of the changes in 483200a7feb22f28b64c9fda72c. The changed escaping of `DoctrineMigrations\\Version` passed to mysql is purely due to changing from `/bin/sh` to `/bin/bash`. Somehow passing that to the `mysql` wrapper function, it got unescaped before, even though `/bin/sh` is just a symlink to bash... --- sql/dj_setup_database.in | 51 +++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index fc803939c3..9b5e35cf6d 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # @configure_input@ # This script allows one to perform DOMjudge database setup actions. @@ -52,30 +52,50 @@ not have to pass any of the options above. EOF } +urlencode() +{ + # We need to escape for PHP ' and \ by prefixing them with a \. + local str="${1//\\/\\\\}" + str="${str//\'/\\\'}" + php -r "echo rawurlencode('$str');" +} + +# This is global variable to be able to return the output from +# mysql_options() below as an array, which is not possible otherwise. +declare -a _mysql_options + mysql_options() { + local user pass + _mysql_options=() + # shellcheck disable=SC2153 if [ -n "$DBUSER" ]; then - _user="-u $DBUSER" - else - _user="${DBA_USER:+-u ${DBA_USER}}" + _mysql_options+=('-u' "$DBUSER") + elif [ -n "$DBA_USER" ]; then + _mysql_options+=('-u' "$DBA_USER") fi # shellcheck disable=SC2153 if [ -n "$PASSWD" ]; then - _pass="-p$PASSWD" - else - [ -n "$PROMPT_PASSWD" ] && _pass="-p" - [ -n "$DBA_PASSWD" ] && _pass="-p$DBA_PASSWD" + _mysql_options+=("-p$PASSWD") + elif [ -n "$DBA_PASSWD" ]; then + _mysql_options+=("-p$DBA_PASSWD") + elif [ -n "$PROMPT_PASSWD" ]; then + _mysql_options+=('-p') fi - [ -z "$USE_SOCKET" ] && port="-P$DBPORT" - echo $_user ${_pass:+"$_pass"} -h "$DBHOST" ${port:+"$port"} + _mysql_options+=('-h' "$DBHOST") + + if [ -z "$USE_SOCKET" ]; then + _mysql_options+=("-P$DBPORT") + fi } # Wrapper around mysql command to allow setting options, user, etc. mysql() { - command mysql $(mysql_options) --silent --skip-column-names "$@" + mysql_options + command mysql "${_mysql_options[@]}" --silent --skip-column-names "$@" } # Quick shell hack to get a key from an INI file. @@ -126,10 +146,13 @@ symfony_console() fi if [ -n "$DBA_USER" ]; then + user=$(urlencode "${DBA_USER}") + host=$(urlencode "${domjudge_DBHOST}") + db=$(urlencode "${domjudge_DBNAME}") if [ -n "$DBA_PASSWD" ]; then - DATABASE_URL=mysql://${DBA_USER}:${DBA_PASSWD}@${domjudge_DBHOST}:${domjudge_DBPORT}/${domjudge_DBNAME} + DATABASE_URL="mysql://$user:$(urlencode "${DBA_PASSWD}")@$host:${domjudge_DBPORT}/$db" else - DATABASE_URL=mysql://${DBA_USER}@${domjudge_DBHOST}:${domjudge_DBPORT}/${domjudge_DBNAME} + DATABASE_URL="mysql://$user@$host:${domjudge_DBPORT}/$db" fi fi fi @@ -356,7 +379,7 @@ upgrade) # shellcheck disable=SC2016,SC2028 echo 'INSERT INTO `doctrine_migration_versions` (version, executed_at, execution_time) - SELECT concat("DoctrineMigrations\\\\Version", version), executed_at, 1 + SELECT concat("DoctrineMigrations\\Version", version), executed_at, 1 FROM migration_versions;' | mysql "$DBNAME" echo "DROP TABLE \`migration_versions\`" | mysql "$DBNAME" fi From e2030d22ec5f95e7972316054674624618886e7c Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 25 Aug 2024 22:09:37 +0200 Subject: [PATCH 06/13] Test plenty of weird characters in CI job MySQL password --- .github/jobs/baseinstall.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index c1764bae53..3c7550c4c1 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -78,7 +78,7 @@ mysql_log "CREATE DATABASE IF NOT EXISTS \`$DATABASE_NAME\` DEFAULT CHARACTER SE mysql_log "CREATE USER IF NOT EXISTS \`domjudge\`@'%' IDENTIFIED BY 'domjudge';" mysql_log "GRANT SELECT, INSERT, UPDATE, DELETE ON \`$DATABASE_NAME\`.* TO 'domjudge'@'%';" mysql_log "FLUSH PRIVILEGES;" -echo "unused:sqlserver:$DATABASE_NAME:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret +echo "unused:sqlserver:$DATABASE_NAME:domjudge:"'domjudge_db-pw+% #$*)@(!/;,.:3306' > /opt/domjudge/domserver/etc/dbpasswords.secret # Show some MySQL debugging mysql_log "show databases" @@ -96,6 +96,7 @@ if [ "${db}" = "install" ]; then section_end elif [ "${db}" = "upgrade" ]; then section_start "Upgrade DOMjudge database" + /opt/domjudge/domserver/bin/dj_setup_database update-password /opt/domjudge/domserver/bin/dj_setup_database upgrade /opt/domjudge/domserver/webapp/bin/console domjudge:reset-user-password admin "${ADMIN_PASSWORD}" section_end From 3bea86d9fa7f2b5666cf3bb60ba4696b9bcc7b3a Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Fri, 28 Nov 2025 11:08:15 +0100 Subject: [PATCH 07/13] Consistency: use mysql_log and credentials already set in baseinstall.sh --- .github/jobs/unit-tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/jobs/unit-tests.sh b/.github/jobs/unit-tests.sh index 3357a7868b..2b9ede6aa1 100755 --- a/.github/jobs/unit-tests.sh +++ b/.github/jobs/unit-tests.sh @@ -14,7 +14,7 @@ unittest=$2 export unit=1 # Add team to admin user -echo "UPDATE user SET teamid = 1 WHERE userid = 1;" | mysql domjudge_test +mysql_log "UPDATE user SET teamid = 1 WHERE userid = 1;" domjudge_test # Copy the .env.test file, as this is normally not done during # installation and we need it. @@ -33,7 +33,6 @@ if [ "$CODECOVERAGE" -eq 1 ]; then pcov="--coverage-html=${DIR}/coverage-html --coverage-clover coverage.xml" fi set +e -echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret php $phpcov webapp/bin/phpunit -c webapp/phpunit.xml.dist webapp/tests/$unittest --log-junit ${ARTIFACTS}/unit-tests.xml --colors=never $pcov | tee "$ARTIFACTS"/phpunit.out UNITSUCCESS=$? From c147be75d475af7effa2c10974ae79d8e5806576 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Fri, 28 Nov 2025 13:59:27 +0100 Subject: [PATCH 08/13] Create the domjudge DB user with wildcard hostname We don't care where the user is connecting from, and this helps when the MySQL server is not running on localhost. --- sql/dj_setup_database.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 9b5e35cf6d..27eaae8b5c 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -239,8 +239,8 @@ create_db_users() # in etc/domserver-config.php updated after installation. echo "CREATE DATABASE IF NOT EXISTS \`$DBNAME\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" - echo "CREATE USER IF NOT EXISTS '$domjudge_DBUSER'@'localhost' IDENTIFIED BY '$domjudge_PASSWD';" - echo "GRANT SELECT, INSERT, UPDATE, DELETE ON \`$DBNAME\`.* TO '$domjudge_DBUSER'@'localhost';" + echo "CREATE USER IF NOT EXISTS '$domjudge_DBUSER'@'%' IDENTIFIED BY '$domjudge_PASSWD';" + echo "GRANT SELECT, INSERT, UPDATE, DELETE ON \`$DBNAME\`.* TO '$domjudge_DBUSER'@'%';" echo "FLUSH PRIVILEGES;" ) | mysql @@ -251,7 +251,7 @@ remove_db_users() { ( echo "DROP DATABASE IF EXISTS \`$DBNAME\`;" - echo "DROP USER IF EXISTS '$domjudge_DBUSER'@'localhost';" + echo "DROP USER IF EXISTS '$domjudge_DBUSER'@'%';" echo "FLUSH PRIVILEGES;" ) | mysql -f verbose "DOMjudge database and user(s) removed." @@ -261,10 +261,10 @@ update_password() { read_dbpasswords ( - echo "ALTER USER '$domjudge_DBUSER'@'localhost' IDENTIFIED BY '$domjudge_PASSWD';" + echo "ALTER USER '$domjudge_DBUSER'@'%' IDENTIFIED BY '$domjudge_PASSWD';" echo "FLUSH PRIVILEGES;" ) | mysql - verbose "ALTER USER '$domjudge_DBUSER'@'localhost' IDENTIFIED BY '$domjudge_PASSWD';" + verbose "ALTER USER '$domjudge_DBUSER'@'%' IDENTIFIED BY '$domjudge_PASSWD';" verbose "Database user password updated from credentials file." } From 1510732807a28f35312bd1a445102afcb686e1eb Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Fri, 28 Nov 2025 13:58:40 +0100 Subject: [PATCH 09/13] Use our standard script to create database --- .github/jobs/baseinstall.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index 3c7550c4c1..523f3915a4 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -74,12 +74,10 @@ cat ~/.my.cnf # TODO: Remove after fixing https://github.com/DOMjudge/domjudge/issues/2848 mysql_log "SET GLOBAL innodb_snapshot_isolation = OFF;" -mysql_log "CREATE DATABASE IF NOT EXISTS \`$DATABASE_NAME\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" -mysql_log "CREATE USER IF NOT EXISTS \`domjudge\`@'%' IDENTIFIED BY 'domjudge';" -mysql_log "GRANT SELECT, INSERT, UPDATE, DELETE ON \`$DATABASE_NAME\`.* TO 'domjudge'@'%';" -mysql_log "FLUSH PRIVILEGES;" echo "unused:sqlserver:$DATABASE_NAME:domjudge:"'domjudge_db-pw+% #$*)@(!/;,.:3306' > /opt/domjudge/domserver/etc/dbpasswords.secret +/opt/domjudge/domserver/bin/dj_setup_database create-db-users + # Show some MySQL debugging mysql_log "show databases" mysql_log "SELECT CURRENT_USER();" @@ -96,7 +94,6 @@ if [ "${db}" = "install" ]; then section_end elif [ "${db}" = "upgrade" ]; then section_start "Upgrade DOMjudge database" - /opt/domjudge/domserver/bin/dj_setup_database update-password /opt/domjudge/domserver/bin/dj_setup_database upgrade /opt/domjudge/domserver/webapp/bin/console domjudge:reset-user-password admin "${ADMIN_PASSWORD}" section_end From fc44d08cda449bb7ebdf9d5963095ab3e662f21e Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Fri, 28 Nov 2025 15:55:26 +0100 Subject: [PATCH 10/13] Prevent double _test database suffix --- .github/jobs/unit-tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/jobs/unit-tests.sh b/.github/jobs/unit-tests.sh index 2b9ede6aa1..8a9f821cc2 100755 --- a/.github/jobs/unit-tests.sh +++ b/.github/jobs/unit-tests.sh @@ -25,6 +25,9 @@ cp webapp/composer.json /opt/domjudge/domserver/webapp/ cd /opt/domjudge/domserver +# The tests add a '_test' suffix to the database name already. +sed -i "s!:domjudge_test:!:domjudge:!" /opt/domjudge/domserver/etc/dbpasswords.secret + # Run phpunit tests. pcov="" phpcov="" From 51894fea23f2305ae571530bd502549efa2aa3e2 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sat, 5 Jul 2025 21:57:54 +0200 Subject: [PATCH 11/13] Call read_dbpasswords outside of update_passwords function This is consistent with all the other functions. --- sql/dj_setup_database.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 27eaae8b5c..0313bf2ed9 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -259,7 +259,6 @@ remove_db_users() update_password() { - read_dbpasswords ( echo "ALTER USER '$domjudge_DBUSER'@'%' IDENTIFIED BY '$domjudge_PASSWD';" echo "FLUSH PRIVILEGES;" @@ -354,6 +353,7 @@ create-db-users) ;; update-password) + read_dbpasswords update_password ;; From 2fd0d5000b7f7f02c261935cef65a182f9441116 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Mon, 25 Nov 2024 12:43:25 +0100 Subject: [PATCH 12/13] Don't pass DBUSER/PASSWD to symfony_console, always construct URL --- sql/dj_setup_database.in | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 0313bf2ed9..2d9fbd1ccf 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -155,6 +155,8 @@ symfony_console() DATABASE_URL="mysql://$user@$host:${domjudge_DBPORT}/$db" fi fi + else + DATABASE_URL="mysql://$DBUSER:$(urlencode "${PASSWD}")@$host:${domjudge_DBPORT}/$db" fi if [ -n "$DATABASE_URL" ]; then @@ -269,7 +271,7 @@ update_password() install_examples() { - DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-example-data + symfony_console domjudge:load-example-data "$EXAMPLEPROBDIR"/generate-contest-yaml ( cd "$EXAMPLEPROBDIR" && yes y | "$BINDIR"/import-contest ) } @@ -343,8 +345,8 @@ install-loadtest) read_dbpasswords create_db_users symfony_console doctrine:migrations:migrate -n - DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-default-data - DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-gatling-data + symfony_console domjudge:load-default-data + symfony_console domjudge:load-gatling-data ;; create-db-users) @@ -361,7 +363,7 @@ bare-install|install) read_dbpasswords create_db_users symfony_console doctrine:migrations:migrate -n - DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-default-data + symfony_console domjudge:load-default-data if [ "$1" = "install" ]; then install_examples verbose "SQL structure and default/example data installed." @@ -385,9 +387,8 @@ upgrade) fi symfony_console doctrine:migrations:migrate -n - DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-default-data - symfony_console domjudge:refresh-cache + symfony_console domjudge:load-default-data verbose "DOMjudge database upgrade completed." ;; From fa5654a27fdf42f8f9641767ee865cb1d3c6895a Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sat, 29 Nov 2025 18:21:35 +0100 Subject: [PATCH 13/13] Add " and some non-ASCII characters to database password in CI tests --- .github/jobs/baseinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index 523f3915a4..dd6a209457 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -74,7 +74,7 @@ cat ~/.my.cnf # TODO: Remove after fixing https://github.com/DOMjudge/domjudge/issues/2848 mysql_log "SET GLOBAL innodb_snapshot_isolation = OFF;" -echo "unused:sqlserver:$DATABASE_NAME:domjudge:"'domjudge_db-pw+% #$*)@(!/;,.:3306' > /opt/domjudge/domserver/etc/dbpasswords.secret +echo "unused:sqlserver:$DATABASE_NAME:domjudge:"'domjudge_db-pw+% #$*")@(!/;,.ßéçü主機:3306' > /opt/domjudge/domserver/etc/dbpasswords.secret /opt/domjudge/domserver/bin/dj_setup_database create-db-users