From 66081616b79942e4481dd68d43ca041050ddf522 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Fri, 2 Feb 2018 12:17:06 +0100 Subject: [PATCH 01/27] initiall version of PR build jobs It will create one job for each repository. Each job perform following steps: - prepare *build_server* machine (or use existing one) - clone desired repository and checkout PR branch - update *Release* with git snapshot and PR number - build source package - submit srpm to copr and wait for the result - download built packages and prepare PR repo on artifacts.ci.centos.org --- .../tendrl-build-triggers-pr.yml | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 jobs/build-triggers/tendrl-build-triggers-pr.yml diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml new file mode 100644 index 0000000..6b5048a --- /dev/null +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -0,0 +1,245 @@ +--- +- job-template: + name: "tendrl-build-1-pr-{package_name}" + display-name: 'Tendrl build: ({project_name}) - {package_name}' + description: | + Job building Tendrl package '{package_name}' from PR + and pushing it to '{target_repo}' repo.
+ Maintainer: Daniel Horak
+ Team Contact: tendrl-devel

+ JJB Code Location: tendrl-ci
+ Managed by Jenkins Job Builder. Do not edit via web.
+ project-type: freestyle + copr_config: '~/.config/copr' + package_prefix: "tendrl-" + defaults: global + disabled: false + concurrent: false + quiet-period: 0 + block-downstream: false + block-upstream: false + node: 'tendrl-ci-slave01' + triggers: "{obj:triggers}" + scm: "{obj:scm}" + properties: + - github: + url: 'https://github.com/Tendrl/{package_name}/' + wrappers: + - timeout: + timeout: 60 + abort: true + type: absolute + parameters: + - string: + name: TARGET_REPO + default: "{target_repo}" + description: "Target copr repo name." + #- string: + # name: sha1 + # default: "master" + # description: "" + builders: + - shell: | + #!/bin/bash -xe + #echo "==============================================================" + #env | grep -iv password + #echo "==============================================================" + echo "GIT_COMMIT: ${{GIT_COMMIT}}" + echo "GIT_BRANCH: ${{GIT_BRANCH}}" + echo "TARGET_REPO: ${{TARGET_REPO}}" + # prepare unique identifier for this build (BUILD_HASH) + export BUILD_HASH=$(mktemp -du "build-{target_repo}-{package_prefix}{package_name}.XXXXXXXXXX") + echo "BUILD_HASH=${{BUILD_HASH}}" > ${{WORKSPACE}}/env.sh + # "reserve" build server for this job + echo "${{BUILD_HASH}}" >> ${{HOME}}/build_server_{project_name}_used_by + + # prepare building server (if not already reserved) + - conditional-step: + condition-kind: shell + condition-command: "(( $(grep -v '^ *$' $HOME/build_server_{project_name}_status | wc -l) == 0))" + on-evaluation-failure: dont-run + steps: + - trigger-builds: + - project: 'tendrl-build-X-build_server-{project_name}-prepare' + block: true + property-file: env.sh + + - shell: | + #!/bin/bash -xe + # load environment file with BUILD_HASH + source ${{WORKSPACE}}/env.sh + echo "GIT_COMMIT: ${{GIT_COMMIT}}" + echo "GIT_BRANCH: ${{GIT_BRANCH}}" + # parse PR# and head/merge variant + if [[ "${{ghprbPullId}}" ]]; then + PR_NUMBER=${{ghprbPullId}} + PR_VARIANT="merge" + else + PR_NUMBER=$(echo $GIT_BRANCH | awk -F / '{{ print $3 }}') + PR_VARIANT=$(echo $GIT_BRANCH | awk -F / '{{ print $4 }}') + fi + echo "PR_NUMBER: ${{PR_NUMBER}}" + echo "PR_VARIANT: ${{PR_VARIANT}}" + echo "ghprbPullId: ${{ghprbPullId}}" + + # wait for build_server to be ready + until grep -q "READY" ${{HOME}}/build_server_{project_name}_status; do + sleep 10 + done + + # remote working directory + rwd="/tmp/${{BUILD_HASH}}" + # get build server hostname + build_server=$(grep -A1 '\[build_server\]' ${{HOME}}/build_server_{project_name}.hosts | tail -1) + # check if build_server is available. + ssh root@${{build_server}} "hostname" + # prepare remote working directory + ssh root@${{build_server}} "mkdir ${{rwd}}" + + # clone git repository with source code + ssh root@${{build_server}} \ + "git clone https://github.com/Tendrl/{package_name} ${{rwd}}/{package_name}" + # fetch and checkout the PR branch + branch="pr${{PR_NUMBER}}.${{PR_VARIANT}}" + ssh root@${{build_server}} \ + "cd ${{rwd}}/{package_name}; \ + git fetch origin refs/pull/${{PR_NUMBER}}/${{PR_VARIANT}}:${{branch}}; \ + git checkout ${{branch}}; \ + git status;" + + # build srpm package + # FIXME: we should unify the usage of target update-release or gitversion + ssh root@${{build_server}} \ + "cd ${{rwd}}/{package_name}; \ + make update-release ;\ + make gitversion ;\ + sed -i \"s/\(^Release:.*\)/\1.${{branch}}/\" {package_prefix}{package_name}.spec ;\ + make srpm ;\ + ls -l;" + + # prepare copr configuration + scp {copr_config} root@${{build_server}}:${{rwd}}/copr.conf + + # build package in copr + ssh root@${{build_server}} \ + "copr-cli --config ${{rwd}}/copr.conf build ${{TARGET_REPO}} \ + ${{rwd}}/{package_name}/{package_prefix}{package_name}*.src.rpm 2>&1 | tee ${{rwd}}/copr-build.log" + + # check status of copr build + ssh root@${{build_server}} "cat ${{rwd}}/copr-build.log" | \ + grep 'Build .*: succeeded' || exit 1 + + + # Copr build ID + COPR_BUILD_ID=$(ssh root@${{build_server}} "grep 'Created builds:' ${{rwd}}/copr-build.log | \ + sed 's/Created builds: //'") + echo "COPR_BUILD_ID: ${{COPR_BUILD_ID}}" + + # download the packages + ssh root@${{build_server}} \ + "copr-cli --config ${{rwd}}/copr.conf download-build \ + --dest ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}} \ + ${{COPR_BUILD_ID}}" + + # cleanup the repo + ssh root@${{build_server}} \ + "find ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}} \ + -type f ! -iname '*.rpm' -delete" + + # prepare rpm repo + ssh root@${{build_server}} \ + "createrepo ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}}/" + + # prepare repofile + repo_file="${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}}/{package_prefix}{package_name}-${{branch}}.repo" + ssh root@${{build_server}} \ + "echo '[{package_prefix}{package_name}-${{branch}}]' > ${{repo_file}} ;\ + echo 'name={package_prefix}{package_name}-${{branch}}' >> ${{repo_file}} ;\ + echo 'baseurl=http://artifacts.ci.centos.org/tendrl/pr/{package_prefix}{package_name}/${{branch}}/' >> ${{repo_file}} ;\ + echo 'skip_if_unavailable=True' >> ${{repo_file}} ;\ + echo 'gpgcheck=1' >> ${{repo_file}} ;\ + echo 'gpgkey=https://copr-be.cloud.fedoraproject.org/results/tendrl/${{TARGET_REPO}}/pubkey.gpg' >> ${{repo_file}} ;\ + echo 'repo_gpgcheck=0' >> ${{repo_file}} ;\ + echo 'enabled=1' >> ${{repo_file}} ;\ + echo '' >> ${{repo_file}};" + + # upload generated repo to artifacts.ci.centos.org + set +x + key=$(cat ~/duffy.key) + ssh root@${{build_server}} \ + "RSYNC_PASSWORD=${{key:0:13}} rsync -av \ + ${{rwd}}/rpms/pr/ \ + tendrl@artifacts.ci.centos.org::tendrl/pr/" + # cleanup older packages for the particular PR + ssh root@${{build_server}} \ + "RSYNC_PASSWORD=${{key:0:13}} rsync -av --delete \ + ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}}/ \ + tendrl@artifacts.ci.centos.org::tendrl/pr/{package_prefix}{package_name}/${{branch}}/" + set +x + + echo + echo "##############################################################" + echo "REPO: http://artifacts.ci.centos.org/tendrl/pr/{package_prefix}{package_name}/${{branch}}/{package_prefix}{package_name}-${{branch}}.repo" + echo "##############################################################" + echo + + publishers: + - trigger-parameterized-builds: + - project: 'tendrl-build-X-build_server-{project_name}-teardown' + property-file: env.sh + condition: ALWAYS + + - email: + recipients: dahorak@redhat.com + + +- project: + name: 'PR' + project_name: 'PR' + target_repo: 'Features' + copr_config: '~/.copr-tendrl.conf' + copr_user: 'tendrl' + run_package_validation: true + triggers: + - github-pull-request: + admin-list: + - dahorak + - nthomas-redhat + - r0h4n + - TimothyAsir + - TimothyAsirJeyasing + org-list: + - Tendrl + allow-whitelist-orgs-as-admins: true + #cron: 'H * * * *' + trigger-phrase: '.*(re)?run test.*' + skip-build-phrase: 'no tests' + github-hooks: true + cancel-builds-on-update: true + #auth-id: 'centos-ci' + scm: + - git: + url: 'https://github.com/Tendrl/{package_name}' + #refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + refspec: "+refs/pull/${{ghprbPullId}}/*:refs/remotes/origin/pr/${{ghprbPullId}}/*" + #branches: + # - "${{ghprbActualCommit}}" + basedir: tendrl-{package_name} + skip-tag: true + wipe-workspace: true + jobs: + - "tendrl-build-1-pr-{package_name}" + - "tendrl-build-X-build_server-{project_name}-prepare" + - "tendrl-build-X-build_server-{project_name}-teardown" + package_name: + - api + - commons + - gluster-integration + - monitoring-integration + - node-agent + - notifier + - tendrl-ansible: + package_prefix: "" + - tendrl-selinux: + package_prefix: "" + - ui From 5a063adfca8d621c5107eb51a1f3aeaf6b9c33c1 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Mon, 5 Feb 2018 08:19:01 +0100 Subject: [PATCH 02/27] PR builds: email notifications send-to-individuals --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index 6b5048a..864da63 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -191,6 +191,7 @@ - email: recipients: dahorak@redhat.com + send-to-individuals: true - project: From 342980fc7d7923a66cc433d39c784e5a0a7fe1a3 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Mon, 5 Feb 2018 10:13:00 +0100 Subject: [PATCH 03/27] stop the job in case of any failure --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index 864da63..b61c0c1 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -102,7 +102,8 @@ # fetch and checkout the PR branch branch="pr${{PR_NUMBER}}.${{PR_VARIANT}}" ssh root@${{build_server}} \ - "cd ${{rwd}}/{package_name}; \ + "set -xe; \ + cd ${{rwd}}/{package_name}; \ git fetch origin refs/pull/${{PR_NUMBER}}/${{PR_VARIANT}}:${{branch}}; \ git checkout ${{branch}}; \ git status;" @@ -110,7 +111,8 @@ # build srpm package # FIXME: we should unify the usage of target update-release or gitversion ssh root@${{build_server}} \ - "cd ${{rwd}}/{package_name}; \ + "set -xe; \ + cd ${{rwd}}/{package_name}; \ make update-release ;\ make gitversion ;\ sed -i \"s/\(^Release:.*\)/\1.${{branch}}/\" {package_prefix}{package_name}.spec ;\ From baccc2114adba38289799f91d9082eb5b2791a79 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Mon, 5 Feb 2018 10:21:11 +0100 Subject: [PATCH 04/27] workaround inconsistency in Makefile targets following two make targets are inconsistently used accross the Tendrl repos - update-release - gitversion --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index b61c0c1..daa7c8e 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -113,8 +113,10 @@ ssh root@${{build_server}} \ "set -xe; \ cd ${{rwd}}/{package_name}; \ + set +e ;\ make update-release ;\ make gitversion ;\ + set -e ;\ sed -i \"s/\(^Release:.*\)/\1.${{branch}}/\" {package_prefix}{package_name}.spec ;\ make srpm ;\ ls -l;" From d2619c73a6bc61c24273752a523ea9cf87d149f6 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Mon, 5 Feb 2018 10:21:48 +0100 Subject: [PATCH 05/27] fix indentation --- .../tendrl-build-triggers-pr.yml | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index daa7c8e..fb67d80 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -207,21 +207,23 @@ run_package_validation: true triggers: - github-pull-request: - admin-list: - - dahorak - - nthomas-redhat - - r0h4n - - TimothyAsir - - TimothyAsirJeyasing - org-list: - - Tendrl - allow-whitelist-orgs-as-admins: true - #cron: 'H * * * *' - trigger-phrase: '.*(re)?run test.*' - skip-build-phrase: 'no tests' - github-hooks: true - cancel-builds-on-update: true - #auth-id: 'centos-ci' + admin-list: + - dahorak + - nthomas-redhat + - r0h4n + - TimothyAsir + - TimothyAsirJeyasing + org-list: + - Tendrl + allow-whitelist-orgs-as-admins: true + #cron: 'H * * * *' + trigger-phrase: '.*(re)?run test.*' + skip-build-phrase: 'no tests' + github-hooks: true + cancel-builds-on-update: true + status-context: 'CentOS CI' + status-add-test-results: false + #auth-id: 'centos-ci' scm: - git: url: 'https://github.com/Tendrl/{package_name}' From 5f5479f5ff7400d21b51646da1e7825e5e47aad3 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 7 Feb 2018 11:02:11 +0100 Subject: [PATCH 06/27] pr builds: cleanup source packages - source packages are not required in the PR rpm repo and use quite huge space --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index fb67d80..60eb7ef 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -149,6 +149,9 @@ ssh root@${{build_server}} \ "find ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}} \ -type f ! -iname '*.rpm' -delete" + ssh root@${{build_server}} \ + "find ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}} \ + -type f -iname '*.src.rpm' -delete" # prepare rpm repo ssh root@${{build_server}} \ From 37f2a45271565a550b423e17513e46d5304a2575 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 7 Feb 2018 15:58:23 +0100 Subject: [PATCH 07/27] pr-builds: set timeout to 40 minutes --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index 60eb7ef..243b182 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -26,7 +26,7 @@ url: 'https://github.com/Tendrl/{package_name}/' wrappers: - timeout: - timeout: 60 + timeout: 40 abort: true type: absolute parameters: From 5a407513ae8fcc70312f41e212181f4af180cbb8 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 7 Feb 2018 15:59:08 +0100 Subject: [PATCH 08/27] pr-build: remove PR_VARIANT from branch name --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index 243b182..0aaf46f 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -100,7 +100,7 @@ ssh root@${{build_server}} \ "git clone https://github.com/Tendrl/{package_name} ${{rwd}}/{package_name}" # fetch and checkout the PR branch - branch="pr${{PR_NUMBER}}.${{PR_VARIANT}}" + branch="pr${{PR_NUMBER}}" ssh root@${{build_server}} \ "set -xe; \ cd ${{rwd}}/{package_name}; \ From 13f09f0d38c254b51811255b3369f727bc311ed2 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 7 Feb 2018 16:00:28 +0100 Subject: [PATCH 09/27] pr-builds: cleanup --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index 0aaf46f..54fac34 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -207,7 +207,6 @@ target_repo: 'Features' copr_config: '~/.copr-tendrl.conf' copr_user: 'tendrl' - run_package_validation: true triggers: - github-pull-request: admin-list: @@ -219,9 +218,8 @@ org-list: - Tendrl allow-whitelist-orgs-as-admins: true - #cron: 'H * * * *' - trigger-phrase: '.*(re)?run test.*' - skip-build-phrase: 'no tests' + trigger-phrase: '.*run build.*' + skip-build-phrase: 'no build' github-hooks: true cancel-builds-on-update: true status-context: 'CentOS CI' From 7224873a335bd42ccc6a05fed8e5607e86a249fc Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 7 Feb 2018 16:01:21 +0100 Subject: [PATCH 10/27] pr-build: add "package build to" status-context --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index 54fac34..5aa5999 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -222,7 +222,7 @@ skip-build-phrase: 'no build' github-hooks: true cancel-builds-on-update: true - status-context: 'CentOS CI' + status-context: 'CentOS CI - package build' status-add-test-results: false #auth-id: 'centos-ci' scm: From ec8a886d6c43006b1fe3e3d55c236bcd30cc2ab8 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 7 Feb 2018 16:30:08 +0100 Subject: [PATCH 11/27] pr-builds: notify also sds-mgmt-dev list --- jobs/build-triggers/tendrl-build-triggers-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr.yml index 5aa5999..6f652ab 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr.yml @@ -197,7 +197,7 @@ condition: ALWAYS - email: - recipients: dahorak@redhat.com + recipients: dahorak@redhat.com sds-mgmt-dev@redhat.com send-to-individuals: true From b6dff30ef43d19371b2b470029b5a409a5c72b02 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Fri, 9 Feb 2018 08:29:15 +0100 Subject: [PATCH 12/27] rename tendrl-build-triggers-pr.yml to tendrl-build-triggers-pr-builds.yml --- ...-build-triggers-pr.yml => tendrl-build-triggers-pr-builds.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jobs/build-triggers/{tendrl-build-triggers-pr.yml => tendrl-build-triggers-pr-builds.yml} (100%) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml similarity index 100% rename from jobs/build-triggers/tendrl-build-triggers-pr.yml rename to jobs/build-triggers/tendrl-build-triggers-pr-builds.yml From b03bceb07b618acb15c4ef2f6cbd058a9f9f036c Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Fri, 9 Feb 2018 08:33:33 +0100 Subject: [PATCH 13/27] pr-builds: cleanup config --- .../tendrl-build-triggers-pr-builds.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 6f652ab..f8ec5df 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -34,16 +34,8 @@ name: TARGET_REPO default: "{target_repo}" description: "Target copr repo name." - #- string: - # name: sha1 - # default: "master" - # description: "" builders: - shell: | - #!/bin/bash -xe - #echo "==============================================================" - #env | grep -iv password - #echo "==============================================================" echo "GIT_COMMIT: ${{GIT_COMMIT}}" echo "GIT_BRANCH: ${{GIT_BRANCH}}" echo "TARGET_REPO: ${{TARGET_REPO}}" @@ -224,14 +216,10 @@ cancel-builds-on-update: true status-context: 'CentOS CI - package build' status-add-test-results: false - #auth-id: 'centos-ci' scm: - git: url: 'https://github.com/Tendrl/{package_name}' - #refspec: "+refs/pull/*:refs/remotes/origin/pr/*" refspec: "+refs/pull/${{ghprbPullId}}/*:refs/remotes/origin/pr/${{ghprbPullId}}/*" - #branches: - # - "${{ghprbActualCommit}}" basedir: tendrl-{package_name} skip-tag: true wipe-workspace: true From 8b57ae2169d0dcc26989ce2c17534cbf17d4f294 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Tue, 13 Feb 2018 10:21:06 +0100 Subject: [PATCH 14/27] pr-builds: allow build for members of Tendrl org - There seems to be some problem with enabling builds globally for whole the Tendrl organization, so adding specifically particular people to the admin list. --- .../tendrl-build-triggers-pr-builds.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index f8ec5df..55c3e7d 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -202,9 +202,27 @@ triggers: - github-pull-request: admin-list: + - a2batic + - cloudbehl - dahorak + - fbalak + - gnehapk + - GowthamShanmugam + - julienlim + - ktdreyer + - ltrilety + - mbukatov + - mcarrano + - nnDarshan - nthomas-redhat - r0h4n + - rishubhjain + - sankarshanmukhopadhyay + - scuttlemonkey + - shirshendu + - shtripat + - sidhax + - tigert - TimothyAsir - TimothyAsirJeyasing org-list: From a5d33284cac7c6c62dabed849251aacf8944551d Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Tue, 13 Feb 2018 12:20:41 +0100 Subject: [PATCH 15/27] pr-builds: disable send-to-individuals - it seems to send the notification to strange list of email addresses, so disabling it --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 55c3e7d..7ac48b6 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -190,7 +190,7 @@ - email: recipients: dahorak@redhat.com sds-mgmt-dev@redhat.com - send-to-individuals: true + send-to-individuals: false - project: From ae65e088d4e33fa9ea0827e45b6c52c498bd95c3 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 14 Feb 2018 08:26:02 +0100 Subject: [PATCH 16/27] pr-builds: specify branches to origin/pr/** - limit the job execution only for PR updates --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 7ac48b6..a1d50f5 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -238,6 +238,8 @@ - git: url: 'https://github.com/Tendrl/{package_name}' refspec: "+refs/pull/${{ghprbPullId}}/*:refs/remotes/origin/pr/${{ghprbPullId}}/*" + branches: + - "origin/pr/**" basedir: tendrl-{package_name} skip-tag: true wipe-workspace: true From 30249142234aa0a4b3efa5423fa285a9189225fa Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 14 Feb 2018 08:27:18 +0100 Subject: [PATCH 17/27] pr-builds: add "99" before release string - to ensure PR builds have higher version and will not be overwritten by newest schnapshot packages from master branch --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index a1d50f5..98976c0 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -110,6 +110,7 @@ make gitversion ;\ set -e ;\ sed -i \"s/\(^Release:.*\)/\1.${{branch}}/\" {package_prefix}{package_name}.spec ;\ + sed -i \"s/\(^Release: *\)/\199/\" {package_prefix}{package_name}.spec ;\ make srpm ;\ ls -l;" From 5040281e8f27d5d2bec7bfe7a898512822a5206d Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 14 Feb 2018 08:34:14 +0100 Subject: [PATCH 18/27] pr-builds: separate admin and white-listed people --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 98976c0..e695ec1 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -203,9 +203,10 @@ triggers: - github-pull-request: admin-list: + - dahorak + white-list: - a2batic - cloudbehl - - dahorak - fbalak - gnehapk - GowthamShanmugam @@ -228,7 +229,7 @@ - TimothyAsirJeyasing org-list: - Tendrl - allow-whitelist-orgs-as-admins: true + allow-whitelist-orgs-as-admins: false trigger-phrase: '.*run build.*' skip-build-phrase: 'no build' github-hooks: true From d68eb11e58f1b32559b2bcb1e7a1044c33ccab94 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 21 Feb 2018 14:45:30 +0100 Subject: [PATCH 19/27] pr-builds: build localy using mock instead of Copr --- .../tendrl-build-triggers-pr-builds.yml | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index e695ec1..30f2ef2 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -114,29 +114,25 @@ make srpm ;\ ls -l;" - # prepare copr configuration - scp {copr_config} root@${{build_server}}:${{rwd}}/copr.conf - - # build package in copr + # prepare mock configuration + mock_root="/var/lib/mock/epel-7-x86_64-${{BUILD_HASH}}" + mock_cfg="${{rwd}}/mock-epel-7-x86_64.cfg" ssh root@${{build_server}} \ - "copr-cli --config ${{rwd}}/copr.conf build ${{TARGET_REPO}} \ - ${{rwd}}/{package_name}/{package_prefix}{package_name}*.src.rpm 2>&1 | tee ${{rwd}}/copr-build.log" - - # check status of copr build - ssh root@${{build_server}} "cat ${{rwd}}/copr-build.log" | \ - grep 'Build .*: succeeded' || exit 1 - + "set -xe; \ + cp /etc/mock/epel-7-x86_64.cfg ${{mock_cfg}}; \ + sed -i 's/epel-7-x86_64/epel-7-x86_64-${{BUILD_HASH}}/' ${{mock_cfg}}" - # Copr build ID - COPR_BUILD_ID=$(ssh root@${{build_server}} "grep 'Created builds:' ${{rwd}}/copr-build.log | \ - sed 's/Created builds: //'") - echo "COPR_BUILD_ID: ${{COPR_BUILD_ID}}" + # build RPM packages using mock + ssh root@${{build_server}} \ + "set -xe; \ + mock -r ${{mock_cfg}} rebuild \ + ${{rwd}}/{package_name}/{package_prefix}{package_name}*.src.rpm" - # download the packages + # prepare RPM repo ssh root@${{build_server}} \ - "copr-cli --config ${{rwd}}/copr.conf download-build \ - --dest ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}} \ - ${{COPR_BUILD_ID}}" + "set -xe; \ + mkdir -p ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}}; \ + cp ${{mock_root}}/result/*.rpm ${{rwd}}/rpms/pr/{package_prefix}{package_name}/${{branch}};" # cleanup the repo ssh root@${{build_server}} \ From 5d2cd7f3cd9f7ccb490a178ae7802cadf6c06baa Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 21 Feb 2018 15:26:57 +0100 Subject: [PATCH 20/27] pr-builds: use master branch for merged PRs --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 30f2ef2..e0ad78f 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -91,12 +91,18 @@ # clone git repository with source code ssh root@${{build_server}} \ "git clone https://github.com/Tendrl/{package_name} ${{rwd}}/{package_name}" + + # prepare refs, use master if it is for closed PR + refs="refs/pull/${{PR_NUMBER}}/${{PR_VARIANT}}" + if [[ "${{ghprbActualCommitAuthorEmail}}" == "noreply@github.com" ]]; then + refs="master" + fi # fetch and checkout the PR branch branch="pr${{PR_NUMBER}}" ssh root@${{build_server}} \ "set -xe; \ cd ${{rwd}}/{package_name}; \ - git fetch origin refs/pull/${{PR_NUMBER}}/${{PR_VARIANT}}:${{branch}}; \ + git fetch origin ${{refs}}:${{branch}}; \ git checkout ${{branch}}; \ git status;" From c55546be5a44b4764c325f7c59b16948c7828a5e Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Mon, 26 Feb 2018 09:39:16 +0100 Subject: [PATCH 21/27] pr-builds: unify make gitversion target --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index e0ad78f..4f03187 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -107,14 +107,10 @@ git status;" # build srpm package - # FIXME: we should unify the usage of target update-release or gitversion ssh root@${{build_server}} \ "set -xe; \ cd ${{rwd}}/{package_name}; \ - set +e ;\ - make update-release ;\ make gitversion ;\ - set -e ;\ sed -i \"s/\(^Release:.*\)/\1.${{branch}}/\" {package_prefix}{package_name}.spec ;\ sed -i \"s/\(^Release: *\)/\199/\" {package_prefix}{package_name}.spec ;\ make srpm ;\ From 937fae74ef66d3e5d4eb764c6d056c1fe323c661 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Mon, 26 Feb 2018 11:37:31 +0100 Subject: [PATCH 22/27] pr-builds: yamllint fix (remove trailing spaces) --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 4f03187..182c159 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -190,7 +190,7 @@ - email: recipients: dahorak@redhat.com sds-mgmt-dev@redhat.com send-to-individuals: false - + - project: name: 'PR' From d4bb5db17b3fad765138a8857a98ecfb2c8606bc Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Tue, 6 Mar 2018 10:45:34 +0100 Subject: [PATCH 23/27] pr-builds: log last 5 git commits --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 182c159..f5a1eef 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -104,6 +104,7 @@ cd ${{rwd}}/{package_name}; \ git fetch origin ${{refs}}:${{branch}}; \ git checkout ${{branch}}; \ + git log -5; \ git status;" # build srpm package From 61fb637b52cea0870729378c2847319304d43d93 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Tue, 6 Mar 2018 10:46:07 +0100 Subject: [PATCH 24/27] pr-builds: use more clear git hash for release - for not merged PR, it will contain two hashes for both parents of the temporary merge to master commit (one is for commit from master branch, the second is the last commit in the PR) - for merged PR, it will contain only the hash of the merge commit (or last commit from master) --- .../tendrl-build-triggers-pr-builds.yml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index f5a1eef..688aab0 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -107,6 +107,26 @@ git log -5; \ git status;" + # prepare git hash based release string + # - for not merged PR, it will contain two hashes for both + # parents of the temporary merge to master commit + # - for merged PR, it will contain only the hash of the merge commit + # (or last commit from master) + git_commit_and_parrents=$(ssh root@${{build_server}} \ + "set -xe; \ + cd ${{rwd}}/{package_name}; \ + git rev-list --parents -n 1 HEAD") + git_last_commit=$(echo "${{git_commit_and_parrents}}" | awk '{{ print $1 }}' | cut -c1-7 ) + if [[ "${{ghprbActualCommitAuthorEmail}}" == "noreply@github.com" ]]; then + git_hash_release_string="${{git_last_commit}}" + else + git_parrent_a=$(echo "${{git_commit_and_parrents}}" | awk '{{ print $2 }}' | cut -c1-7 ) + git_parrent_b=$(echo "${{git_commit_and_parrents}}" | awk '{{ print $3 }}' | cut -c1-7 ) + git_hash_release_string="${{git_parrent_a}}_${{git_parrent_b}}" + fi + echo "${{git_last_commit}}" + echo "${{git_hash_release_string}}" + # build srpm package ssh root@${{build_server}} \ "set -xe; \ @@ -114,6 +134,7 @@ make gitversion ;\ sed -i \"s/\(^Release:.*\)/\1.${{branch}}/\" {package_prefix}{package_name}.spec ;\ sed -i \"s/\(^Release: *\)/\199/\" {package_prefix}{package_name}.spec ;\ + sed -i \"s/${{git_last_commit}}/${{git_hash_release_string}}/\" {package_prefix}{package_name}.spec ;\ make srpm ;\ ls -l;" From 12278a286e0ca352a6d2834880702dd3d3db3781 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Tue, 6 Mar 2018 11:14:22 +0100 Subject: [PATCH 25/27] pr-builds: disable gpgcheck - packages built in mock are not signed --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index 688aab0..f87fae4 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -177,7 +177,7 @@ echo 'name={package_prefix}{package_name}-${{branch}}' >> ${{repo_file}} ;\ echo 'baseurl=http://artifacts.ci.centos.org/tendrl/pr/{package_prefix}{package_name}/${{branch}}/' >> ${{repo_file}} ;\ echo 'skip_if_unavailable=True' >> ${{repo_file}} ;\ - echo 'gpgcheck=1' >> ${{repo_file}} ;\ + echo 'gpgcheck=0' >> ${{repo_file}} ;\ echo 'gpgkey=https://copr-be.cloud.fedoraproject.org/results/tendrl/${{TARGET_REPO}}/pubkey.gpg' >> ${{repo_file}} ;\ echo 'repo_gpgcheck=0' >> ${{repo_file}} ;\ echo 'enabled=1' >> ${{repo_file}} ;\ From a7e5ac4b82d366c5f96a61c0d340d2c983709ef1 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 18 Apr 2018 16:55:00 +0200 Subject: [PATCH 26/27] remove sds-mgmt-dev@redhat.com list - it is not possible to send emails to this list outside of RH (from ci@centos.org) --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 2 +- jobs/build-triggers/tendrl-build-triggers.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index f87fae4..fb38bb2 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -210,7 +210,7 @@ condition: ALWAYS - email: - recipients: dahorak@redhat.com sds-mgmt-dev@redhat.com + recipients: dahorak@redhat.com send-to-individuals: false diff --git a/jobs/build-triggers/tendrl-build-triggers.yml b/jobs/build-triggers/tendrl-build-triggers.yml index 287b02d..d263137 100644 --- a/jobs/build-triggers/tendrl-build-triggers.yml +++ b/jobs/build-triggers/tendrl-build-triggers.yml @@ -113,7 +113,7 @@ condition: ALWAYS - email: - recipients: dahorak@redhat.com sds-mgmt-dev@redhat.com + recipients: dahorak@redhat.com send-to-individuals: false From 5ea1e38856292e4e317837e3c3cc8792a5fc1fe6 Mon Sep 17 00:00:00 2001 From: Daniel Horak Date: Wed, 2 May 2018 12:56:47 +0200 Subject: [PATCH 27/27] pr-build-jobs: add verbosity (-v) to mock --- jobs/build-triggers/tendrl-build-triggers-pr-builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml index fb38bb2..1e7ed58 100644 --- a/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml +++ b/jobs/build-triggers/tendrl-build-triggers-pr-builds.yml @@ -149,7 +149,7 @@ # build RPM packages using mock ssh root@${{build_server}} \ "set -xe; \ - mock -r ${{mock_cfg}} rebuild \ + mock -v -r ${{mock_cfg}} rebuild \ ${{rwd}}/{package_name}/{package_prefix}{package_name}*.src.rpm" # prepare RPM repo