From c6ace7f9297f89aafc7f90c69e0b51158bc57155 Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Wed, 17 Sep 2025 11:49:57 -0400 Subject: [PATCH 1/2] CBG-4852 use gotestsum --- Jenkinsfile | 119 +++++++++++++++++------------------ jenkins-integration-build.sh | 36 +++++------ 2 files changed, 72 insertions(+), 83 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5e4dff3f5f..28931575b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,12 +52,9 @@ pipeline { } stage('Go Tools') { steps { - // unhandled error checker - sh 'go install github.com/kisielk/errcheck@latest' // goveralls is used to send coverprofiles to coveralls.io sh 'go install github.com/mattn/goveralls@latest' - // Jenkins test reporting tools - sh 'go install github.com/tebeka/go2xunit@latest' + sh 'go install gotest.tools/gotestsum@latest' } } } @@ -90,89 +87,85 @@ pipeline { stage('CE') { when { branch 'main' } steps { - // Travis-related variables are required as coveralls.io only officially supports a certain set of CI tools. - withEnv(["PATH+GO=${env.GOTOOLS}/bin", "TRAVIS_BRANCH=${env.BRANCH}", "TRAVIS_PULL_REQUEST=${env.CHANGE_ID}", "TRAVIS_JOB_ID=${env.BUILD_NUMBER}"]) { - githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ce-unit-tests', description: 'CE Unit Tests Running', status: 'PENDING') - - // Build CE coverprofiles - sh '2>&1 go test -shuffle=on -timeout=20m -coverpkg=./... -coverprofile=cover_ce.out -race -count=1 -v ./... > verbose_ce.out.raw || true' - - // Print total coverage stats - sh 'go tool cover -func=cover_ce.out | awk \'END{print "Total SG CE Coverage: " $3}\'' - - sh 'mkdir -p reports' - - // strip non-printable characters from the raw verbose test output - sh 'LC_CTYPE=C tr -dc [:print:][:space:] < verbose_ce.out.raw > verbose_ce.out' - - // Grab test fail/total counts so we can print them later - sh "grep '\\-\\-\\- PASS: ' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-pass.count" - sh "grep '\\-\\-\\- FAIL: ' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-fail.count" - sh "grep '\\-\\-\\- SKIP: ' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-skip.count" - sh "grep '=== RUN' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-total.count" - script { + script { + // Travis-related variables are required as coveralls.io only officially supports a certain set of CI tools. + withEnv(["PATH+GO=${env.GOTOOLS}/bin", "TRAVIS_BRANCH=${env.BRANCH}", "TRAVIS_PULL_REQUEST=${env.CHANGE_ID}", "TRAVIS_JOB_ID=${env.BUILD_NUMBER}"]) { + githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ce-unit-tests', description: 'CE Unit Tests Running', status: 'PENDING') + + sh 'mkdir -p reports' + // --junitfile-project-name is used so that Jenkins doesn't collapse CE and EE results together. + def testExitCode = sh( + script: 'gotestsum --junitfile=test-ce.xml --junitfile-project-name CE --junitfile-testcase-classname relative --format standard-verbose -- -shuffle=on -timeout=20m -coverpkg=./... -coverprofile=cover_ce.out -race -count=1 ./... > verbose_ce.out', + returnStatus: true, + ) + + // convert the junit file to prepend CE- to all test names to differentiate from EE tests + sh '''xmlstarlet ed -u '//testcase/@classname' -x 'concat("CE-", .)' test-ce.xml > reports/test-ce.xml''' + + // Print total coverage stats + sh 'go tool cover -func=cover_ce.out | awk \'END{print "Total SG CE Coverage: " $3}\'' + + // Grab test fail/total counts so we can print them later + sh "grep '\\-\\-\\- PASS: ' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-pass.count" + sh "grep '\\-\\-\\- FAIL: ' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-fail.count" + sh "grep '\\-\\-\\- SKIP: ' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-skip.count" + sh "grep '=== RUN' verbose_ce.out | wc -l | awk '{printf \$1}' > test-ce-total.count" env.TEST_CE_PASS = readFile 'test-ce-pass.count' env.TEST_CE_FAIL = readFile 'test-ce-fail.count' env.TEST_CE_SKIP = readFile 'test-ce-skip.count' env.TEST_CE_TOTAL = readFile 'test-ce-total.count' - } - // Generate junit-formatted test report - script { - try { - sh 'which go2xunit' // check if go2xunit is installed - sh 'go2xunit -fail -suite-name-prefix="CE-" -input verbose_ce.out -output reports/test-ce.xml' + if (testExitCode == 0) { githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ce-unit-tests', description: env.TEST_CE_PASS + '/' + env.TEST_CE_TOTAL + ' passed (' + env.TEST_CE_SKIP + ' skipped)', status: 'SUCCESS') - } catch (Exception e) { + } else { githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ce-unit-tests', description: env.TEST_CE_FAIL + '/' + env.TEST_CE_TOTAL + ' failed (' + env.TEST_CE_SKIP + ' skipped)', status: 'FAILURE') // archive verbose test logs in the event of a test failure archiveArtifacts artifacts: 'verbose_ce.out', fingerprint: false unstable('At least one CE unit test failed') } - } - // Publish CE coverage to coveralls.io - // Replace covermode values with set just for coveralls to reduce the variability in reports. - sh 'awk \'NR==1{print "mode: set";next} $NF>0{$NF=1} {print}\' cover_ce.out > cover_ce_coveralls.out' - sh 'which goveralls' // check if goveralls is installed - sh 'goveralls -coverprofile=cover_ce_coveralls.out -service=uberjenkins -repotoken=$COVERALLS_TOKEN || true' + // Publish CE coverage to coveralls.io + // Replace covermode values with set just for coveralls to reduce the variability in reports. + sh 'awk \'NR==1{print "mode: set";next} $NF>0{$NF=1} {print}\' cover_ce.out > cover_ce_coveralls.out' + sh 'which goveralls' // check if goveralls is installed + sh 'goveralls -coverprofile=cover_ce_coveralls.out -service=uberjenkins -repotoken=$COVERALLS_TOKEN || true' + } } } } stage('EE') { steps { - withEnv(["PATH+GO=${env.GOTOOLS}/bin"]) { - githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ee-unit-tests', description: 'EE Unit Tests Running', status: 'PENDING') - - // Build EE coverprofiles - sh "2>&1 go test -shuffle=on -timeout=20m -tags ${EE_BUILD_TAG} -coverpkg=./... -coverprofile=cover_ee.out -race -count=1 -v ./... > verbose_ee.out.raw || true" - - sh 'go tool cover -func=cover_ee.out | awk \'END{print "Total SG EE Coverage: " $3}\'' - - sh 'mkdir -p reports' - - // strip non-printable characters from the raw verbose test output - sh 'LC_CTYPE=C tr -dc [:print:][:space:] < verbose_ee.out.raw > verbose_ee.out' - - // Grab test fail/total counts so we can print them later - sh "grep '\\-\\-\\- PASS: ' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-pass.count" - sh "grep '\\-\\-\\- FAIL: ' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-fail.count" - sh "grep '\\-\\-\\- SKIP: ' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-skip.count" - sh "grep '=== RUN' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-total.count" - script { + script { + withEnv(["PATH+GO=${env.GOTOOLS}/bin"]) { + githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ee-unit-tests', description: 'EE Unit Tests Running', status: 'PENDING') + + sh 'mkdir -p reports' + // --junitfile-project-name is used so that Jenkins doesn't collapse CE and EE results together. + def testExitCode = sh( + script: "gotestsum --junitfile=test-ee.xml --junitfile-project-name EE --junitfile-testcase-classname relative --format standard-verbose -- -shuffle=on -timeout=20m -tags ${EE_BUILD_TAG} -coverpkg=./... -coverprofile=cover_ee.out -race -count=1 ./... 2>&1 > verbose_ee.out", + returnStatus: true, + ) + + // convert the junit file to prepend EE- to all test names to differentiate from EE tests + sh '''xmlstarlet ed -u '//testcase/@classname' -x 'concat("EE-", .)' test-ee.xml > reports/test-ee.xml''' + + sh 'go tool cover -func=cover_ee.out | awk \'END{print "Total SG EE Coverage: " $3}\'' + + // Grab test fail/total counts so we can print them later + sh "grep '\\-\\-\\- PASS: ' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-pass.count" + sh "grep '\\-\\-\\- FAIL: ' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-fail.count" + sh "grep '\\-\\-\\- SKIP: ' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-skip.count" + sh "grep '=== RUN' verbose_ee.out | wc -l | awk '{printf \$1}' > test-ee-total.count" env.TEST_EE_PASS = readFile 'test-ee-pass.count' env.TEST_EE_FAIL = readFile 'test-ee-fail.count' env.TEST_EE_SKIP = readFile 'test-ee-skip.count' env.TEST_EE_TOTAL = readFile 'test-ee-total.count' - } - // Generate junit-formatted test report - script { - try { - sh 'go2xunit -fail -suite-name-prefix="EE-" -input verbose_ee.out -output reports/test-ee.xml' + // Generate junit-formatted test report + if (testExitCode == 0) { githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ee-unit-tests', description: env.TEST_EE_PASS + '/' + env.TEST_EE_TOTAL + ' passed (' + env.TEST_EE_SKIP + ' skipped)', status: 'SUCCESS') - } catch (Exception e) { + } else { githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-ee-unit-tests', description: env.TEST_EE_FAIL + '/' + env.TEST_EE_TOTAL + ' failed (' + env.TEST_EE_SKIP + ' skipped)', status: 'FAILURE') // archive verbose test logs in the event of a test failure archiveArtifacts artifacts: 'verbose_ee.out', fingerprint: false diff --git a/jenkins-integration-build.sh b/jenkins-integration-build.sh index 377a1107c0..cf897c5bd8 100755 --- a/jenkins-integration-build.sh +++ b/jenkins-integration-build.sh @@ -56,13 +56,10 @@ if [ "${USE_GO_MODULES:-}" == "false" ]; then go get -u -v github.com/axw/gocov/gocov go get -u -v github.com/AlekSi/gocov-xml else - # Install tools to use after job has completed - # go2xunit will fail with 1.23 with name mismatch (try disabling parallel mode), but without any t.Parallel() - go install golang.org/dl/go1.22.8@latest - ~/go/bin/go1.22.8 download - ~/go/bin/go1.22.8 install -v github.com/tebeka/go2xunit@latest - go install -v github.com/axw/gocov/gocov@latest - go install -v github.com/AlekSi/gocov-xml@latest + # Install tools to use + go install github.com/axw/gocov/gocov@latest + go install github.com/AlekSi/gocov-xml@latest + go install gotest.tools/gotestsum@latest fi if [ "${SG_TEST_X509:-}" == "true" -a "${COUCHBASE_SERVER_PROTOCOL}" != "couchbases" ]; then @@ -77,7 +74,7 @@ INT_LOG_FILE_NAME="verbose_int" if [ -d "godeps" ]; then export GOPATH=$(pwd)/godeps fi -export PATH=$PATH:$(go env GOPATH)/bin +export PATH=$PATH:$(go env GOPATH)/bin:~/go/bin echo "PATH: $PATH" if [ "${TEST_DEBUG:-}" == "true" ]; then @@ -109,10 +106,15 @@ if [ "${SG_TEST_PROFILE_FREQUENCY:-}" == "true" ]; then fi if [ "${RUN_WALRUS}" == "true" ]; then + set +e # EE - go test -coverprofile=coverage_walrus_ee.out -coverpkg=github.com/couchbase/sync_gateway/... -tags cb_sg_devmode,cb_sg_enterprise $GO_TEST_FLAGS github.com/couchbase/sync_gateway/${TARGET_PACKAGE} > verbose_unit_ee.out.raw 2>&1 | true + gotestsum --junitfile=rosmar-ee.xml --junitfile-project-name rosmar-EE --junitfile-testcase-classname relative --format standard-verbose -- -coverprofile=coverage_walrus_ee.out -coverpkg=github.com/couchbase/sync_gateway/... -tags cb_sg_devmode,cb_sg_enterprise $GO_TEST_FLAGS github.com/couchbase/sync_gateway/${TARGET_PACKAGE} > verbose_unit_ee.out 2>&1 + xmlstarlet ed -u '//testcase/@classname' -x 'concat("rosmar-EE-", .)' rosmar-ee.xml > verbose_unit_ee.xml # CE - go test -coverprofile=coverage_walrus_ce.out -coverpkg=github.com/couchbase/sync_gateway/... -tags cb_sg_devmode $GO_TEST_FLAGS github.com/couchbase/sync_gateway/${TARGET_PACKAGE} > verbose_unit_ce.out.raw 2>&1 | true + gotestsum --junitfile=rosmar-ce.xml --junitfile-project-name rosmar-CE --junitfile-testcase-classname relative --format standard-verbose -- -coverprofile=coverage_walrus_ce.out -coverpkg=github.com/couchbase/sync_gateway/... -tags cb_sg_devmode $GO_TEST_FLAGS github.com/couchbase/sync_gateway/${TARGET_PACKAGE} > verbose_unit_ce.out 2>&1 + xmlstarlet ed -u '//testcase/@classname' -x 'concat("rosmar-CE-", .)' rosmar-ce.xml > verbose_unit_ce.xml + set -e + fi # Run CBS @@ -141,26 +143,20 @@ else GO_TEST_FLAGS="${GO_TEST_FLAGS} -tags cb_sg_devmode" fi -go test ${GO_TEST_FLAGS} -coverprofile=coverage_int.out -coverpkg=github.com/couchbase/sync_gateway/... github.com/couchbase/sync_gateway/${TARGET_PACKAGE} 2>&1 | stdbuf -oL tee "${INT_LOG_FILE_NAME}.out.raw" | stdbuf -oL grep -a -E '(--- (FAIL|PASS|SKIP):|github.com/couchbase/sync_gateway(/.+)?\t|TEST: |panic: )' +gotestsum --junitfile=integration.xml --junitfile-project-name integration --junitfile-testcase-classname relative --format standard-verbose -- ${GO_TEST_FLAGS} -coverprofile=coverage_int.out -coverpkg=github.com/couchbase/sync_gateway/... github.com/couchbase/sync_gateway/${TARGET_PACKAGE} 2>&1 | stdbuf -oL tee "${INT_LOG_FILE_NAME}.out" | stdbuf -oL grep -a -E '(--- (FAIL|PASS|SKIP):|github.com/couchbase/sync_gateway(/.+)?\t|TEST: |panic: )' if [ "${PIPESTATUS[0]}" -ne "0" ]; then # If test exit code is not 0 (failed) echo "Go test failed! Parsing logs to find cause..." TEST_FAILED=true fi # Collect CBS logs if server error occurred -if [ "${SG_CBCOLLECT_ALWAYS:-}" == "true" ] || grep -a -q "server logs for details\|Timed out after 1m0s waiting for a bucket to become available" "${INT_LOG_FILE_NAME}.out.raw"; then +if [ "${SG_CBCOLLECT_ALWAYS:-}" == "true" ] || grep -a -q "server logs for details\|Timed out after 1m0s waiting for a bucket to become available" "${INT_LOG_FILE_NAME}.out"; then docker exec -t couchbase /opt/couchbase/bin/cbcollect_info /workspace/cbcollect.zip fi -# Generate xunit test report that can be parsed by the JUnit Plugin -LC_CTYPE=C tr -dc [:print:][:space:] < ${INT_LOG_FILE_NAME}.out.raw > ${INT_LOG_FILE_NAME}.out # Strip non-printable characters -~/go/bin/go2xunit -input "${INT_LOG_FILE_NAME}.out" -output "${INT_LOG_FILE_NAME}.xml" +# If rosmar tests were run, then prepend classname with integration to tell them apart if [ "${RUN_WALRUS}" == "true" ]; then - # Strip non-printable characters before xml creation - LC_CTYPE=C tr -dc [:print:][:space:] < "verbose_unit_ee.out.raw" > "verbose_unit_ee.out" - LC_CTYPE=C tr -dc [:print:][:space:] < "verbose_unit_ce.out.raw" > "verbose_unit_ce.out" - ~/go/bin/go2xunit -input "verbose_unit_ee.out" -output "verbose_unit_ee.xml" - ~/go/bin/go2xunit -input "verbose_unit_ce.out" -output "verbose_unit_ce.xml" + xmlstarlet ed -u '//testcase/@classname' -x 'concat("integration-EE-", .)' integration.xml > "${INT_LOG_FILE_NAME}.xml" fi # Get coverage From fd494b15e6fdae08267ae4e5ad3647d7e141f7ed Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Thu, 18 Sep 2025 15:19:37 -0400 Subject: [PATCH 2/2] no-op copy integragtion test file --- jenkins-integration-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-integration-build.sh b/jenkins-integration-build.sh index cf897c5bd8..268ee827e0 100755 --- a/jenkins-integration-build.sh +++ b/jenkins-integration-build.sh @@ -157,6 +157,8 @@ fi # If rosmar tests were run, then prepend classname with integration to tell them apart if [ "${RUN_WALRUS}" == "true" ]; then xmlstarlet ed -u '//testcase/@classname' -x 'concat("integration-EE-", .)' integration.xml > "${INT_LOG_FILE_NAME}.xml" +else + cp integration.xml "${INT_LOG_FILE_NAME}.xml" fi # Get coverage