Revise CI running of conformance tests. (#1901)

- Have the `Makefile` provide better messaging in the conformance
runners isn't found.
- Have `build.yml` run the conformance tests as it is in the protobuf
submodule.
- Revise the cron based conformance run to be a protobuf HEAD run, so it
happens on every PR and still happens on a cron as an early signal for
when something might need attention.

Author: thomasvl

.github/workflows/build.yml

@@ -1,7 +1,5 @@
 name: Build and Test
 
-# NOTE: If making changes to most of the steps, please also look to update
-# regular_conformance.yml also.
 permissions:
   contents: read
 
@@ -42,8 +40,12 @@ jobs:
       with:
         submodules: true
     - name: Update and install dependencies
-      # dependencies from https://github.com/protocolbuffers/protobuf/blob/main/src/README.md
-      run: apt-get update && apt-get install -y make g++ cmake
+      # Just ensure we have the latest `g++` for building protoc (since it is
+      # built by SwiftPM), full dependencies available at
+      # https://github.com/protocolbuffers/protobuf/blob/main/src/README.md
+      #
+      # `make` is needed for using our own `Makefile`.
+      run: apt-get update && apt-get install -y make g++
     - name: Build
       run: make build ${{ matrix.swift.hook }}
     - name: Test runtime
@@ -60,23 +62,6 @@ jobs:
       # The protoc build isn't part of Package@swift-5.10.swift
       if: ${{ matrix.swift.version != '5.10' }}
       run: make compile-tests
-    - name: Build protobuf
-      working-directory: Sources/protobuf/protobuf
-      # https://github.com/protocolbuffers/protobuf/blob/main/cmake/README.md#c-version
-      run: |
-        mkdir cmake_build
-        cd cmake_build
-        cmake \
-          -DCMAKE_CXX_STANDARD=17 \
-          -DCMAKE_BUILD_TYPE=Release \
-          -Dprotobuf_BUILD_TESTS=OFF \
-          -Dprotobuf_INSTALL=OFF \
-          -Dprotobuf_BUILD_CONFORMANCE=ON \
-          -S ..
-        NUM_CPUS=$(getconf _NPROCESSORS_ONLN)
-        make -j "${NUM_CPUS}" conformance_test_runner
-    - name: Test conformance
-      run: make test-conformance CONFORMANCE_TEST_RUNNER=Sources/protobuf/protobuf/cmake_build/conformance_test_runner
 
   api-breakage:
     name: Api Breakage Compared to main branch

.github/workflows/head_conformance.yml

@@ -0,0 +1,61 @@
+name: Run Protobuf Head Conformance Tests
+
+# This work workflow checks out head protocolbuffers/protobuf and runs the
+# conformances with that.
+#
+# It runs on all PRs/Commits and also runs on a Cron to keep an eye out for
+# new conformance tests that get added upstream.
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  # Every Sunday at 5am to get warning on any new conformance tests upstream.
+  schedule:
+    - cron: '0 5 * * 0'
+  # Also allow manual triggering from the github UX to revalidate things.
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container:
+      # Unlike build.yml we only worry about the conformance test with the
+      # latest Swift release.
+      image: swift:latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        path: swift-protobuf
+        submodules: true
+    - name: Update and install dependencies
+      # Dependencies from https://github.com/protocolbuffers/protobuf/blob/main/src/README.md
+      run: apt-get update && apt-get install -y make g++ cmake
+    - name: Checkout protobufbuffers/protobuf
+      uses: actions/checkout@v4
+      with:
+        repository: protocolbuffers/protobuf
+        path: protobuf
+    - name: Build protobuf
+      working-directory: protobuf
+      # https://github.com/protocolbuffers/protobuf/blob/main/cmake/README.md#c-version
+      run: |
+        mkdir cmake_build
+        cd cmake_build
+        cmake \
+          -DCMAKE_CXX_STANDARD=17 \
+          -DCMAKE_BUILD_TYPE=Release \
+          -Dprotobuf_BUILD_TESTS=OFF \
+          -Dprotobuf_INSTALL=OFF \
+          -Dprotobuf_BUILD_CONFORMANCE=ON \
+          -S ..
+        NUM_CPUS=$(getconf _NPROCESSORS_ONLN)
+        make -j "${NUM_CPUS}" conformance_test_runner
+    - name: Test conformance
+      working-directory: swift-protobuf
+      run: make test-conformance GOOGLE_PROTOBUF_CHECKOUT=../protobuf

.github/workflows/regular_conformance.yml

@@ -61,9 +61,9 @@ GENERATE_SRCS=${GENERATE_SRCS_BASE} -I Protos/Sources/SwiftProtobuf
 SWIFT_CONFORMANCE_PLUGIN=.build/debug/Conformance
 
 # Where to find the conformance-test-runner. Defaults to being in your protobuf
-# checkout. Invoke make with CONFORMANCE_TEST_RUNNER=[PATH_TO_BINARY] to
-# override this value.
-CONFORMANCE_TEST_RUNNER?=${GOOGLE_PROTOBUF_CHECKOUT}/conformance_test_runner
+# checkout when built with CMake. Invoke make with
+# CONFORMANCE_TEST_RUNNER=[PATH_TO_BINARY] to override this value.
+CONFORMANCE_TEST_RUNNER?=${GOOGLE_PROTOBUF_CHECKOUT}/cmake_build/conformance_test_runner
 
 # Hook to pass arge to swift build|test (mainly for the CI setup)
 SWIFT_BUILD_TEST_HOOK?=
@@ -76,6 +76,7 @@ PROTOS_DIRS=Sources/Conformance Sources/SwiftProtobuf Sources/SwiftProtobufPlugi
 	all \
 	build \
 	check \
+	check-for-conformance-runner \
 	check-for-protobuf-checkout \
 	check-proto-files \
 	check-version-numbers \
@@ -645,8 +646,16 @@ check-proto-files: check-for-protobuf-checkout
 	    rm -f _check_protos.txt; \
 	fi
 
+check-for-conformance-runner:
+	@if [ ! -x "${CONFORMANCE_TEST_RUNNER}" ]; then \
+	  echo "ERROR: ${CONFORMANCE_TEST_RUNNER} does not appear to exist"; \
+	  echo "ERROR:   built build it or set CONFORMANCE_TEST_RUNNER to point"; \
+	  echo "ERROR:   a runner."; \
+	  exit 1; \
+	fi
+
 # Runs the conformance tests.
-test-conformance: build check-for-protobuf-checkout Sources/Conformance/failure_list_swift.txt Sources/Conformance/text_format_failure_list_swift.txt
+test-conformance: check-for-conformance-runner build Sources/Conformance/failure_list_swift.txt Sources/Conformance/text_format_failure_list_swift.txt
 	$(CONFORMANCE_TEST_RUNNER) \
 	  --enforce_recommended \
 	  --failure_list Sources/Conformance/failure_list_swift.txt \