Skip to content

Commit 7227930

Browse files
dennisYehCienetdennisYehCienet
authored
feat(ci): Record vLLM and tpu-inference commit hashes (#795)
Signed-off-by: dennis yeh <dennis.yeh@cienet.com>
1 parent 83249b2 commit 7227930

File tree

10 files changed

+108
-39
lines changed

10 files changed

+108
-39
lines changed

‎.buildkite/main.yml‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,32 +27,35 @@ steps:
2727
commands:
2828
- bash .buildkite/scripts/generate_support_matrices.sh
2929

30-
# - label: "Notify test results"
31-
## if: build.env("NIGHTLY") == "1"
32-
# key: notify_test_results
33-
# depends_on: generate_support_matrices
34-
# agents:
35-
# queue: cpu
36-
# commands:
37-
# - bash .buildkite/scripts/notify_test_results.sh
38-
39-
# - label: "Record verified commit hashes"
40-
# key: record_verified_commit_hashes
41-
# depends_on: notify_test_results
42-
# branches: "main"
43-
# agents:
44-
# queue: cpu
45-
# commands:
46-
# # TODO : record verified vllm-tpu_inference commit hash pair if test successful (to be handled in separate PR)
47-
# - echo "committing verified commit hashes"
48-
#
49-
- label: "Commit support matrices"
30+
- label: "Commit support matrices if is nightly build or release tag specified"
5031
<<: *run-on-nightly-or-tag
51-
key: commit_nightly_support_matrices
32+
key: commit_support_matrices
5233
depends_on: generate_support_matrices
5334
agents:
5435
queue: cpu
5536
secrets:
5637
- GITHUB_PAT
5738
command:
5839
- bash .buildkite/scripts/commit_support_matrices.sh
40+
41+
- label: "Notify test results"
42+
<<: *run-on-nightly-or-tag
43+
key: notify_test_results
44+
depends_on: commit_support_matrices
45+
agents:
46+
queue: cpu
47+
commands:
48+
- bash .buildkite/scripts/notify_test_results.sh
49+
50+
- label: "Record verified commit hashes"
51+
if: build.env("NIGHTLY") == "1"
52+
key: record_verified_commit_hashes
53+
depends_on:
54+
- notify_test_results
55+
- tpu_test_notification
56+
agents:
57+
queue: cpu
58+
secrets:
59+
- GITHUB_PAT
60+
command:
61+
- bash .buildkite/scripts/commit_verified_commit_hashes.sh

‎.buildkite/pipeline_jax.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ steps:
265265
# NOTIFICATION STEP
266266
# -----------------------------------------------------------------
267267
- label: "TPU Test Notification"
268+
key: tpu_test_notification
268269
depends_on:
269270
- test_0
270271
- test_1

‎.buildkite/scripts/bootstrap.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ else
2121
fi
2222

2323
upload_pipeline() {
24+
VLLM_COMMIT_HASH=$(git ls-remote https://github.com/vllm-project/vllm.git HEAD | awk '{ print $1}')
25+
buildkite-agent meta-data set "VLLM_COMMIT_HASH" "${VLLM_COMMIT_HASH}"
26+
echo "Using vllm commit hash: $(buildkite-agent meta-data get "VLLM_COMMIT_HASH")"
2427
buildkite-agent pipeline upload .buildkite/pipeline_jax.yml
2528
# buildkite-agent pipeline upload .buildkite/pipeline_torch.yml
2629
buildkite-agent pipeline upload .buildkite/main.yml

‎.buildkite/scripts/check_results.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@ done
2323

2424
if [ "${ANY_FAILED}" = "true" ] ; then
2525
cat <<- YAML | buildkite-agent pipeline upload
26-
steps:
27-
- label: "${FAILURE_LABEL}"
28-
agents:
29-
queue: cpu
30-
command: echo "${FAILURE_LABEL}"
26+
steps:
27+
- label: "${FAILURE_LABEL}"
28+
agents:
29+
queue: cpu
30+
command: echo "${FAILURE_LABEL}"
3131
YAML
3232
exit 1
3333
else

‎.buildkite/scripts/commit_support_matrices.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ git config user.name "Buildkite Bot"
3030
git config user.email "buildkite-bot@users.noreply.github.com"
3131

3232
echo "--- Fetching and checking out the target branch"
33-
git fetch origin ${TARGET_BRANCH}
34-
git checkout ${TARGET_BRANCH}
35-
git reset --hard origin/${TARGET_BRANCH}
33+
git fetch origin "${TARGET_BRANCH}"
34+
git checkout "${TARGET_BRANCH}"
35+
git reset --hard origin/"${TARGET_BRANCH}"
3636

3737
echo "--- Downloading CSV artifacts"
3838
mkdir -p "${ARTIFACT_DOWNLOAD_PATH}"

‎.buildkite/scripts/commit_verified_commit_hashes.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/sh
2+
set -e
3+
4+
# --- Configuration ---
5+
REPO_URL="https://github.com/vllm-project/tpu-inference.git"
6+
TARGET_BRANCH="main"
7+
8+
COMMIT_MESSAGE="Update verified commit hashes"
9+
10+
# Construct the repository URL with the access token for authentication.
11+
AUTHENTICATED_REPO_URL="https://x-access-token:${GITHUB_PAT}@${REPO_URL#https://}"
12+
13+
# Ensure the GITHUB_PAT is available before proceeding.
14+
if [ -z "${GITHUB_PAT:-}" ]; then
15+
echo "--- ERROR: GITHUB_PAT secret not found. Cannot proceed."
16+
exit 1
17+
fi
18+
19+
echo "--- Configuring Git user details"
20+
git config user.name "Buildkite Bot"
21+
git config user.email "buildkite-bot@users.noreply.github.com"
22+
23+
echo "--- Fetching and checking out the target branch"
24+
git fetch origin "${TARGET_BRANCH}"
25+
git checkout "${TARGET_BRANCH}"
26+
git reset --hard origin/"${TARGET_BRANCH}"
27+
28+
VLLM_COMMIT_HASH=$(buildkite-agent meta-data get "VLLM_COMMIT_HASH" --default "")
29+
30+
if [ -z "${VLLM_COMMIT_HASH}" ]; then
31+
echo "VLLM_COMMIT_HASH not found in buildkite meta-data"
32+
exit 1
33+
fi
34+
35+
if [ -z "${BUILDKITE_COMMIT:-}" ]; then
36+
echo "BUILDKITE_COMMIT not found"
37+
exit 1
38+
fi
39+
40+
if [ ! -f verified_commit_hashes.csv ]; then
41+
echo "timestamp,vllm_commit_hash,tpu_inference_commit_hash" > verified_commit_hashes.csv
42+
fi
43+
echo "$(date '+%Y-%m-%d %H:%M:%S'),${VLLM_COMMIT_HASH},${BUILDKITE_COMMIT}" >> verified_commit_hashes.csv
44+
45+
git add verified_commit_hashes.csv
46+
47+
# --- Check for changes before committing ---
48+
if git diff --quiet --cached; then
49+
echo "No changes to commit. Exiting successfully."
50+
exit 0
51+
else
52+
echo "--- Committing changes"
53+
git commit -s -m "${COMMIT_MESSAGE}"
54+
55+
echo "--- Pushing changes to '${TARGET_BRANCH}'"
56+
git push "${AUTHENTICATED_REPO_URL}" "HEAD:${TARGET_BRANCH}"
57+
fi

‎.buildkite/scripts/notify_test_results.sh‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,8 @@ FAILURE_LABEL="Not all models and/or features passed"
77
echo "--- Checking test outcomes"
88

99
if [ "${ANY_FAILED}" = "true" ] ; then
10-
cat <<- YAML | buildkite-agent pipeline upload
11-
steps:
12-
- label: "${FAILURE_LABEL}"
13-
agents:
14-
queue: tpu_v6_queue
15-
command: echo "${FAILURE_LABEL}"
16-
YAML
17-
exit 1
10+
echo "${FAILURE_LABEL}"
11+
exit 1
1812
else
19-
echo "All models & features passed."
13+
echo "All models & features passed."
2014
fi

‎.buildkite/scripts/run_in_docker.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,11 @@ python3 -m pip install --progress-bar off buildkite-test-collector==0.1.9
9494
echo "Python dependencies installed"
9595

9696
IMAGE_NAME="vllm-tpu"
97-
docker build --no-cache -f docker/Dockerfile -t "${IMAGE_NAME}:${BUILDKITE_COMMIT}" .
97+
VLLM_COMMIT_HASH=$(buildkite-agent meta-data get "VLLM_COMMIT_HASH" --default "")
98+
99+
docker build \
100+
--build-arg VLLM_COMMIT_HASH="${VLLM_COMMIT_HASH}" \
101+
--no-cache -f docker/Dockerfile -t "${IMAGE_NAME}:${BUILDKITE_COMMIT}" .
98102

99103
exec docker run \
100104
--privileged \

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ cython_debug/
175175

176176
# Results
177177
*.csv
178+
!verified_commit_hashes.csv
178179

179180
# Python pickle files
180181
*.pkl

‎docker/Dockerfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
ARG NIGHTLY_DATE="20250714"
22
ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.12_tpuvm_$NIGHTLY_DATE"
3+
# The latest main will be used if arg unspecified
4+
ARG VLLM_COMMIT_HASH=""
35

46
FROM $BASE_IMAGE
57

@@ -16,7 +18,11 @@ RUN apt-get update && apt-get install -y \
1618
# Build vLLM
1719
WORKDIR /workspace/vllm
1820
ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
21+
ARG VLLM_COMMIT_HASH
1922
RUN git clone $VLLM_REPO /workspace/vllm
23+
RUN if [ -n "$VLLM_COMMIT_HASH" ]; then \
24+
git checkout $VLLM_COMMIT_HASH; \
25+
fi
2026
RUN pip install -r requirements/tpu.txt --retries 3
2127
RUN VLLM_TARGET_DEVICE="tpu" pip install -e .
2228

0 commit comments

Comments
 (0)