Skip to content

Commit 20d0478

Browse files
clee2000clee2000
committed
copy build tutorials to reusable workflow file
1 parent 9dc8613 commit 20d0478

File tree

1 file changed

+183
-0
lines changed

1 file changed

+183
-0
lines changed

‎.github/workflows/_build-tutorials-base.yml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
name: Build tutorials
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
worker:
15+
name: pytorch_tutorial_build_worker
16+
strategy:
17+
matrix:
18+
include:
19+
- { shard: 1, num_shards: 15, runner: "linux.g5.12xlarge.nvidia.gpu" }
20+
- { shard: 2, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
21+
- { shard: 3, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
22+
- { shard: 4, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
23+
- { shard: 5, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
24+
- { shard: 6, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
25+
- { shard: 7, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
26+
- { shard: 8, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
27+
- { shard: 9, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
28+
- { shard: 10, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
29+
- { shard: 11, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
30+
- { shard: 12, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
31+
- { shard: 13, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
32+
- { shard: 14, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
33+
- { shard: 15, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
34+
fail-fast: false
35+
runs-on: ${{ matrix.runner }}
36+
steps:
37+
- name: Setup SSH (Click me for login details)
38+
uses: pytorch/test-infra/.github/actions/setup-ssh@main
39+
with:
40+
github-secret: ${{ secrets.GITHUB_TOKEN }}
41+
instructions: |
42+
All testing is done inside the container, to start an interactive session run:
43+
docker exec -it $(docker container ps --format '{{.ID}}') bash
44+
45+
- name: Checkout Tutorials
46+
uses: actions/checkout@v3
47+
with:
48+
fetch-depth: 0
49+
50+
- name: Setup Linux
51+
uses: pytorch/pytorch/.github/actions/setup-linux@main
52+
53+
- name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
54+
uses: pytorch/test-infra/.github/actions/setup-nvidia@main
55+
56+
- name: Calculate/build docker image
57+
id: calculate-docker-image
58+
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
59+
with:
60+
docker-image-name: tutorials
61+
62+
- name: Pull docker image
63+
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
64+
with:
65+
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
66+
67+
- name: Build
68+
shell: bash
69+
env:
70+
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
71+
NUM_WORKERS: ${{ matrix.num_shards }}
72+
WORKER_ID: ${{ matrix.shard }}
73+
COMMIT_ID: ${{ github.sha }}
74+
JOB_TYPE: worker
75+
COMMIT_SOURCE: ${{ github.ref }}
76+
run: |
77+
set -ex
78+
79+
chmod +x ".jenkins/build.sh"
80+
81+
container_name=$(docker run \
82+
${GPU_FLAG:-} \
83+
-e WORKER_ID \
84+
-e NUM_WORKERS \
85+
-e COMMIT_ID \
86+
-e JOB_TYPE \
87+
-e COMMIT_SOURCE \
88+
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
89+
--tty \
90+
--detach \
91+
--shm-size=2gb \
92+
--name="${container_name}" \
93+
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
94+
-w /var/lib/workspace \
95+
"${DOCKER_IMAGE}"
96+
)
97+
98+
docker exec -u ci-user -t "${container_name}" sh -c ".jenkins/build.sh"
99+
100+
- name: Teardown Linux
101+
uses: pytorch/test-infra/.github/actions/teardown-linux@main
102+
if: always()
103+
104+
manager:
105+
name: pytorch_tutorial_build_manager
106+
needs: worker
107+
runs-on: [self-hosted, linux.2xlarge]
108+
environment: ${{ github.ref == 'refs/heads/main' && 'pytorchbot-env' || '' }}
109+
steps:
110+
- name: Setup SSH (Click me for login details)
111+
uses: pytorch/test-infra/.github/actions/setup-ssh@main
112+
with:
113+
github-secret: ${{ secrets.GITHUB_TOKEN }}
114+
instructions: |
115+
All testing is done inside the container, to start an interactive session run:
116+
docker exec -it $(docker container ps --format '{{.ID}}') bash
117+
118+
- name: Checkout Tutorials
119+
uses: actions/checkout@v3
120+
with:
121+
fetch-depth: 0
122+
123+
- name: Setup Linux
124+
uses: pytorch/pytorch/.github/actions/setup-linux@main
125+
126+
- name: Calculate/build docker image
127+
id: calculate-docker-image
128+
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
129+
with:
130+
docker-image-name: tutorials
131+
132+
- name: Pull docker image
133+
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
134+
with:
135+
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
136+
137+
- name: Build
138+
shell: bash
139+
env:
140+
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
141+
NUM_WORKERS: 15
142+
WORKER_ID: ${{ matrix.shard }}
143+
COMMIT_ID: ${{ github.sha }}
144+
JOB_TYPE: manager
145+
COMMIT_SOURCE: ${{ github.ref }}
146+
GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.PYTORCHBOT_TOKEN }}
147+
run: |
148+
set -ex
149+
150+
chmod +x ".jenkins/build.sh"
151+
152+
container_name=$(docker run \
153+
${GPU_FLAG:-} \
154+
-e WORKER_ID \
155+
-e NUM_WORKERS \
156+
-e COMMIT_ID \
157+
-e JOB_TYPE \
158+
-e COMMIT_SOURCE \
159+
-e GITHUB_PYTORCHBOT_TOKEN \
160+
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
161+
--tty \
162+
--detach \
163+
--name="${container_name}" \
164+
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
165+
-w /var/lib/workspace \
166+
"${DOCKER_IMAGE}"
167+
)
168+
169+
docker exec -u ci-user -t "${container_name}" sh -c ".jenkins/build.sh"
170+
171+
- name: Upload docs preview
172+
uses: seemethere/upload-artifact-s3@v5
173+
if: ${{ github.event_name == 'pull_request' }}
174+
with:
175+
retention-days: 14
176+
s3-bucket: doc-previews
177+
if-no-files-found: error
178+
path: docs
179+
s3-prefix: pytorch/tutorials/${{ github.event.pull_request.number }}
180+
181+
- name: Teardown Linux
182+
uses: pytorch/test-infra/.github/actions/teardown-linux@main
183+
if: always()

0 commit comments

Comments
 (0)