diff --git a/build-bot/CheckActionsService.sh b/build-bot/CheckActionsService.sh new file mode 100644 index 0000000..9631e3b --- /dev/null +++ b/build-bot/CheckActionsService.sh @@ -0,0 +1,2 @@ +#! /bin/bash +ansible-playbook -i inventory.ini CheckActionsService.yaml diff --git a/build-bot/CheckActionsService.yaml b/build-bot/CheckActionsService.yaml new file mode 100644 index 0000000..a8af250 --- /dev/null +++ b/build-bot/CheckActionsService.yaml @@ -0,0 +1,20 @@ +--- +- name: Check for Actions Runner Service + hosts: bbots_linux + gather_facts: no # Skips gathering OS info to make it run faster + + tasks: + - name: Search for the service + # We use 'shell' instead of 'command' so we can use the pipe (|) + shell: "systemctl list-units --type=service --state=running | grep actions.runner" + register: grep_output + # Grep returns exit code 1 if nothing is found. + # We tell Ansible: "Only consider this a failure if the code is > 1" (actual errors) + failed_when: grep_output.rc > 1 + changed_when: false + + - name: Display the found line + debug: + msg: "{{ grep_output.stdout }}" + # Only print the message if grep actually found something (return code 0) + when: grep_output.rc == 0 \ No newline at end of file diff --git a/build-bot/UpdateBBots.sh b/build-bot/UpdateBBots.sh new file mode 100644 index 0000000..f66e9e3 --- /dev/null +++ b/build-bot/UpdateBBots.sh @@ -0,0 +1,2 @@ +#! /bin/bash +ansible-playbook -i inventory.ini UpdateBBots.yml -K diff --git a/build-bot/UpdateBBots.yml b/build-bot/UpdateBBots.yml new file mode 100644 index 0000000..fefda89 --- /dev/null +++ b/build-bot/UpdateBBots.yml @@ -0,0 +1,35 @@ +--- +- name: Update MDOLAB BuildBots + hosts: bbots_linux + become: yes # Requires sudo password + timeout: 1200 # 20 minute time limit + + tasks: + - name: Update apt repo and cache + apt: + update_cache: yes + cache_valid_time: 3600 + + - name: Upgrade all packages (dist-upgrade) + apt: + upgrade: dist + + - name: Remove useless packages + apt: + autoclean: yes + + - name: Remove unused dependencies + apt: + autoremove: yes + + - name: Check for reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Reboot if required + reboot: + msg: "Rebooting for updates" + pre_reboot_delay: 0 + post_reboot_delay: 30 + when: reboot_required_file.stat.exists diff --git a/build-bot/UpdateDocker.sh b/build-bot/UpdateDocker.sh new file mode 100644 index 0000000..e4d825a --- /dev/null +++ b/build-bot/UpdateDocker.sh @@ -0,0 +1,2 @@ +#! /bin/bash +ansible-playbook -i inventory.ini UpdateDocker.yml -K diff --git a/build-bot/UpdateDocker.yml b/build-bot/UpdateDocker.yml new file mode 100644 index 0000000..5455102 --- /dev/null +++ b/build-bot/UpdateDocker.yml @@ -0,0 +1,88 @@ +--- +- name: Install/Update Docker CE (Official Repo) + hosts: bbots_linux + become: yes + vars: + # We define the list of old packages to remove exactly as listed in your command + conflicting_packages: + - docker.io + - docker-compose + - docker-compose-v2 + - docker-doc + - podman-docker + - containerd + - runc + + tasks: + # --- NEW: CLEANUP SECTION --- + - name: Remove legacy Docker source list file + file: + path: /etc/apt/sources.list.d/docker.list + state: absent + + - name: Remove legacy Docker GPG key (optional but good for hygiene) + file: + path: /usr/share/keyrings/docker-archive-keyring.gpg + state: absent + # ---------------------------- + # 1. Uninstall all conflicting packages + # Equivalent to: sudo apt remove ... + - name: Remove conflicting/old Docker packages + apt: + name: "{{ conflicting_packages }}" + state: absent + purge: yes + + # 2. Setup Keyrings Directory + # Equivalent to: sudo install -m 0755 -d /etc/apt/keyrings + - name: Create directory for apt keyrings + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + # 3. Download Docker's GPG Key + # Equivalent to: sudo curl ... -o /etc/apt/keyrings/docker.asc + - name: Download Docker's official GPG key + get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' # Equivalent to chmod a+r + + # 4. Add the Repository + # Equivalent to: sudo tee /etc/apt/sources.list.d/docker.sources ... + # We use {{ ansible_distribution_release }} to automatically fetch "jammy", "noble", etc. + - name: Add Docker repository to Apt sources + copy: + dest: /etc/apt/sources.list.d/docker.sources + content: | + Types: deb + URIs: https://download.docker.com/linux/ubuntu + Suites: {{ ansible_distribution_release }} + Components: stable + Signed-By: /etc/apt/keyrings/docker.asc + mode: '0644' + + # 5. Install Docker + # Equivalent to: sudo apt update && sudo apt install ... + - name: Install the latest version of Docker + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: latest + update_cache: yes + + # 6. Verify Installation + # Equivalent to: sudo docker run hello-world + - name: Verify Docker installation + command: docker run --rm hello-world + register: docker_test + changed_when: false + + - name: Show verification output + debug: + msg: "{{ docker_test.stdout_lines }}" diff --git a/build-bot/inventory.ini b/build-bot/inventory.ini new file mode 100644 index 0000000..8141f70 --- /dev/null +++ b/build-bot/inventory.ini @@ -0,0 +1,19 @@ +# --- The Linux Group --- +[bbots_linux] +bbot1 +bbot2 +bbot3 +bbot4 +bbot5 +bbot6 +bbot8 +bbot9 + +# --- The Mac Group --- +[bbots_mac] +bbot7 + +# --- The Parent Group (so you can still target 'all') --- +[bbots:children] +bbots_linux +bbots_mac \ No newline at end of file diff --git a/build-bot/readme.md b/build-bot/readme.md index 303754a..f711756 100644 --- a/build-bot/readme.md +++ b/build-bot/readme.md @@ -14,3 +14,54 @@ This directory contains the script that should almost entirely automate the setu 4. Run `bash buildbot_setup.sh` and follow the prompts 5. If the process finished successfully, check that the actions runner is running using `systemctl list-units --type=service --state=running`, you should see a line that looks like `actions.runner.mdolab.mdolabbuildbot-N.service loaded active running GitHub Actions Runner (mdolab.mdolabbuildbot-N)` 6. Just to be sure everything is working, reboot the machine and repeat the check above + +## Maintaining buildbots via ssh + +For the purposes of maintaining and updating the buildbots, it is significantly to use ssh than to physically access the machines. +To do this, setup your `~/.ssh/config` file to include the following entries for buildbots 1-9: + +```text +host bbot? + User mdolab_mnt + ControlMaster auto + ControlPath ~/.ssh/sockets/%r@%h-%p + ControlPersist 600 + +host bbot1 + Hostname + +host bbot2 + Hostname +. +. +. +``` + +You will need to use password authentication the first time you connect to each buildbot (you may need to add `PubkeyAuthentication no` under `host bbot?` to force this), then you can add your public key to the buildbots for passwordless authentication in the future by running: + +```bash +ssh-copy-id -i bbot +``` + +for each buildbot. + +### Ansible playbooks + +This directory also contains some "Ansible playbooks" that can be run to run maintenance tasks on all the linux buildbots at once via ssh. +Once you've installed Ansible (`pip install ansible`), you can run the playbooks like so: + +```bash +ansible-playbook -i inventory.ini CheckActionsService.yaml +``` + +Will check that the actions runner service is running on all buildbots. + +```bash +ansible-playbook -i inventory.ini UpdateBBots.yml -K +``` + +Will update the apt packages on all buildbots and reboot them if necessary. + +These commands can also be run via the bash scripts `CheckActionsService.sh` and `UpdateBBots.sh` respectively. + +Note that these commands rely on you having the buildbots set up in your `~/.ssh/config` file as described above, and do not run on the mac buildbot (bbot7).