diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cab495a46..199153d1d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,60 +2,75 @@ name: tests on: [push, pull_request] -env: - DOCKER_REGISTRY: ghcr.io/commaai - RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name msgq msgq /bin/sh -c - RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm msgq /bin/sh -c - CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm msgqci /bin/bash -c - BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/msgq:latest -t msgq -f Dockerfile . - PYTHONWARNINGS: error - jobs: - build: - name: build - runs-on: ubuntu-latest + + static_analysis: + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: Push to dockerhub - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/msgq' - run: | - docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - docker tag msgq $DOCKER_REGISTRY/msgq:latest - docker push $DOCKER_REGISTRY/msgq:latest + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Installing requirements + run: + python -m pip install -r requirements.txt + - name: Installing ubuntu requirements + run: "sudo apt-get install -y cppcheck" + - name: Static analysis + run: pre-commit run --all + + scons_test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"] + python: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Installing python requirements + run: + python -m pip install -r requirements.txt + - name: Installing ubuntu requirements + if: ${{startsWith(matrix.os, 'ubuntu')}} + run: + scripts/ubuntu_dependencies.sh + - name: Installing macos requirements + if: ${{startsWith(matrix.os, 'macos')}} + run: + scripts/macos_dependencies.sh + - name: Building + run: scons -j$(nproc || sysctl -n hw.logicalcpu) + - unit_tests: - name: unit tests - runs-on: ubuntu-latest + full_tests: + runs-on: ${{ matrix.os }} strategy: matrix: + os: ["ubuntu-20.04", "ubuntu-24.04"] + python: ["3.11", "3.12"] flags: ['', '--asan', '--ubsan'] backend: ['MSGQ', 'ZMQ'] steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Installing ubuntu requirements + run: + scripts/ubuntu_dependencies.sh + - name: Building msgq + run: | + pip3 install -e .[dev] + - name: Python tests + run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append - name: C++ tests run: | - $RUN "export ${{ matrix.backend }}=1 && \ - scons ${{ matrix.flags }} -j$(nproc) && \ - msgq/test_runner && \ - msgq/visionipc/test_runner" - - name: python tests - run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m pytest" + export ${{ matrix.backend }}=1 + scons ${{ matrix.flags }} -j$(nproc) + msgq/test_runner + msgq/visionipc/test_runner - name: Upload coverage - run: | - docker commit msgq msgqci - $CI_RUN "cd /project/msgq && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}" - - static_analysis: - name: static analysis - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: Static analysis - # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed - run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all" + run: "bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..57ef0fa3d --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,50 @@ +name: wheels + +on: [push, pull_request] + +jobs: + wheels: + runs-on: ${{ matrix.platform.os }} + strategy: + fail-fast: true + matrix: + platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64", arch: "x86_64"}, + {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"}, + {os: "macos-14", target: "macosx_arm64", arch: "arm64"} ] + python: [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ] + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Building wheel + uses: pypa/cibuildwheel@v2.19.2 + env: + CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}" + CIBW_ARCHS: "${{ matrix.platform.arch }}" + CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh" + CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" + CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28" + MACOSX_DEPLOYMENT_TARGET: "14.0" + + - uses: actions/setup-python@v5 + if: ${{ matrix.platform.arch != 'aarch64' }} + with: + python-version: ${{ matrix.python.py }} + + - name: Installing the wheel + if: ${{ matrix.platform.arch != 'aarch64' }} + run: | + pip install --break-system-packages ./wheelhouse/*.whl + + - name: Saving wheel + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }} + path: ./wheelhouse/*.whl diff --git a/.gitignore b/.gitignore index 6db3c7cd4..d7e38f211 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ services.h .sconsign.dblite libcereal_shared.* .mypy_cache/ +msgq.egg-info +build/ +dist/ +catch2/ diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 77ef04c29..000000000 --- a/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -FROM ubuntu:24.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - build-essential \ - ca-certificates \ - capnproto \ - clang \ - cppcheck \ - curl \ - git \ - libbz2-dev \ - libcapnp-dev \ - libclang-rt-dev \ - libffi-dev \ - liblzma-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libzmq3-dev \ - llvm \ - make \ - cmake \ - ocl-icd-opencl-dev \ - opencl-headers \ - python3-dev \ - python3-pip \ - tk-dev \ - wget \ - xz-utils \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy pytest - -WORKDIR /project/msgq/ -RUN cd /tmp/ && \ - git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \ - cd Catch2 && \ - mv single_include/* /project/msgq/ && \ - cd .. \ - rm -rf Catch2 - -WORKDIR /project/msgq - -ENV PYTHONPATH=/project - -COPY . . -RUN ls && rm -rf .git && \ - scons -c && scons -j$(nproc) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..a65dcef48 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +exclude msgq/**/*.o +include SConscript +include SConstruct +recursive-include site_scons * diff --git a/SConstruct b/SConstruct index c1e7fc521..476672be5 100644 --- a/SConstruct +++ b/SConstruct @@ -61,6 +61,7 @@ env = Environment( "-Werror", "-Wshadow", "-Wno-vla-cxx-extension", + "-Wno-unknown-warning-option", ] + ccflags, LDFLAGS=ldflags, LINKFLAGS=ldflags, diff --git a/pyproject.toml b/pyproject.toml index c0d9b3f40..bd7b187c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,30 @@ +[project] +name = "msgq" +requires-python = ">= 3.11" +version = "0.1.0" + +dependencies = [ + "numpy", + "Cython", + "pycapnp", + "setuptools", +] + +[project.optional-dependencies] +dev = [ + "scons", + "pre-commit", + "ruff", + "parameterized", + "coverage", + "pytest", + "pytest-cov", +] + +[build-system] +requires = ["setuptools","scons","Cython","numpy"] +build-backend = "setuptools.build_meta" + # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml [tool.ruff] lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..e179397e2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,50 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --all-extras -o requirements.txt pyproject.toml +cfgv==3.4.0 + # via pre-commit +coverage==7.6.0 + # via + # msgq (pyproject.toml) + # pytest-cov +cython==3.0.10 + # via msgq (pyproject.toml) +distlib==0.3.8 + # via virtualenv +filelock==3.15.4 + # via virtualenv +identify==2.6.0 + # via pre-commit +iniconfig==2.0.0 + # via pytest +nodeenv==1.9.1 + # via pre-commit +numpy==2.0.0 + # via msgq (pyproject.toml) +packaging==24.1 + # via pytest +parameterized==0.9.0 + # via msgq (pyproject.toml) +platformdirs==4.2.2 + # via virtualenv +pluggy==1.5.0 + # via pytest +pre-commit==3.7.1 + # via msgq (pyproject.toml) +pycapnp==2.0.0 + # via msgq (pyproject.toml) +pytest==8.2.2 + # via + # msgq (pyproject.toml) + # pytest-cov +pytest-cov==5.0.0 + # via msgq (pyproject.toml) +pyyaml==6.0.1 + # via pre-commit +ruff==0.5.3 + # via msgq (pyproject.toml) +scons==4.8.0 + # via msgq (pyproject.toml) +setuptools==71.0.4 + # via msgq (pyproject.toml) +virtualenv==20.26.3 + # via pre-commit diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh new file mode 100755 index 000000000..bac87f705 --- /dev/null +++ b/scripts/macos_dependencies.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +ROOT="$(cd $DIR/../ && pwd)" +ARCH=$(uname -m) + +if [[ $SHELL == "/bin/zsh" ]]; then + RC_FILE="$HOME/.zshrc" +elif [[ $SHELL == "/bin/bash" ]]; then + RC_FILE="$HOME/.bash_profile" +fi + +# Install brew if required +if [[ $(command -v brew) == "" ]]; then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" + + if [[ $ARCH == "x86_64" ]]; then + echo 'eval "$(/usr/local/homebrew/bin/brew shellenv)"' >> $RC_FILE + eval "$(/usr/local/homebrew/bin/brew shellenv)" + else + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> $RC_FILE + eval "$(/opt/homebrew/bin/brew shellenv)" + fi +fi + +brew bundle --file=- <<-EOS +brew "zeromq" +cask "gcc-arm-embedded" +brew "gcc@13" +EOS + +if [[ -z "$NO_CATCH2" ]]; then + cd /tmp + git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git + cd Catch2 + mv single_include/* "$ROOT" + rm -rf Catch2 +fi diff --git a/scripts/manylinux_dependencies.sh b/scripts/manylinux_dependencies.sh new file mode 100755 index 000000000..9bd69863f --- /dev/null +++ b/scripts/manylinux_dependencies.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +dnf install -y clang opencl-headers ocl-icd-devel cppcheck wget + +wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz +tar -xvf zeromq-4.3.5.tar.gz +cd zeromq-4.3.5 +./configure +make -j$(nproc) +make install diff --git a/scripts/ubuntu_dependencies.sh b/scripts/ubuntu_dependencies.sh new file mode 100755 index 000000000..855a937fe --- /dev/null +++ b/scripts/ubuntu_dependencies.sh @@ -0,0 +1,22 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +ROOT=$DIR/../ + +if [[ ! $(id -u) -eq 0 ]]; then + if [[ -z $(which sudo) ]]; then + echo "Please install sudo or run as root" + exit 1 + fi + SUDO="sudo" +fi + +$SUDO apt-get update +$SUDO apt-get install -y --no-install-recommends clang opencl-headers libzmq3-dev ocl-icd-opencl-dev cppcheck + +if [[ -z "$NO_CATCH2" ]]; then + cd /tmp + git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git + cd Catch2 + mv single_include/* "$ROOT" + rm -rf Catch2 +fi diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..fb0eafd08 --- /dev/null +++ b/setup.py @@ -0,0 +1,31 @@ +from setuptools import Command, setup, Distribution +from setuptools.command.build import build +import subprocess +import os + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + +class SconsBuild(Command): + def initialize_options(self) -> None: + pass + + def finalize_options(self) -> None: + pass + + def run(self) -> None: + subprocess.run([f"scons --minimal -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode() + + +class CustomBuild(build): + sub_commands = [('scons_build', None)] + build.sub_commands + +setup( + packages = ["msgq", "msgq.visionipc"], + package_data={'': ['**/*.cc', '**/*.c', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.py', '**/*.so', '**/*.npy']}, + include_package_data=True, + cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}, + distclass=BinaryDistribution, + )