diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7631560..fb5eab0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,55 +1,46 @@ name: Build on: [push] -env: - KERNEL_VERSION: 5.15.63-ctsi-2 - RISCV_PK_VERSION: 1.0.0-ctsi-1 - TOOLCHAIN_VERSION: 0.13.0 jobs: build: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + + - name: Retrieve environment variables + run: make env >> $GITHUB_ENV - name: Copy default Cartesi Linux config run: make cartesi-linux-config - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Docker meta id: docker_meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ${{ secrets.DOCKER_ORGANIZATION }}/linux-kernel tags: | type=ref,event=branch type=semver,pattern={{version}} - - name: Login to Docker Hub - uses: docker/login-action@v1 + - name: Login to GHCR + uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Download linux - run: > - gh release download -D dep/ -R ${{ github.repository_owner }}/linux v${{ env.KERNEL_VERSION }} --archive=tar.gz - env: - GITHUB_TOKEN: ${{ secrets.CI_TOKEN }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Download riscv-pk - run: > - gh release download -D dep/ -R ${{ github.repository_owner }}/riscv-pk v${{ env.RISCV_PK_VERSION }} --archive=tar.gz - env: - GITHUB_TOKEN: ${{ secrets.CI_TOKEN }} + - name: Download dependencies + run: make download - name: Build docker image id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . builder: ${{ steps.buildx.outputs.name }} @@ -58,31 +49,32 @@ jobs: push: false load: true build-args: | - TOOLCHAIN_REPOSITORY=${{ secrets.DOCKER_ORGANIZATION }}/toolchain - TOOLCHAIN_VERSION=${{ env.TOOLCHAIN_VERSION }} KERNEL_VERSION=${{ env.KERNEL_VERSION }} + IMAGE_KERNEL_VERSION=${{ env.IMAGE_KERNEL_VERSION }} RISCV_PK_VERSION=${{ env.RISCV_PK_VERSION }} cache-from: type=gha,mode=xmax,scope=regular cache-to: type=gha,scope=regular - - name: Export linux.bin artifact + - name: Export artifacts run: make copy IMG=`echo "${{ steps.docker_meta.outputs.tags }}" | head -1 | cut -d "," -f 1 | xargs` + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: artifacts/* + - uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/v') with: prerelease: true files: | - linux-headers-${{ env.KERNEL_VERSION }}.tar.xz - linux-nobbl-${{ env.KERNEL_VERSION }}.bin - linux-${{ env.KERNEL_VERSION }}.bin - linux-selftest-${{ env.KERNEL_VERSION }}.ext2 + artifacts/* env: GITHUB_TOKEN: ${{ secrets.CI_TOKEN }} - name: Push docker image id: docker_push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 if: ${{ startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/develop') }} with: context: . @@ -92,9 +84,8 @@ jobs: push: true load: false build-args: | - TOOLCHAIN_REPOSITORY=${{ secrets.DOCKER_ORGANIZATION }}/toolchain - TOOLCHAIN_VERSION=${{ env.TOOLCHAIN_VERSION }} KERNEL_VERSION=${{ env.KERNEL_VERSION }} + IMAGE_KERNEL_VERSION=${{ env.IMAGE_KERNEL_VERSION }} RISCV_PK_VERSION=${{ env.RISCV_PK_VERSION }} cache-from: type=gha,mode=max,scope=regular cache-to: type=gha,scope=regular diff --git a/.gitignore b/.gitignore index 2036887..cc615d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,4 @@ -linux-*.elf -linux-*.bin -linux-nobbl-*.bin -linux-headers-*.tar.xz -linux-selftest-*.ext2 +artifacts/* +linux-*.tar.gz +riscv-pk-*.tar.gz cartesi-linux-config -dep diff --git a/CHANGELOG.md b/CHANGELOG.md index bd5e8a7..7506d1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ## Changed - Make kernel build timestamp deterministic +- Updated CI downloads to public infrastructure +- Updated CI actions versions +- Added deb generation and upload to CI as artifact ## [0.16.0] - 2023-03-30 ### Changed diff --git a/Dockerfile b/Dockerfile index 9b14aef..6568c99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,7 @@ # the License. # -ARG TOOLCHAIN_REPOSITORY=cartesi/toolchain -ARG TOOLCHAIN_VERSION=latest -FROM ${TOOLCHAIN_REPOSITORY}:${TOOLCHAIN_VERSION} +FROM debian:bookworm LABEL maintainer="Diego Nehab " @@ -25,36 +23,67 @@ ENV DEBIAN_FRONTEND=noninteractive ENV OLDPATH=$PATH +ENV BASE=/opt/riscv ENV BUILD_BASE=$BASE/kernel # setup dirs # ------------------------------------------------------------------------------ RUN \ + useradd developer && \ mkdir -p ${BUILD_BASE}/artifacts && \ chown -R developer:developer ${BUILD_BASE} && \ chmod go+w ${BUILD_BASE} +RUN \ + apt-get update && \ + DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + bc \ + bison \ + build-essential \ + flex \ + gcc-riscv64-linux-gnu \ + genext2fs \ + libc6-dev-riscv64-cross \ + rsync \ + && \ + rm -rf /var/lib/apt/lists/* + WORKDIR ${BUILD_BASE} USER developer # copy kernel # ------------------------------------------------------------------------------ -COPY --chown=developer:developer dep/linux-${KERNEL_VERSION}.tar.gz ${BUILD_BASE}/dep/ +COPY --chown=developer:developer linux-${KERNEL_VERSION}.tar.gz ${BUILD_BASE}/dep/ RUN tar xzf ${BUILD_BASE}/dep/linux-${KERNEL_VERSION}.tar.gz \ --strip-components=1 --one-top-level=${BUILD_BASE}/work/linux && \ rm ${BUILD_BASE}/dep/linux-${KERNEL_VERSION}.tar.gz # copy riscv-pk # ------------------------------------------------------------------------------ -COPY --chown=developer:developer dep/riscv-pk-${RISCV_PK_VERSION}.tar.gz ${BUILD_BASE}/dep/ +COPY --chown=developer:developer riscv-pk-${RISCV_PK_VERSION}.tar.gz ${BUILD_BASE}/dep/ RUN tar xzf ${BUILD_BASE}/dep/riscv-pk-${RISCV_PK_VERSION}.tar.gz \ --strip-components=1 --one-top-level=${BUILD_BASE}/work/riscv-pk && \ rm ${BUILD_BASE}/dep/riscv-pk-${RISCV_PK_VERSION}.tar.gz COPY cartesi-linux-config ${BUILD_BASE}/work/linux/.config +# build +# ------------------------------------------------------------------------------ +ARG IMAGE_KERNEL_VERSION=0.0.0 COPY build.mk build.mk -RUN make -f build.mk KERNEL_TIMESTAMP="${KERNEL_TIMESTAMP}" +RUN make -f build.mk KERNEL_TIMESTAMP="${KERNEL_TIMESTAMP}" IMAGE_KERNEL_VERSION="${IMAGE_KERNEL_VERSION}" + +# deb headers +# ------------------------------------------------------------------------------ +COPY tools tools +RUN \ + make -f build.mk KERNEL_TIMESTAMP="${KERNEL_TIMESTAMP}" IMAGE_KERNEL_VERSION="${IMAGE_KERNEL_VERSION}" \ + DESTDIR=${BUILD_BASE}/work/_install cross-deb && \ + rm -rf ${BUILD_BASE}/work/_install +RUN \ + make -f build.mk KERNEL_TIMESTAMP="${KERNEL_TIMESTAMP}" IMAGE_KERNEL_VERSION="${IMAGE_KERNEL_VERSION}" \ + DESTDIR=${BUILD_BASE}/work/_install native-deb && \ + rm -rf ${BUILD_BASE}/work/_install USER root diff --git a/Makefile b/Makefile index debb8f3..0c19f3c 100644 --- a/Makefile +++ b/Makefile @@ -11,16 +11,20 @@ # the License. # -.PHONY: all build push run pull share copy clean clean-config checksum +.PHONY: all build download push run pull share copy clean clean-config checksum + +MAJOR := 0 +MINOR := 16 +PATCH := 0 +LABEL := +IMAGE_KERNEL_VERSION?= $(MAJOR).$(MINOR).$(PATCH)$(LABEL) TAG ?= devel -TOOLCHAIN_DOCKER_REPOSITORY ?= cartesi/toolchain -TOOLCHAIN_TAG ?= 0.13.0 KERNEL_TIMESTAMP ?= $(shell date -Rud @$(shell git log -1 --format=%ct 2> /dev/null || date +%s)) KERNEL_VERSION ?= 5.15.63-ctsi-2 -KERNEL_SRCPATH := dep/linux-${KERNEL_VERSION}.tar.gz +KERNEL_SRCPATH := linux-$(KERNEL_VERSION).tar.gz RISCV_PK_VERSION ?= 1.0.0-ctsi-1 -RISCV_PK_SRCPATH := dep/riscv-pk-${RISCV_PK_VERSION}.tar.gz +RISCV_PK_SRCPATH := riscv-pk-$(RISCV_PK_VERSION).tar.gz KERNEL_CONFIG ?= configs/default-linux-config CONTAINER_BASE := /opt/cartesi/kernel @@ -28,20 +32,10 @@ CONTAINER_BASE := /opt/cartesi/kernel IMG ?= cartesi/linux-kernel:$(TAG) BASE:=/opt/riscv -HEADERS := linux-headers-$(KERNEL_VERSION).tar.xz -IMAGE := linux-nobbl-$(KERNEL_VERSION).bin -LINUX := linux-$(KERNEL_VERSION).bin -LINUX_ELF:= linux-$(KERNEL_VERSION).elf -SELFTEST := linux-selftest-$(KERNEL_VERSION).ext2 - BUILD_ARGS := -ifneq ($(TOOLCHAIN_DOCKER_REPOSITORY),) -BUILD_ARGS += --build-arg TOOLCHAIN_REPOSITORY=$(TOOLCHAIN_DOCKER_REPOSITORY) -endif - -ifneq ($(TOOLCHAIN_TAG),) -BUILD_ARGS += --build-arg TOOLCHAIN_VERSION=$(TOOLCHAIN_TAG) +ifneq ($(IMAGE_KERNEL_VERSION),) +BUILD_ARGS += --build-arg IMAGE_KERNEL_VERSION=$(IMAGE_KERNEL_VERSION) endif ifneq ($(KERNEL_VERSION),) @@ -59,7 +53,7 @@ endif .NOTPARALLEL: all all: build copy -build: cartesi-linux-config checksum +build: cartesi-linux-config download docker build -t $(IMG) $(BUILD_ARGS) . push: @@ -68,55 +62,43 @@ push: pull: docker pull $(IMG) -run: - docker run --hostname toolchain-env -it --rm \ - -e USER=$$(id -u -n) \ - -e GROUP=$$(id -g -n) \ - -e UID=$$(id -u) \ - -e GID=$$(id -g) \ - -v `pwd`:$(CONTAINER_BASE) \ - -w $(CONTAINER_BASE) \ - $(IMG) $(CONTAINER_COMMAND) - -run-as-root: - docker run --hostname toolchain-env -it --rm \ - -v `pwd`:$(CONTAINER_BASE) \ - $(IMG) $(CONTAINER_COMMAND) - config: CONTAINER_COMMAND := $(CONTAINER_BASE)/scripts/update-linux-config config: cartesi-linux-config run-as-root +env: + @echo KERNEL_VERSION="$(KERNEL_VERSION)" + @echo IMAGE_KERNEL_VERSION="$(IMAGE_KERNEL_VERSION)" + @echo RISCV_PK_VERSION="$(RISCV_PK_VERSION)" + @echo TOOLCHAIN_VERSION="$(TOOLCHAIN_TAG)" + @make -srf build.mk KERNEL_VERSION=$(KERNEL_VERSION) IMAGE_KERNEL_VERSION=$(IMAGE_KERNEL_VERSION) env copy: ID=`docker create $(IMG)` && \ - docker cp $$ID:$(BASE)/kernel/artifacts/$(HEADERS) . && \ - docker cp $$ID:$(BASE)/kernel/artifacts/$(IMAGE) . && \ - docker cp $$ID:$(BASE)/kernel/artifacts/$(LINUX) . && \ - docker cp $$ID:$(BASE)/kernel/artifacts/$(LINUX_ELF) . && \ - docker cp $$ID:$(BASE)/kernel/artifacts/$(SELFTEST) . && \ + docker cp $$ID:$(BASE)/kernel/artifacts/ . && \ docker rm -v $$ID cartesi-linux-config: cp $(KERNEL_CONFIG) ./cartesi-linux-config +$(KERNEL_SRCPATH): + wget -O $@ https://github.com/cartesi/linux/archive/v$(KERNEL_VERSION).tar.gz + +$(RISCV_PK_SRCPATH): + wget -O $@ https://github.com/cartesi/riscv-pk/archive/v$(RISCV_PK_VERSION).tar.gz + +checksum: $(KERNEL_SRCPATH) $(RISCV_PK_SRCPATH) + shasum -ca 256 shasumfile + +shasumfile: $(KERNEL_SRCPATH) $(RISCV_PK_SRCPATH) + @shasum -a 256 $^ > $@ + +download: checksum + clean-config: rm -f ./cartesi-linux-config clean: clean-config - rm -f $(HEADERS) $(IMAGE) $(LINUX) $(SELFTEST) + rm -rf artifacts/ depclean: clean rm -f \ $(KERNEL_SRCPATH) $(RISCV_PK_SRCPATH) - -checksum: $(KERNEL_SRCPATH) $(RISCV_PK_SRCPATH) - shasum -ca 256 shasumfile - -dep: - mkdir dep -$(KERNEL_SRCPATH): URL=https://github.com/cartesi/linux/archive/v${KERNEL_VERSION}.tar.gz -$(KERNEL_SRCPATH): | dep - T=`mktemp` && wget "$(URL)" -O $$T && mv $$T $@ || rm $$T - -$(RISCV_PK_SRCPATH): URL=https://github.com/cartesi/riscv-pk/archive/v${RISCV_PK_VERSION}.tar.gz -$(RISCV_PK_SRCPATH): | dep - T=`mktemp` && wget "$(URL)" -O $$T && mv $$T $@ || rm $$T diff --git a/build.mk b/build.mk index a980b69..223c665 100644 --- a/build.mk +++ b/build.mk @@ -1,4 +1,4 @@ -TOOLCHAIN_PREFIX := riscv64-cartesi-linux-gnu +TOOLCHAIN_PREFIX := riscv64-linux-gnu RISCV_PK_DIR := work/riscv-pk RISCV_PK_BUILD_DIR := $(RISCV_PK_DIR)/build @@ -10,16 +10,32 @@ JOBS := -j$(shell nproc) KERNEL_VERSION ?= $(shell make -sC $(LINUX_DIR) kernelversion) KERNEL_TIMESTAMP ?= $(shell date -Ru) -HEADERS := artifacts/linux-headers-$(KERNEL_VERSION).tar.xz -IMAGE := artifacts/linux-nobbl-$(KERNEL_VERSION).bin -LINUX := artifacts/linux-$(KERNEL_VERSION).bin -LINUX_ELF := artifacts/linux-$(KERNEL_VERSION).elf -SELFTEST := artifacts/linux-selftest-$(KERNEL_VERSION).ext2 +IMAGE_KERNEL_VERSION?= 0.0.0 +HEADERS := artifacts/linux-headers-$(KERNEL_VERSION)-v$(IMAGE_KERNEL_VERSION).tar.xz +IMAGE := artifacts/linux-nobbl-$(KERNEL_VERSION)-v$(IMAGE_KERNEL_VERSION).bin +LINUX := artifacts/linux-$(KERNEL_VERSION)-v$(IMAGE_KERNEL_VERSION).bin +LINUX_ELF := artifacts/linux-$(KERNEL_VERSION)-v$(IMAGE_KERNEL_VERSION).elf +SELFTEST := artifacts/linux-selftest-$(KERNEL_VERSION)-v$(IMAGE_KERNEL_VERSION).ext2 +CROSS_DEB_FILENAME := artifacts/linux-libc-dev-riscv64-cross-$(KERNEL_VERSION)-v$(IMAGE_KERNEL_VERSION).deb +NATIVE_DEB_FILENAME := artifacts/linux-libc-dev-$(KERNEL_VERSION)-v$(IMAGE_KERNEL_VERSION).deb ARTIFACTS := $(HEADERS) $(IMAGE) $(LINUX) $(SELFTEST) all: $(ARTIFACTS) +env: + @echo KBUILD_BUILD_TIMESTAMP=\""$(KERNEL_TIMESTAMP)"\" + @echo KBUILD_BUILD_USER=dapp + @echo KBUILD_BUILD_HOST=cartesi + + @echo HEADERS="$(HEADERS)" + @echo IMAGE="$(IMAGE)" + @echo LINUX="$(LINUX)" + @echo LINUX_ELF="$(LINUX_ELF)" + @echo SELFTEST="$(SELFTEST)" + @echo CROSS_DEB_FILENAME="$(CROSS_DEB_FILENAME)" + @echo NATIVE_DEB_FILENAME="$(NATIVE_DEB_FILENAME)" + # build linux # ------------------------------------------------------------------------------ LINUX_OPTS=$(JOBS) ARCH=riscv CROSS_COMPILE=$(TOOLCHAIN_PREFIX)- KBUILD_BUILD_TIMESTAMP="$(KERNEL_TIMESTAMP)" KBUILD_BUILD_USER=dapp KBUILD_BUILD_HOST=cartesi @@ -33,12 +49,21 @@ $(LINUX_DIR)/vmlinux $(IMAGE) $(HEADERS) &: $(LINUX_DIR)/.config cp work/linux/arch/riscv/boot/Image $(IMAGE) cp $(LINUX_DIR)/vmlinux $(LINUX_ELF) -env: - @echo export ARCH=riscv - @echo export CROSS_COMPILE=$(TOOLCHAIN_PREFIX)- - @echo export KBUILD_BUILD_TIMESTAMP="$(KERNEL_TIMESTAMP)" - @echo export KBUILD_BUILD_USER=dapp - @echo export KBUILD_BUILD_HOST=cartesi +cross-deb: # TARGET == riscv64 + mkdir -p $(DESTDIR)/DEBIAN + cat tools/template/cross-control.template | sed 's|ARG_KERNEL_VERSION|$(KERNEL_VERSION)|g' > $(DESTDIR)/DEBIAN/control + $(MAKE) -rC $(LINUX_DIR) $(LINUX_OPTS) headers_install \ + INSTALL_HDR_PATH=$(abspath $(DESTDIR))/usr/riscv64-linux-gnu + find $(DESTDIR) -exec touch -d "$(KERNEL_TIMESTAMP)" {} \; + SOURCE_DATE_EPOCH="1" dpkg-deb -Zxz --root-owner-group --build $(DESTDIR) $(CROSS_DEB_FILENAME) + +native-deb: # HOST == riscv64 + mkdir -p $(DESTDIR)/DEBIAN + cat tools/template/native-control.template | sed 's|ARG_KERNEL_VERSION|$(KERNEL_VERSION)|g' > $(DESTDIR)/DEBIAN/control + $(MAKE) -rC $(LINUX_DIR) $(LINUX_OPTS) headers_install \ + INSTALL_HDR_PATH=$(abspath $(DESTDIR))/usr + find $(DESTDIR) -exec touch -d "$(KERNEL_TIMESTAMP)" {} \; + SOURCE_DATE_EPOCH="1" dpkg-deb -Zxz --root-owner-group --build $(DESTDIR) $(NATIVE_DEB_FILENAME) # configure riscv-pk # ------------------------------------------------------------------------------ @@ -85,14 +110,14 @@ run-selftest: # clone (for non CI environment) # ------------------------------------------------------------------------------ -clone: LINUX_BRANCH ?= linux-5.5.19-ctsi-y +clone: LINUX_BRANCH ?= linux-5.15.63-ctsi-y clone: RISCV_PK_BRANCH ?= v1.0.0-ctsi-1 clone: git clone --depth 1 --branch $(LINUX_BRANCH) \ - git@github.com:cartesi-corp/linux.git $(LINUX_DIR) || \ + git@github.com:cartesi/linux.git $(LINUX_DIR) || \ cd $(LINUX_DIR) && git pull git clone --depth 1 --branch $(RISCV_PK_BRANCH) \ - git@github.com:cartesi-corp/riscv-pk.git $(RISCV_PK_DIR) || \ + git@github.com:cartesi/riscv-pk.git $(RISCV_PK_DIR) || \ cd $(RISCV_PK_DIR) && git pull run: IMG=cartesi/toolchain:devel diff --git a/shasumfile b/shasumfile index f2b67b6..d7d37cc 100644 --- a/shasumfile +++ b/shasumfile @@ -1,2 +1,2 @@ -e8d4d1882632eac7fd8e433b4eb4db014fd56e645fdad3b89be7d1ca4f20ca07 dep/linux-5.15.63-ctsi-2.tar.gz -9a873345b9914940e7bf03a167da823910c8a2acadd818b780ffbd1a3edcc4c5 dep/riscv-pk-1.0.0-ctsi-1.tar.gz +e8d4d1882632eac7fd8e433b4eb4db014fd56e645fdad3b89be7d1ca4f20ca07 linux-5.15.63-ctsi-2.tar.gz +9a873345b9914940e7bf03a167da823910c8a2acadd818b780ffbd1a3edcc4c5 riscv-pk-1.0.0-ctsi-1.tar.gz diff --git a/tools/template/cross-control.template b/tools/template/cross-control.template new file mode 100644 index 0000000..9ad6172 --- /dev/null +++ b/tools/template/cross-control.template @@ -0,0 +1,8 @@ +Package: linux-libc-dev-riscv64-cross +Version: ARG_KERNEL_VERSION +Architecture: any +Priority: optional +Section: devel +Maintainer: Machine Reference Unit +Provides: linux-kernel-headers-riscv64-cross, linux-libc-dev-riscv64-dcv1 +Description: Linux Kernel Headers for development (for cross-compiling) diff --git a/tools/template/native-control.template b/tools/template/native-control.template new file mode 100644 index 0000000..db70869 --- /dev/null +++ b/tools/template/native-control.template @@ -0,0 +1,8 @@ +Package: linux-libc-dev +Version: ARG_KERNEL_VERSION +Architecture: riscv64 +Priority: optional +Section: devel +Maintainer: Machine Reference Unit +Provides: linux-kernel-headers +Description: Linux Kernel Headers for development