GitHub Actions build refactor

.github/workflows/build_and_publish_template.yml

@@ -0,0 +1,88 @@
+# This workflow builds ArrayRecord wheels and uploads them as artifacts.
+
+name: Build & Publish Template
+
+on:
+  workflow_call:
+    inputs:
+      pypi_project_url:
+        required: false
+        type: string
+      upload_wheels:
+        required: true
+        type: boolean
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-test:
+    name: "Python ${{ matrix.python-version }} on ${{ matrix.os }}"
+    runs-on: "${{ matrix.os }}"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        os: [ubuntu-22.04, ubuntu-22.04-arm, macos-14]
+
+    env:
+      USE_BAZEL_VERSION: "7.2.1"
+    steps:
+      - name: Set up Bazel
+        uses: bazel-contrib/setup-bazel@0.15.0
+      - name: Check Bazel installation
+        run: |
+          which bazel
+          bazel version
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: "actions/checkout@v3"
+      - name: Create directory
+        run: |
+          mkdir -p /tmp/array_record
+          cp -r . /tmp/array_record
+      - name: Build package
+        run: |
+          set -xe
+          export PYTHON_VERSION=${{ matrix.python-version }}
+          export PYTHON_MAJOR_VERSION=$(echo $PYTHON_VERSION | cut -d. -f1)
+          export PYTHON_MINOR_VERSION=$(echo $PYTHON_VERSION | cut -d. -f2)
+          export BAZEL_VERSION="7.2.1"
+          export OUTPUT_DIR="/tmp/array_record"
+          export SOURCE_DIR="/tmp/array_record"
+          . "${SOURCE_DIR}"'/oss/runner_common.sh'
+          build_and_test_array_record
+      - name: Upload ArrayRecord artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: built-array-record-wheels-${{ matrix.os }}-${{ matrix.python-version }}
+          path: /tmp/array_record/all_dist/*.whl
+
+  publish-wheel:
+    if: ${{ inputs.upload_wheels }}
+    runs-on: ubuntu-22.04
+    needs: build-and-test
+    permissions:
+      id-token: write
+    environment:
+      name: pypi
+      url: ${{ inputs.pypi_project_url }}
+    steps:
+      - name: Download ArrayRecord artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: built-array-record-wheels-*
+          path: dist/
+          merge-multiple: true
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          attestations: false
+          verbose: true

.github/workflows/publish_release.yml

@@ -0,0 +1,13 @@
+name: Build and Publish Release
+
+on: workflow_dispatch
+
+jobs:
+  call-workflow:
+    uses: ./.github/workflows/build_and_publish_template.yml
+    permissions:
+      contents: read
+      id-token: write
+    with:
+      pypi_project_url: https://pypi.org/project/array-record
+      upload_wheels: true

.github/workflows/python-tests.yml

@@ -5,37 +5,7 @@ on:
     branches: [main]
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['3.9', '3.10', '3.11']
-    env:
-      DOCKER_BUILDKIT: 1
-      TMP_FOLDER: /tmp/array_record
-    steps:
-      - uses: actions/checkout@v2
-      - name: Build Docker image
-        run: |
-          docker build --progress=plain --no-cache \
-          --build-arg PYTHON_VERSION=${{ matrix.python-version }} \
-          -t array_record:latest - < oss/build.Dockerfile
-      - name: Build wheels and test
-        run: |
-          docker run --rm -a stdin -a stdout -a stderr \
-            --env PYTHON_VERSION=${{ matrix.python-version }} \
-            --volume ${GITHUB_WORKSPACE}:${TMP_FOLDER} --name array_record array_record:latest \
-            bash oss/build_whl.sh
-      - name: Install in a blank Docker and test the import in Python
-        run: |
-          docker run --rm -a stdin -a stdout -a stderr \
-            --env PYTHON_VERSION=${{ matrix.python-version }} \
-            --volume ${GITHUB_WORKSPACE}:/root \
-            python:${{ matrix.python-version }} bash -c "
-              ARRAY_RECORD_VERSION=\$(python /root/setup.py --version 2>&1 /dev/null)
-              SHORT_PYTHON_VERSION=\${PYTHON_VERSION//./}
-              ARRAY_RECORD_WHEEL=\"/root/all_dist/array_record-\${ARRAY_RECORD_VERSION}-py\${SHORT_PYTHON_VERSION}-none-any.whl\"
-              python -m pip install \${ARRAY_RECORD_WHEEL} &&
-              python -c 'import array_record' &&
-              python -c 'from array_record.python import array_record_data_source'
-            "
+  call-workflow:
+    uses: ./.github/workflows/build_and_publish_template.yml
+    with:
+      upload_wheels: false

.gitignore

@@ -0,0 +1,12 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Bazel outputs
+bazel-array_record
+bazel-bin
+bazel-out
+bazel-testlogs
+
+MODULE.bazel.lock

BUILD

@@ -1,8 +1,10 @@
 # ArrayRecord is a new file format for IO intensive applications.
 # It supports efficient random access and various compression algorithms.
 
-load("@rules_python//python:pip.bzl", "compile_pip_requirements")
-
+load("@python//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements")
+load("@python//3.11:defs.bzl", compile_pip_requirements_3_11 = "compile_pip_requirements")
+load("@python//3.12:defs.bzl", compile_pip_requirements_3_12 = "compile_pip_requirements")
+load("@python//3.13:defs.bzl", compile_pip_requirements_3_13 = "compile_pip_requirements")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -15,8 +17,26 @@ py_library(
     srcs = ["setup.py"],
 )
 
-compile_pip_requirements(
-    name = "requirements",
-    requirements_in = "requirements.in",
-    requirements_txt = "requirements_lock.txt",
+compile_pip_requirements_3_10(
+    name = "requirements_3_10",
+    requirements_in = "test_requirements.in",
+    requirements_txt = "test_requirements_lock_3_10.txt",
+)
+
+compile_pip_requirements_3_11(
+    name = "requirements_3_11",
+    requirements_in = "test_requirements.in",
+    requirements_txt = "test_requirements_lock_3_11.txt",
+)
+
+compile_pip_requirements_3_12(
+    name = "requirements_3_12",
+    requirements_in = "test_requirements.in",
+    requirements_txt = "test_requirements_lock_3_12.txt",
+)
+
+compile_pip_requirements_3_13(
+    name = "requirements_3_13",
+    requirements_in = "test_requirements.in",
+    requirements_txt = "test_requirements_lock_3_13.txt",
 )

MODULE.bazel

@@ -12,41 +12,59 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO(fchern): automate version string alignment with setup.py
-VERSION = "0.6.0"
-
 module(
     name = "array_record",
-    version = VERSION,
+    version = "0.7.3",
     repo_name = "com_google_array_record",
 )
 
 bazel_dep(name = "rules_proto", version = "7.0.2")
-bazel_dep(name = "rules_python", version = "0.40.0")
+bazel_dep(name = "rules_python", version = "0.37.0")
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "protobuf", version = "24.4")  # aligns with pygrain
+bazel_dep(name = "protobuf", version = "28.3")
 bazel_dep(name = "googletest", version = "1.15.2")
 bazel_dep(name = "abseil-cpp", version = "20240722.0")
 bazel_dep(name = "abseil-py", version = "2.1.0")
 bazel_dep(name = "eigen", version = "3.4.0.bcr.3")
 bazel_dep(name = "riegeli", version = "0.0.0-20241218-3385e3c")
 bazel_dep(name = "pybind11_bazel", version = "2.12.0")
 
-PYTHON_VERSION = "3.10"
+SUPPORTED_PYTHON_VERSIONS = [
+    "3.10",
+    "3.11",
+    "3.12",
+    "3.13",
+]
+
+DEFAULT_PYTHON_VERSION = "3.10"
+
+python_configure = use_extension("@pybind11_bazel//:python_configure.bzl", "extension")
+use_repo(python_configure, "local_config_python")
 
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
-python.toolchain(
-    ignore_root_user_error = True,  # Required for our containerized CI environments.
-    python_version = PYTHON_VERSION,
-)
+
+[
+    python.toolchain(
+        ignore_root_user_error = True,
+        is_default = python_version == DEFAULT_PYTHON_VERSION,
+        python_version = python_version,
+    )
+    for python_version in SUPPORTED_PYTHON_VERSIONS
+]
+
+use_repo(python, python = "python_versions")
 
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 
 # requirements_lock.txt is generated by
 # bazel run //:requirements.update
-pip.parse(
-    hub_name = "pypi",
-    python_version = PYTHON_VERSION,
-    requirements_lock = "//:requirements_lock.txt",
-)
+[
+    pip.parse(
+        hub_name = "pypi",
+        python_version = version,
+        requirements_lock = "test_requirements_lock_" + version.replace(".", "_") + ".txt",
+    )
+    for version in SUPPORTED_PYTHON_VERSIONS
+]
+
 use_repo(pip, "pypi")

cpp/array_record_reader.cc

@@ -80,7 +80,7 @@ class ChunkOffset {
  public:
   virtual ~ChunkOffset() {}
   virtual uint64_t operator[](size_t idx) const = 0;
-  virtual size_t size() const = 0;
+  virtual uint64_t size() const = 0;
   bool empty() const { return size() == 0; }
 };
 

oss/build.Dockerfile → oss/Dockerfile

@@ -6,13 +6,15 @@ ARG AUDITWHEEL_PLATFORM
 FROM quay.io/pypa/${AUDITWHEEL_PLATFORM}
 
 ARG PYTHON_VERSION
-ARG PYTHON_BIN
 ARG BAZEL_VERSION
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN ulimit -n 1024 && yum install -y rsync
-ENV PATH="${PYTHON_BIN}:${PATH}"
+ENV PYTHON_BIN_PATH=/opt/python/cp${PYTHON_VERSION}-cp${PYTHON_VERSION}/bin
+ENV PATH="${PYTHON_BIN_PATH}:${PATH}"
+
+ENV PYTHON_BIN=${PYTHON_BIN_PATH}/python
 
 # Download the correct bazel version and make sure it's on path.
 RUN BAZEL_ARCH_SUFFIX="$(uname -m | sed s/aarch64/arm64/)" \
@@ -21,7 +23,7 @@ RUN BAZEL_ARCH_SUFFIX="$(uname -m | sed s/aarch64/arm64/)" \
 
 # Install dependencies needed for array_record.
 RUN --mount=type=cache,target=/root/.cache \
-  ${PYTHON_BIN}/python -m pip install -U \
+  $PYTHON_BIN -m pip install -U \
     absl-py \
     auditwheel \
     etils[epath] \
@@ -30,4 +32,4 @@ RUN --mount=type=cache,target=/root/.cache \
     twine \
     wheel;
 
-WORKDIR "/tmp/array_record"
+WORKDIR "/tmp/array_record"

oss/README.md

@@ -1,15 +1,26 @@
-# Steps to build a new array_record pip package
+# Steps to build and publish a new `array_record` PyPI package
 
-1. Update the version number in setup.py
+`array_record` supports automatic publishing to PyPI via GitHub Actions.
+Once you're ready to create a new release you need to:
 
-2. In the root folder, run
+1. Update the version number in `setup.py`.
 
-  ```
-  ./oss/build_whl.sh
-  ```
-  to use the current `python3` version.  Otherwise, optionally set
-  ```
-  PYTHON_VERSION=3.9 ./oss/build_whl.sh
-  ```
+2. Go to [GitHub Actions page](https://github.com/google/array_record/actions),
+   select `Build and Publish Release` workflow, and run it. It will spin up a few
+   test jobs, and once all of them complete successfully, a `publish-wheel` will start.
 
-3. Wheels are in `all_dist/`.
+3. On completion you should notice a new release on https://pypi.org/project/array-record/#history.
+
+---
+
+If you want to build a wheel locally in your development environment in the root folder, run:
+
+```sh
+./oss/build_whl.sh
+```
+to use the current `python3` version. Otherwise, optionally set:
+```sh
+PYTHON_VERSION=3.9 ./oss/build_whl.sh
+```
+
+Wheels are in `all_dist/`.