From e1f7ecc0579be6f6b764092364a402374f80418d Mon Sep 17 00:00:00 2001 From: Marzieh Lenjani Date: Wed, 1 Apr 2026 09:44:52 -0700 Subject: [PATCH] DjangoBench: Add Ubuntu 24.04 aarch64 support and fix proxygen_binding runtime errors (#550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: DjangoBench fails to install and run on Ubuntu 24.04 aarch64 (Phoenix CRB). This diff fixes three issues: 1. Install fails — no Python 3.10 on Ubuntu 24.04 The install dispatcher routes all Ubuntu aarch64 installs to the Ubuntu 22.04 script, which expects python3.10 as a system package. Ubuntu 24.04 ships Python 3.12 and has no python3.10 package. Fix: Added install_django_workload_aarch64_ubuntu24.sh which builds Python 3.10 from source, skips Cinder (ARM jobs use interpreter=cpython with use_jit=0), and handles Ubuntu 24.04 package differences. Updated the dispatcher to detect Ubuntu major version ≥24 and route accordingly. 2. Runtime crash — shared libraries not found After install, run.sh fails because LD_LIBRARY_PATH doesn't include the proxygen and boost library directories. The proxygen_binding.so can't locate its dynamic dependencies (libfolly, libboost, etc.). Fix: Added proxygen/staging/lib, proxygen/proxygen/_build/deps/lib, and proxygen/proxygen/_build/deps/lib64 to LD_LIBRARY_PATH in run.sh. 3. Runtime crash — undefined symbol boost::match_results::maybe_assign proxygen_binding.cpython-310-aarch64-linux-gnu.so fails to import with an undefined boost::regex symbol. The boost_regex library is built by build_proxygen.sh (via --with-libraries=...regex...) but was never linked into the extension. Reviewed By: excelle08 Differential Revision: D97857317 --- .../install_django_workload.sh | 8 +- ...nstall_django_workload_aarch64_ubuntu24.sh | 547 ++++++++++++++++++ packages/django_workload/srcs/bin/run.sh | 5 +- 3 files changed, 558 insertions(+), 2 deletions(-) create mode 100755 packages/django_workload/install_django_workload_aarch64_ubuntu24.sh diff --git a/packages/django_workload/install_django_workload.sh b/packages/django_workload/install_django_workload.sh index e5e076f1..85be0f8e 100755 --- a/packages/django_workload/install_django_workload.sh +++ b/packages/django_workload/install_django_workload.sh @@ -15,7 +15,13 @@ DJANGO_REPO_ROOT="${DJANGO_WORKLOAD_ROOT}/django-workload" source "${BENCHPRESS_ROOT}/packages/common/os-distro.sh" if distro_is_like ubuntu && [ "$(uname -m)" = "aarch64" ]; then - "${DJANGO_PKG_ROOT}"/install_django_workload_aarch64_ubuntu22.sh + UBUNTU_VERSION="$(awk -F "=" '/^VERSION_ID=/ {print $2}' /etc/os-release | tr -d '"')" + UBUNTU_MAJOR="${UBUNTU_VERSION%%.*}" + if [ "${UBUNTU_MAJOR}" -ge 24 ] 2>/dev/null; then + "${DJANGO_PKG_ROOT}"/install_django_workload_aarch64_ubuntu24.sh + else + "${DJANGO_PKG_ROOT}"/install_django_workload_aarch64_ubuntu22.sh + fi exit $? fi if distro_is_like ubuntu && [ "$(uname -m)" = "x86_64" ]; then diff --git a/packages/django_workload/install_django_workload_aarch64_ubuntu24.sh b/packages/django_workload/install_django_workload_aarch64_ubuntu24.sh new file mode 100755 index 00000000..54e3db91 --- /dev/null +++ b/packages/django_workload/install_django_workload_aarch64_ubuntu24.sh @@ -0,0 +1,547 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# Django workload install script for Ubuntu 24.04 on aarch64. +# +# Key differences from the Ubuntu 22.04 script: +# - Builds Python 3.10 from source (Ubuntu 24.04 ships Python 3.12) +# - Skips Cinder build (ARM jobs use interpreter=cpython, use_jit=0) +# - Uses source-built Python for venv creation and pip installs +set -Eeuo pipefail + +DJANGO_PKG_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) +BENCHPRESS_ROOT="$(readlink -f "${DJANGO_PKG_ROOT}/../..")" +TEMPLATES_DIR="${DJANGO_PKG_ROOT}/templates" +BENCHMARKS_ROOT="${BENCHPRESS_ROOT}/benchmarks" +DJANGO_WORKLOAD_ROOT="${BENCHMARKS_ROOT}/django_workload" +DJANGO_REPO_ROOT="${DJANGO_WORKLOAD_ROOT}/django-workload" +DJANGO_SERVER_ROOT="${DJANGO_REPO_ROOT}/django-workload" +DJANGO_WORKLOAD_DEPS="${DJANGO_SERVER_ROOT}/third_party" + +# Source the build parallelism utility and calculate optimal job count +# This considers both CPU cores and memory limits (including cgroup constraints) +# shellcheck source=../common/get_build_parallelism.sh +source "${BENCHPRESS_ROOT}/packages/common/get_build_parallelism.sh" + +# Number of parallel build jobs - use environment variable if set, otherwise calculate optimal value +if [ -n "${NUM_BUILD_JOBS:-}" ]; then + echo "Using user-specified NUM_BUILD_JOBS=${NUM_BUILD_JOBS}" +else + NUM_BUILD_JOBS=$(get_build_parallelism) + echo "Calculated optimal NUM_BUILD_JOBS=${NUM_BUILD_JOBS} based on CPU and memory constraints" +fi +print_build_parallelism_info + +# ===================================================================== +# Step 1: Install System Dependencies +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 1: Installing System Dependencies" +echo "=====================================================================" + +# Ubuntu 24.04 does not have python3.10 in its default repos, so we +# install build dependencies to compile Python 3.10 from source instead. +apt install -y memcached libmemcached-dev zlib1g-dev screen \ + python3 rpm libffi-dev \ + libssl-dev libcrypt-dev haproxy libxxhash-dev \ + perl ninja-build build-essential libbz2-dev libreadline-dev \ + libsqlite3-dev libncurses-dev liblzma-dev wget + +echo "System dependencies installed successfully" + +# Copy django-workload from srcs directory instead of cloning from GitHub +mkdir -p "${DJANGO_WORKLOAD_ROOT}" +pushd "${DJANGO_WORKLOAD_ROOT}" +if ! [ -d "django-workload" ]; then + mkdir -p "django-workload" + cp -r "${DJANGO_PKG_ROOT}/srcs/django-workload/"* "django-workload/" +else + echo "[SKIPPED] copying django-workload" +fi + +# ===================================================================== +# Step 2: Download pip third-party dependencies for django-workload +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 2: Downloading pip third-party dependencies" +echo "=====================================================================" + +# Download pip third-party dependencies for django-workload +if ! [ -d "${DJANGO_WORKLOAD_DEPS}" ]; then + mkdir -p "${DJANGO_WORKLOAD_DEPS}" +else + shopt -s expand_aliases + alias wget='wget --no-clobber' +fi +pushd "${DJANGO_WORKLOAD_DEPS}" +# cassandra-driver-3.26_aarch64.whl +wget "https://files.pythonhosted.org/packages/b5/5e/54c58c98a4eeea12a2fee7220e7ac9e8b021ea5c3d84c84adb9106c4ed43/cassandra_driver-3.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" +# Removed Cython download as it's not needed +# Django-5.2.3-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" +# Dulwich 0.21.2.tar.gz +wget "https://files.pythonhosted.org/packages/14/a5/cf61f9209d48abf47d48086e0a0388f1030bb5f7cf2661972eee56ccee3d/dulwich-0.21.2.tar.gz" +# django-cassandra-engine-1.6.2.tar.gz +wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# django-statsd-mozilla-0.4.3-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" +# numpy-1.22.1-aarch64.whl +wget "https://files.pythonhosted.org/packages/d6/ec/a8b5f1b6d00bc4fd1bc91043d5dfb029536ec5c7769588d3f4c982240008/numpy-1.22.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" +# psutil-5.8.0.tar.gz +wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" +# pylibmc-1.6.1.tar.gz +wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pytz-2021.1-py2.py3-none-any.whl +wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" +# six-1.16.0-py2.py3-none-any.whl +wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" +# statsd-3.3.0-py2.py3-none-any.whl +wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" +# uwsgi-2.0.22.tar.gz +wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# geomet-0.2.1.post1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" +# click-7.1.2.tar.gz +wget "https://files.pythonhosted.org/packages/27/6f/be940c8b1f1d69daceeb0032fee6c34d7bd70e3e649ccac0951500b4720e/click-7.1.2.tar.gz" +# typing_extensions-4.14.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl" +# asgiref-3.8.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/39/e3/893e8757be2612e6c266d9bb58ad2e3651524b5b40cf56761e985a28b13e/asgiref-3.8.1-py3-none-any.whl" +# sqlparse-0.5.3-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/a9/5c/bfd6bd0bf979426d405cc6e71eceb8701b148b16c21d2dc3c261efc61c7b/sqlparse-0.5.3-py3-none-any.whl" +# rapidfuzz-2.10.2.tar.gz +wget "https://files.pythonhosted.org/packages/ee/92/0c0366b108f658dd29fdf7e9ae73874e9b0c36a9d7c72e7690d075132a3d/rapidfuzz-2.10.2.tar.gz" +# scikit-learn-0.15.0.tar.gz +wget "https://files.pythonhosted.org/packages/a2/f4/ea25fe640fadca8a8d860a397f77c427737fbdbc3edb04e8070680f850a0/scikit-learn-0.15.0.tar.gz" +# filelock-3.12.4-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/5e/5d/97afbafd9d584ff1b45fcb354a479a3609bd97f912f8f1f6c563cb1fae21/filelock-3.12.4-py3-none-any.whl" +# msgpack-0.5.2.tar.gz +wget "https://files.pythonhosted.org/packages/17/99/1929902c6d0bffce866be5ceadfe6f395041813ad8004a24eb3f82231564/msgpack-0.5.2.tar.gz" +# wheel-0.41.2-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl" +# setuptools-67.0.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/bf/27/969c914650fdf0d08b0b92bdbddfc08bea9df6d86aeefd75ba4730f50bc9/setuptools-67.0.0-py3-none-any.whl" +# platformdirs-3.11.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/56/29/3ec311dc18804409ecf0d2b09caa976f3ae6215559306b5b530004e11156/platformdirs-3.11.0-py3-none-any.whl" +# pkginfo-1.9.6-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/b3/f2/6e95c86a23a30fa205ea6303a524b20cbae27fbee69216377e3d95266406/pkginfo-1.9.6-py3-none-any.whl" +# jsonschema-4.17.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/9f/df/824fdaa0d7228fa2e8a5171a408dbabe2c66955afd5be5211725389640b5/jsonschema-4.17.1-py3-none-any.whl" +# keyring-24.2.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/0e/8f/5772801169cf62e8232721034f91f81e33b0cfa6e51d3bf6ff65c503af2a/keyring-24.2.0-py3-none-any.whl" +# tomlkit-0.12.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/a0/6d/808775ed618e51edaa7bbe6759e22e1c7eafe359af6e084700c6d39d3455/tomlkit-0.12.1-py3-none-any.whl" +# cachecontrol-0.13.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl" +# installer-0.7.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl" +# poetry-1.6.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/7d/25/f3bfda3c458d114005af99441d009936b85a34a730aeb9cf57fb2630d9f7/poetry-1.6.1-py3-none-any.whl" +# poetry_plugin_export-1.5.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/e9/12/43553a79e1d3bf8de119125dfc3e1fcc8f4258d658b603908d02efaed256/poetry_plugin_export-1.5.0-py3-none-any.whl" +# poetry_core-1.7.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/bf/d4/ce72ac247f414d15ff046f0926b76eb42bd743e83c1df28e856f328e3db1/poetry_core-1.7.0-py3-none-any.whl" +# requests_toolbelt-1.0.0-py2.py3-none-any.whl +wget "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl" +# tomli-2.0.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl" +# cleo-2.0.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/b1/ae/0329af2a4c22836010c43760233a181a314853a97e0f2b53b02825c4c9b7/cleo-2.0.1-py3-none-any.whl" +# requests-2.31.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl" +# hooks-1.0.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl" +# shellingham-1.5.4-py2.py3-none-any.whl +wget "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl" +# pexpect-4.8.0-py2.py3-none-any.whl +wget "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl" +# virtualenv-20.24.6-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/7f/19/1f0eddcb9acf00a95793ce83417f69e0fd106c192121360af499cd6fde39/virtualenv-20.24.6-py3-none-any.whl" +# packaging-23.2-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/ec/1a/610693ac4ee14fcdf2d9bf3c493370e4f2ef7ae2e19217d7a237ff42367d/packaging-23.2-py3-none-any.whl" +# trove_classifiers-2023.10.18-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/ec/40/05cb2725ca7e6c844c66af626c5749efd254ec4506f17a1d01ba79ae9da6/trove_classifiers-2023.10.18-py3-none-any.whl" +# build-0.10.0-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl" +# crashtest-0.4.1-py3-none-any.whl +wget "https://files.pythonhosted.org/packages/b0/5c/3ba7d12e7a79566f97b8f954400926d7b6eb33bcdccc1315a857f200f1f1/crashtest-0.4.1-py3-none-any.whl" +# parser_libraries-3.7.tar.gz +wget "https://files.pythonhosted.org/packages/11/35/575091de594677e40440a24be3192c78116b69c1180a77be63d71353b9a8/parser_libraries-3.7.tar.gz" +popd +unalias wget 2>/dev/null || echo "[Finished] downloading dependencies" + +# Copy bin directory from srcs +mkdir -p "${DJANGO_WORKLOAD_ROOT}/bin" +cp -r "${DJANGO_PKG_ROOT}/srcs/bin/"* "${DJANGO_WORKLOAD_ROOT}/bin/" +chmod +x "${DJANGO_WORKLOAD_ROOT}/bin/"*.sh + +echo "Pip dependencies downloaded successfully" + +# ===================================================================== +# Step 3: Install JDK and Cassandra +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 3: Installing JDK and Cassandra" +echo "=====================================================================" + +# Install JDK +JDK_NAME=openjdk-11-jdk +apt install -y "${JDK_NAME}" || { echo "Could not install ${JDK_NAME} package"; exit 1;} +echo "JDK installed successfully" + +# Install Cassandra +# Download Cassandra from third-party source +cassandra_version=3.11.19 +CASSANDRA_NAME="apache-cassandra-${cassandra_version}" +if ! [ -d "${CASSANDRA_NAME}" ]; then + CASSANDRA_TAR="${CASSANDRA_NAME}-bin.tar.gz" + if ! [ -f "${CASSANDRA_TAR}" ]; then + wget "https://dlcdn.apache.org/cassandra/${cassandra_version}/${CASSANDRA_TAR}" + fi + tar -xvf "${CASSANDRA_TAR}" -C "${DJANGO_WORKLOAD_ROOT}" +else + echo "[SKIPPED] downloading ${CASSANDRA_NAME}" +fi +# Rename +CASSANDRA_ROOT="${DJANGO_WORKLOAD_ROOT}/apache-cassandra" +[ ! -d "${CASSANDRA_ROOT}" ] && mv "${DJANGO_WORKLOAD_ROOT}/${CASSANDRA_NAME}" "${CASSANDRA_ROOT}" +pushd "${CASSANDRA_ROOT}" + +# Set JVM Options +if [ -f "conf/jvm.options" ]; then + mv conf/jvm.options conf/jvm.options.bkp || exit 1 +fi +cp "${TEMPLATES_DIR}/jvm.options" "${CASSANDRA_ROOT}/conf/jvm.options" || exit 1 + +# Create data directories to use in configuring Cassandra +mkdir -p /data/cassandra/{commitlog,data,saved_caches,hints}/ +chmod -R 0700 /data/cassandra + +# Create logs directory for Cassandra GC logs +mkdir -p "${CASSANDRA_ROOT}/logs" + +# Copy configurations +cp "${TEMPLATES_DIR}/cassandra.yaml" "${CASSANDRA_ROOT}/conf/cassandra.yaml.template" || exit 1 +popd + +echo "JDK and Cassandra installed successfully" + +# ===================================================================== +# Step 4: Build CPython 3.10 from source +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 4: Building CPython 3.10 from source" +echo "=====================================================================" + +pushd "${DJANGO_SERVER_ROOT}" + +# Ubuntu 24.04 ships Python 3.12; build Python 3.10 from source +# (same approach as install_django_workload_aarch64.sh for CentOS) +if ! [ -d Python-3.10.2 ]; then + wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz + tar -xzf Python-3.10.2.tgz + cd Python-3.10.2 + ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" + make install + cd ../ +fi + +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" + +echo "CPython 3.10 built successfully" + +# ===================================================================== +# Step 5: Cinder 3.10 — skipped on ARM +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 5: Skipping Cinder 3.10 (not used on ARM/aarch64)" +echo "=====================================================================" + +# ARM Django jobs use interpreter=cpython with use_jit=0, so Cinder is +# never executed at runtime. Create a placeholder to satisfy the +# install_markers check in benchmarks.yml. +mkdir -p "${DJANGO_SERVER_ROOT}/cinder/cinder-build/bin" +ln -sf "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" \ + "${DJANGO_SERVER_ROOT}/cinder/cinder-build/bin/python3.10" + +echo "Cinder skipped (placeholder created for install marker)" + +# ===================================================================== +# Step 6: Install Python dependencies in virtual environments +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6: Installing Python dependencies in virtual environments" +echo "=====================================================================" + +# Create CPython virtual env only (Cinder venv not needed on ARM) +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython + +# Install packages in CPython environment +set +u +# shellcheck disable=SC1091 +source ./venv_cpython/bin/activate +set -u + +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include" + +# Install dependencies using third_party pip dependencies +pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" + +echo "Dependencies installed in CPython venv" + +# Configure Java options directly +# shellcheck disable=SC2016 +echo 'JVM_OPTS="$JVM_OPTS -Xss512k"' >> "${DJANGO_WORKLOAD_ROOT}/apache-cassandra/conf/cassandra-env.sh" + +deactivate + +echo "Python dependencies installation completed" + +WRK_VERSION="4.2.0" +pushd "${DJANGO_WORKLOAD_ROOT}" || exit 1 +if ! [ -d wrk ]; then + git clone --branch "${WRK_VERSION}" https://github.com/wg/wrk + pushd wrk || exit 1 + git apply --check "${DJANGO_PKG_ROOT}/templates/wrk.diff" && \ + git apply "${DJANGO_PKG_ROOT}/templates/wrk.diff" + make && echo "Wrk built successfully" + popd # wrk +fi +popd # "${DJANGO_WORKLOAD_ROOT}" + +# ===================================================================== +# Step 7: Build and Install Proxygen (for DjangoBench V2) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 7: Building and Installing Proxygen" +echo "=====================================================================" + +# Clone Proxygen if not already present +PROXYGEN_VERSION="v2025.10.13.00" +if [ ! -d "${DJANGO_WORKLOAD_ROOT}/proxygen" ]; then + echo "Cloning Proxygen from GitHub..." + cd "${DJANGO_WORKLOAD_ROOT}" + git clone https://github.com/facebook/proxygen.git + cd proxygen + git checkout "${PROXYGEN_VERSION}" + echo "Proxygen cloned successfully" +else + echo "Proxygen directory already exists at ${DJANGO_WORKLOAD_ROOT}/proxygen" + cd "${DJANGO_WORKLOAD_ROOT}/proxygen" +fi + +# Overwrite build script with custom version +echo "Installing custom build script with -fPIC support..." +cp "${TEMPLATES_DIR}/build_proxygen.sh" "${DJANGO_WORKLOAD_ROOT}/proxygen/proxygen/build_proxygen.sh" +chmod +x "${DJANGO_WORKLOAD_ROOT}/proxygen/proxygen/build_proxygen.sh" + +# Build Proxygen +echo "Building Proxygen (this may take 10-20 minutes)..." +cd "${DJANGO_WORKLOAD_ROOT}/proxygen/proxygen" +bash -x ./build_proxygen.sh --prefix "${DJANGO_WORKLOAD_ROOT}/proxygen/staging" -j "${NUM_BUILD_JOBS}" +bash -x ./install.sh + +echo "Proxygen built and installed at ${DJANGO_WORKLOAD_ROOT}/proxygen/staging" + +# ===================================================================== +# Step 7.5: Build and Install fbthrift (for Thrift RPC Services) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 7.5: Building and Installing fbthrift" +echo "=====================================================================" + +FBTHRIFT_VERSION="v2025.09.22.00" +FBTHRIFT_PREFIX="${DJANGO_WORKLOAD_ROOT}/proxygen/proxygen/_build/deps" + +# Clone fbthrift if not already present +if [ ! -d "${DJANGO_WORKLOAD_ROOT}/fbthrift" ]; then + echo "Cloning fbthrift from GitHub..." + cd "${DJANGO_WORKLOAD_ROOT}" + git clone https://github.com/facebook/fbthrift.git + cd fbthrift + git checkout "${FBTHRIFT_VERSION}" + echo "fbthrift cloned successfully" +else + echo "fbthrift directory already exists at ${DJANGO_WORKLOAD_ROOT}/fbthrift" + cd "${DJANGO_WORKLOAD_ROOT}/fbthrift" +fi + +# Build fbthrift +if [ ! -f "${FBTHRIFT_PREFIX}/bin/thrift1" ]; then + echo "Building fbthrift (this may take 15-25 minutes)..." + cd "${DJANGO_WORKLOAD_ROOT}/fbthrift" + mkdir -p _build + cd _build + + cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_PREFIX_PATH="${FBTHRIFT_PREFIX}" \ + -DCMAKE_INSTALL_PREFIX="${FBTHRIFT_PREFIX}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=True \ + -DCXX_STD=gnu++20 \ + -DCMAKE_CXX_STANDARD=20 \ + .. + + ninja -v -j "${NUM_BUILD_JOBS}" install + echo "fbthrift built and installed at ${FBTHRIFT_PREFIX}" +else + echo "fbthrift already built and installed" +fi + +# ===================================================================== +# Step 7.6: Download and Extract Silesia Corpus Dataset +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 7.6: Downloading and Extracting Silesia Corpus Dataset" +echo "=====================================================================" + +DATASET_DIR="${DJANGO_SERVER_ROOT}/django_workload/feed_flow/dataset" +mkdir -p "${DATASET_DIR}/text" +mkdir -p "${DATASET_DIR}/binary" + +DATASET_DIR2="${DJANGO_SERVER_ROOT}/django_workload/clips_discovery/dataset" +ln -sf "${DATASET_DIR}" "${DATASET_DIR2}" + +DATASET_DIR3="${DJANGO_SERVER_ROOT}/django_workload/reels_tray/dataset" +ln -sf "${DATASET_DIR}" "${DATASET_DIR3}" + +DATASET_DIR4="${DJANGO_SERVER_ROOT}/django_workload/inbox/dataset" +ln -sf "${DATASET_DIR}" "${DATASET_DIR4}" + +# Download Silesia Corpus if not already present +if [ ! -f "${DJANGO_WORKLOAD_ROOT}/silesia.zip" ]; then + echo "Downloading Silesia Corpus dataset..." + cd "${DJANGO_WORKLOAD_ROOT}" + wget "https://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip" + echo "Silesia Corpus downloaded successfully" +else + echo "Silesia Corpus already downloaded" +fi + +# Extract dataset files +if [ ! -f "${DATASET_DIR}/text/dickens" ]; then + echo "Extracting Silesia Corpus dataset..." + cd "${DJANGO_WORKLOAD_ROOT}" + unzip -o silesia.zip -d silesia_extracted + + # Copy text files + echo "Copying text files to ${DATASET_DIR}/text..." + cp silesia_extracted/dickens "${DATASET_DIR}/text/" + + # Copy binary files + echo "Copying binary files to ${DATASET_DIR}/binary..." + cp silesia_extracted/sao "${DATASET_DIR}/binary/" + + echo "Dataset files extracted and organized successfully" +else + echo "Dataset files already extracted" +fi + +# ===================================================================== +# Step 7.7: Build Mock Thrift Server +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 7.7: Building Mock Thrift Server" +echo "=====================================================================" + +THRIFT_SERVER_DIR="${DJANGO_SERVER_ROOT}/django_workload/thrift" +export FBTHRIFT_PREFIX="${FBTHRIFT_PREFIX}" + +# Build thrift server +echo "Building mock thrift server..." +cd "${THRIFT_SERVER_DIR}" +./build.sh +echo "Mock thrift server built" + +# ===================================================================== +# Step 8: Build and Install proxygen_binding (for DjangoBench V2) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 8: Building and Installing proxygen_binding" +echo "=====================================================================" + +# Copy proxygen_binding to django_workload root +if [ ! -d "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" ]; then + echo "Copying proxygen_binding module..." + cp -r "${DJANGO_PKG_ROOT}/srcs/proxygen_binding" "${DJANGO_WORKLOAD_ROOT}/" + echo "proxygen_binding copied to ${DJANGO_WORKLOAD_ROOT}/proxygen_binding" +else + echo "proxygen_binding directory already exists, updating..." + rm -rf "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" + cp -r "${DJANGO_PKG_ROOT}/srcs/proxygen_binding" "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" +fi + +# Set Proxygen installation directory for building proxygen_binding +export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" + +# Build and install in venv_cpython only (no Cinder venv on ARM) +echo "" +echo "Installing proxygen_binding in venv_cpython..." +cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" +"${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 +"${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . +echo "proxygen_binding installed in venv_cpython" + +# ===================================================================== +# Step 9: Generate Code Variants for FeedFlow +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 9: Generating Code Variants for FeedFlow" +echo "=====================================================================" + +# Install jinja2 in venv_cpython +echo "Installing jinja2 for code generation..." +"${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 + +# Generate code variants +echo "Generating FeedFlow code variants..." +cd "${DJANGO_SERVER_ROOT}" +"${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" generate_code_variants.py + +echo "Code variants generated successfully" + +echo "" +echo "=====================================================================" +echo "DjangoBench installation completed successfully!" +echo "=====================================================================" +echo "" +echo "Installation directory: ${DJANGO_WORKLOAD_ROOT}" +echo "" +echo "DjangoBench V2 Components (Async HTTP with Proxygen):" +echo " - Proxygen: ${DJANGO_WORKLOAD_ROOT}/proxygen/staging" +echo " - proxygen_binding: ${DJANGO_WORKLOAD_ROOT}/proxygen_binding" +echo " - Django workload: ${DJANGO_WORKLOAD_ROOT}/django-workload/django-workload" +echo "" +echo "To run DjangoBench V2 with Proxygen (asynchronous HTTP):" +echo " cd ${DJANGO_WORKLOAD_ROOT}/django-workload/django-workload" +echo " ./run_proxygen.sh" +echo "" +echo "To run DjangoBench V1 with uWSGI (traditional):" +echo " cd ${DJANGO_WORKLOAD_ROOT}/django-workload/django-workload" +echo " ./run.sh" +echo "" +echo "=====================================================================" diff --git a/packages/django_workload/srcs/bin/run.sh b/packages/django_workload/srcs/bin/run.sh index 19770105..b38a2a95 100755 --- a/packages/django_workload/srcs/bin/run.sh +++ b/packages/django_workload/srcs/bin/run.sh @@ -562,7 +562,10 @@ start_django_server() { # Create database schema export PYTHONPATH="${SCRIPT_ROOT}/../django-workload/django-workload/" export LD_LIBRARY_PATH=${SCRIPT_ROOT}/../django-workload/django-workload/:\ -${python_libs}:${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +${SCRIPT_ROOT}/../proxygen/staging/lib:\ +${SCRIPT_ROOT}/../proxygen/proxygen/_build/deps/lib:\ +${SCRIPT_ROOT}/../proxygen/proxygen/_build/deps/lib64:\ +${python_libs}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} echo "Setting LD_LIBRARY_PATH to '${LD_LIBRARY_PATH}'" if [ "$load_a_snapshot" = true ]; then #DJANGO_SETTINGS_MODULE=cluster_settings ./${venv_dir}/bin/django-admin flush