From 07c6832b9ffaef46b36340f09a0daa13d5c168b7 Mon Sep 17 00:00:00 2001 From: Wei Su Date: Tue, 10 Mar 2026 13:29:31 -0700 Subject: [PATCH 1/2] Upgrade to Python 3.14, Cinder 3.14, and add CinderX JIT (#476) Summary: Upgrade DjangoBench V2 to use Python 3.14 and Cinder 3.14 to take advantage of newer language features and performance improvements. This also adds CinderX for JIT compilation support in the Cinder runtime environment. Key changes: - CPython upgraded from 3.10 to 3.14 (built from source on all platforms) - Cinder upgraded from cinder/3.10 branch to meta/3.14 at pinned commit - CinderX installed for JIT functionality with validation - Removed numpy dependency in favor of Python's built-in statistics module - cassandra-driver now builds from PyPI source for Python 3.14 compatibility Differential Revision: D91604990 --- benchpress/config/benchmarks.yml | 4 +- .../install_django_workload_aarch64.sh | 187 ++++++++++++++--- ...nstall_django_workload_aarch64_ubuntu22.sh | 193 +++++++++++++++--- .../install_django_workload_x86_64_centos9.sh | 188 ++++++++++++++--- ...install_django_workload_x86_64_ubuntu22.sh | 192 ++++++++++++++--- packages/django_workload/srcs/bin/run.sh | 42 ++-- .../srcs/django-workload/client/run-siege | 14 +- .../srcs/django-workload/client/run-wrk | 14 +- .../django_workload/cache_backend.py | 53 +++++ .../django_workload/settings.py | 6 +- .../start_loadbalanced_server.py | 12 ++ .../django-workload/django-workload/uwsgi.ini | 4 + .../services/memcached/run-memcached | 10 +- .../patches/pylibmc-1.6.1-py314-compat.patch | 99 +++++++++ .../srcs/proxygen_binding/pyproject.toml | 20 ++ .../srcs/proxygen_binding/setup.py | 1 + 16 files changed, 884 insertions(+), 155 deletions(-) create mode 100644 packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py create mode 100644 packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch create mode 100644 packages/django_workload/srcs/proxygen_binding/pyproject.toml diff --git a/benchpress/config/benchmarks.yml b/benchpress/config/benchmarks.yml index fd1d7430..3f52554e 100644 --- a/benchpress/config/benchmarks.yml +++ b/benchpress/config/benchmarks.yml @@ -28,8 +28,8 @@ django_workload: - ./benchmarks/django_workload/proxygen_binding/proxygen_binding*.so - ./benchmarks/django_workload/django-workload/django-workload/django_workload/thrift/build/gen-py3/mock_services/MockAdsService-remote - ./benchmarks/django_workload/django-workload/django-workload/django_workload/feed_timeline.py - - ./benchmarks/django_workload/django-workload/django-workload/cinder/cinder-build/bin/python3.10 - - ./benchmarks/django_workload/django-workload/django-workload/Python-3.10.2/python-build/bin/python3.10 + - ./benchmarks/django_workload/django-workload/django-workload/cinder/cinder-build/bin/python3.14 + - ./benchmarks/django_workload/django-workload/django-workload/Python-3.14.2/python-build/bin/python3.14 - ./benchmarks/django_workload/wrk/wrk - ./benchmarks/django_workload/apache-cassandra/bin/cassandra path: ./benchmarks/django_workload/bin/run.sh diff --git a/packages/django_workload/install_django_workload_aarch64.sh b/packages/django_workload/install_django_workload_aarch64.sh index fc64ca5b..755d81d9 100755 --- a/packages/django_workload/install_django_workload_aarch64.sh +++ b/packages/django_workload/install_django_workload_aarch64.sh @@ -40,7 +40,7 @@ dnf groupinstall "Development Tools" -y --exclude="texlive*" dnf install -y memcached libmemcached-awesome-devel zlib-devel screen \ openssl-devel bzip2-devel libffi-devel wget make xz-devel haproxy \ xxhash-devel perl-FindBin perl-JSON perl-core liburing-devel \ - ninja-build + ninja-build clang libev libev-devel echo "System dependencies installed successfully" @@ -70,29 +70,21 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra_driver-3.29.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -wget "https://files.pythonhosted.org/packages/cc/60/f8de88175937481be98da88eb88b4fd704093e284e5907775293c496df32/cassandra_driver-3.29.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" -# Removed Cython download as it's not needed # Django-5.2.tar.gz wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django_statsd_mozilla-0.4.0-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -wget "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1-cp36-cp36m-manylinux1_x86_64.whl -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -219,45 +211,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Install python3.10 -if ! [ -d Python-3.10.2 ]; then - wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz - tar -xzf Python-3.10.2.tgz - cd Python-3.10.2 +# Install python3.14 +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install cd ../ fi -CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 built successfully" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -266,7 +261,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -278,11 +273,57 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 @@ -291,12 +332,35 @@ set -u export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -311,16 +375,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -513,6 +628,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -521,6 +637,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -535,6 +652,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh index f248a64e..836b69e5 100755 --- a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh @@ -39,7 +39,7 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl ninja-build + perl ninja-build clang libev4 libev-dev echo "System dependencies installed successfully" @@ -69,31 +69,23 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.26_aarch64.whl -wget "https://files.pythonhosted.org/packages/b5/5e/54c58c98a4eeea12a2fee7220e7ac9e8b021ea5c3d84c84adb9106c4ed43/cassandra_driver-3.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" -# Removed Cython download as it's not needed # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" # Dulwich 0.21.2.tar.gz wget "https://files.pythonhosted.org/packages/14/a5/cf61f9209d48abf47d48086e0a0388f1030bb5f7cf2661972eee56ccee3d/dulwich-0.21.2.tar.gz" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django-statsd-mozilla-0.4.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.22.1-aarch64.whl -wget "https://files.pythonhosted.org/packages/d6/ec/a8b5f1b6d00bc4fd1bc91043d5dfb029536ec5c7769588d3f4c982240008/numpy-1.22.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1.tar.gz -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd-3.3.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -220,34 +212,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Ubuntu 22 comes with python3.10, so we use system python -echo "Using system Python 3.10" +# Install python3.14 from source +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 + ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" + make install + cd ../ +fi -echo "CPython 3.10 ready" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" + +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -256,7 +262,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -268,22 +274,96 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder # Create CPython virtual env -python3.10 -m venv venv_cpython +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython # Create Cinder virtual env export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 source ./venv_cpython/bin/activate set -u +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -298,16 +378,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -500,6 +631,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -508,6 +640,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -522,6 +655,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_x86_64_centos9.sh b/packages/django_workload/install_django_workload_x86_64_centos9.sh index dbb64348..150884b6 100755 --- a/packages/django_workload/install_django_workload_x86_64_centos9.sh +++ b/packages/django_workload/install_django_workload_x86_64_centos9.sh @@ -39,7 +39,8 @@ echo "=====================================================================" dnf groupinstall "Development Tools" -y --exclude="texlive*" dnf install -y memcached libmemcached-awesome-devel zlib-devel screen \ openssl-devel bzip2-devel libffi-devel wget make haproxy xxhash-devel \ - perl-FindBin perl-JSON perl-core liburing-devel ninja-build + perl-FindBin perl-JSON perl-core liburing-devel ninja-build clang \ + libev libev-devel echo "System dependencies installed successfully" @@ -67,21 +68,14 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.29.1_x86_64.whl -wget "https://files.pythonhosted.org/packages/eb/d5/e437271aea182e33db32e0990703b4e0d7025e4fba67829c5fd65dba926a/cassandra_driver-3.29.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" -# Removed Cython download as it's not needed # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django_statsd_mozilla-0.4.0-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-x86_64.whl -wget "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1-cp36-cp36m-manylinux1_x86_64.whl -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl @@ -89,8 +83,7 @@ wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2d # statsd # wget "https://files.pythonhosted.org/packages/2c/a8/714954464435178017e8d2f39ff418e0c9ad4411a416d4acc315298b9cea/statsd-2.1.2.tar.gz" wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -217,45 +210,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Install python3.10 -if ! [ -d Python-3.10.2 ]; then - wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz - tar -xzf Python-3.10.2.tgz - cd Python-3.10.2 +# Install python3.14 +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install cd ../ fi -CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 built successfully" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -264,7 +260,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -276,11 +272,57 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 @@ -289,12 +331,35 @@ set -u export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -308,16 +373,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -510,6 +626,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -518,6 +635,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -532,6 +650,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh index 6885f2ad..80f7a74b 100755 --- a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh @@ -39,7 +39,7 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl ninja-build + perl liburing-dev ninja-build clang libev4 libev-dev echo "System dependencies installed successfully" @@ -69,30 +69,23 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.29.1_x86_64.whl -wget "https://files.pythonhosted.org/packages/eb/d5/e437271aea182e33db32e0990703b4e0d7025e4fba67829c5fd65dba926a/cassandra_driver-3.29.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" # Dulwich 0.21.2.tar.gz wget "https://files.pythonhosted.org/packages/14/a5/cf61f9209d48abf47d48086e0a0388f1030bb5f7cf2661972eee56ccee3d/dulwich-0.21.2.tar.gz" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django-statsd-mozilla-0.4.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-x86_64.whl -wget "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1.tar.gz -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd-3.3.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -217,34 +210,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Ubuntu 22 comes with python3.10, so we use system python -echo "Using system Python 3.10" +# Install python3.14 from source +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 + ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" + make install + cd ../ +fi -echo "CPython 3.10 ready" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" + +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -253,7 +260,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -265,22 +272,96 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder # Create CPython virtual env -python3.10 -m venv venv_cpython +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython # Create Cinder virtual env export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 source ./venv_cpython/bin/activate set -u +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -295,16 +376,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -497,6 +629,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -505,6 +638,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -519,6 +653,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/srcs/bin/run.sh b/packages/django_workload/srcs/bin/run.sh index 19770105..cffc973b 100755 --- a/packages/django_workload/srcs/bin/run.sh +++ b/packages/django_workload/srcs/bin/run.sh @@ -10,7 +10,7 @@ SCRIPT_ROOT="$(dirname "$(readlink -f "$0")")" BENCHPRESS_ROOT="$(readlink -f "${SCRIPT_ROOT}/../../..")" DJANGO_PKG_SRC_ROOT="${BENCHPRESS_ROOT}/packages/django_workload/srcs" DJANGO_SERVER_ROOT="$(readlink -f "${SCRIPT_ROOT}/../django-workload/django-workload")" -CPYTHON_PATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_PATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" CINDER_PATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build" MEMCACHED_PID= CASSANDRA_PID= @@ -142,8 +142,27 @@ cleanup() { # echo "$HAPROXY_PIDS" | xargs -r kill -9 2>/dev/null || true # fi - # Stop memcached - [ -n "$MEMCACHED_PID" ] && { echo "Stopping memcached"; kill "$MEMCACHED_PID" 2>/dev/null || true; } + # Stop memcached using PID file + if [ -f memcached.pid ]; then + MEMCACHED_PID="$(cat memcached.pid)" + echo "Stopping memcached (PID: $MEMCACHED_PID)" + kill "$MEMCACHED_PID" 2>/dev/null || true + + # Wait briefly for graceful shutdown + sleep 1 + + # Force kill if still running + if kill -0 "$MEMCACHED_PID" 2>/dev/null; then + echo "Force killing memcached (PID: $MEMCACHED_PID)" + kill -9 "$MEMCACHED_PID" 2>/dev/null || true + fi + + rm -f memcached.pid + elif [ -n "$MEMCACHED_PID" ]; then + # Fallback to variable if PID file doesn't exist + echo "Stopping memcached (PID: $MEMCACHED_PID)" + kill "$MEMCACHED_PID" 2>/dev/null || true + fi # Stop Cassandra using PID file if [ -f cassandra.pid ]; then @@ -515,15 +534,12 @@ start_django_server() { # Export FBTHRIFT_PREFIX for thrift Python bindings export FBTHRIFT_PREFIX="${SCRIPT_ROOT}/../proxygen/proxygen/_build/deps" - # Enable JIT if requested (Cinder on x86 only) - if [ "${use_jit}" -gt 0 ] && [ "${interpreter}" = "cinder" ]; then - echo "Enabling Cinder JIT..." - export PYTHONJIT=1 - export PYTHONJITWRITEPROFILE=/tmp/cinder-jit.profile - export PYTHONJITPROFILEINTERP=1 - export PYTHONJITPROFILEINTERPPERIOD=10 - #export PYTHONJITDUMPSTATS=1 - export PYTHONJITALLSTATICFUNCTIONS=1 + # Enable JIT if requested (Cinder only) + if [ "${use_jit}" -ge 0 ] && [ "${interpreter}" = "cinder" ]; then + echo "Enabling Cinder JIT (CinderX)..." + export PYTHONJITALL=1 + export PYTHONJITLOGFILE=/tmp/cinder-jit.log + export PYTHONJITDUMPSTATS=1 fi # Export FBTHRIFT_PREFIX for thrift Python bindings @@ -534,6 +550,8 @@ start_django_server() { ./django-workload/services/memcached/run-memcached > memcached.log 2>&1 & MEMCACHED_PID=$! + echo "$MEMCACHED_PID" > memcached.pid + echo "Started memcached with PID: $MEMCACHED_PID" # Start django-workload # Set the cassandra ip in django config file diff --git a/packages/django_workload/srcs/django-workload/client/run-siege b/packages/django_workload/srcs/django-workload/client/run-siege index 85c43f81..d5207400 100755 --- a/packages/django_workload/srcs/django-workload/client/run-siege +++ b/packages/django_workload/srcs/django-workload/client/run-siege @@ -8,7 +8,7 @@ import optparse import os import subprocess import re -import numpy as np +import statistics # Concurrent worker count WORKERS = 185 @@ -274,8 +274,8 @@ def parse_results(iterations, url_dict, url_target): if iterations > 1: del url_dict[url][second_idx] del url_dict[url][first_idx] - arr = np.array(url_dict[url]) - arr_mean = np.mean(arr) * 100 + arr = url_dict[url] + arr_mean = statistics.mean(arr) * 100 if arr else 0 print(padding(url, 3), end="") print(str(arr_mean) + "%, expected " + str(url_target[url]) + "%") print() @@ -287,13 +287,13 @@ def parse_results(iterations, url_dict, url_target): print(padding(metric, 5), end="") - arr = np.array(results[metric]) - arr_mean = np.mean(arr) + arr = results[metric] + arr_mean = statistics.mean(arr) if arr else -1 if arr_mean >= 0: - if arr_mean == 0: + if arr_mean == 0 or len(arr) < 2: arr_rsd = 0 else: - arr_rsd = np.std(arr) / arr_mean + arr_rsd = statistics.stdev(arr) / arr_mean print(str(arr_mean) + " " + unit_measures[metric], end="") print(" ---- RSD " + str(arr_rsd)) else: diff --git a/packages/django_workload/srcs/django-workload/client/run-wrk b/packages/django_workload/srcs/django-workload/client/run-wrk index 8cd44cd1..ba62c315 100755 --- a/packages/django_workload/srcs/django-workload/client/run-wrk +++ b/packages/django_workload/srcs/django-workload/client/run-wrk @@ -8,7 +8,7 @@ import optparse import os import subprocess import re -import numpy as np +import statistics # Concurrent worker count WORKERS = 185 @@ -482,8 +482,8 @@ def parse_results(iterations, url_dict, url_target): if iterations > 1: del base_url_dict[url][second_idx] del base_url_dict[url][first_idx] - arr = np.array(base_url_dict[url]) - arr_mean = np.mean(arr) * 100 + arr = base_url_dict[url] + arr_mean = statistics.mean(arr) * 100 if arr else 0 expected_perc = (base_url_target[url] / total_expected_weight) * 100 print(padding(url, 3), end="") print(str(arr_mean) + "%, expected " + str(expected_perc) + "%") @@ -497,13 +497,13 @@ def parse_results(iterations, url_dict, url_target): print(padding(metric, 5), end="") - arr = np.array(results[metric]) - arr_mean = np.mean(arr) + arr = results[metric] + arr_mean = statistics.mean(arr) if arr else -1 if arr_mean >= 0: - if arr_mean == 0: + if arr_mean == 0 or len(arr) < 2: arr_rsd = 0 else: - arr_rsd = np.std(arr) / arr_mean + arr_rsd = statistics.stdev(arr) / arr_mean print(str(arr_mean) + " " + unit_measures[metric], end="") print(" ---- RSD " + str(arr_rsd)) else: diff --git a/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py b/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py new file mode 100644 index 00000000..b79faa7b --- /dev/null +++ b/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py @@ -0,0 +1,53 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Fault-tolerant PyLibMCCache backend. + +Wraps Django's PyLibMCCache to catch pylibmc connection errors (e.g. ENOTCONN) +and treat them as cache misses instead of crashing the request. This is +necessary because DjangoBench V2's per-worker-port architecture creates 176 +independent pylibmc connections to a single memcached instance, and transient +connection failures are expected under high concurrency. +""" + +import logging + +from django.core.cache.backends.memcached import PyLibMCCache + +logger = logging.getLogger(__name__) + + +class FaultTolerantPyLibMCCache(PyLibMCCache): + """PyLibMCCache that treats connection errors as cache misses.""" + + def get(self, key, default=None, version=None): + try: + return super().get(key, default, version) + except Exception: + return default + + def set(self, key, value, timeout=None, version=None): + try: + return super().set(key, value, timeout, version) + except Exception: + return False + + def delete(self, key, version=None): + try: + return super().delete(key, version) + except Exception: + return False + + def get_many(self, keys, version=None): + try: + return super().get_many(keys, version) + except Exception: + return {} + + def set_many(self, mapping, timeout=None, version=None): + try: + return super().set_many(mapping, timeout, version) + except Exception: + return list(mapping.keys()) diff --git a/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py b/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py index f043a103..9dc27ed5 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py +++ b/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py @@ -140,9 +140,13 @@ STATSD_IPV6 = False # Cache configuration +# Uses FaultTolerantPyLibMCCache to handle transient pylibmc connection errors +# (ENOTCONN, CONNECTION FAILURE) gracefully as cache misses instead of crashing +# requests. This is needed because V2's per-worker-port architecture creates +# 176 independent pylibmc connections to a single memcached instance. CACHES = { "default": { - "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", + "BACKEND": "django_workload.cache_backend.FaultTolerantPyLibMCCache", "LOCATION": "127.0.0.1:11811", } } diff --git a/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py b/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py index c3d64cd9..24feb3b6 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py +++ b/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py @@ -121,6 +121,12 @@ def start_uwsgi(self) -> None: env["PROXYGEN_THREADS"] = str(self.threads_per_worker) env["PYTHONUNBUFFERED"] = "1" + # Ensure LD_LIBRARY_PATH is set for Python 3.14 shared library + # This is critical for uWSGI workers to find libpython3.14.so + if "LD_LIBRARY_PATH" in os.environ: + env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + logger.info(f"Using LD_LIBRARY_PATH: {env['LD_LIBRARY_PATH']}") + # Start uWSGI with config overrides # Use --set to override config variables (standard uWSGI way) self.uwsgi_process = subprocess.Popen( @@ -775,6 +781,12 @@ def start_worker(self, worker_id: int, port: int) -> subprocess.Popen: env = os.environ.copy() env["PYTHONUNBUFFERED"] = "1" # Disable output buffering + # Ensure LD_LIBRARY_PATH is set for Python 3.14 shared library + # This is critical for uWSGI workers to find libpython3.14.so + if "LD_LIBRARY_PATH" in os.environ: + env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + logger.info(f"Using LD_LIBRARY_PATH: {env['LD_LIBRARY_PATH']}") + # Start worker process process = subprocess.Popen( [ diff --git a/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini b/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini index 30e1ab05..f22d6387 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini +++ b/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini @@ -42,3 +42,7 @@ thunder-lock = True stats = 127.0.0.1:9191 logger = file:django-uwsgi.log disable-logging = True + +# Python 3.14 shared library path - required for built-in modules (math, _socket, etc.) +# This must point to the Cinder/CPython lib directory containing libpython3.14.so +# The actual path is set by the run script via environment variable diff --git a/packages/django_workload/srcs/django-workload/services/memcached/run-memcached b/packages/django_workload/srcs/django-workload/services/memcached/run-memcached index f5c898e1..16f690f1 100755 --- a/packages/django_workload/srcs/django-workload/services/memcached/run-memcached +++ b/packages/django_workload/srcs/django-workload/services/memcached/run-memcached @@ -15,4 +15,12 @@ PORT="${PORT:-11811}" # User to run under USER="${USER:-memcache}" -/usr/bin/memcached -u "$USER" -m "$MEMORY" -l "$LISTEN" -p "$PORT" -t 16 +# Scale threads to match uWSGI worker count (1 worker = 1 connection). +# V2 architecture uses per-worker ports, so each worker has an independent +# memcached connection. Default memcached -t 16 is insufficient when +# num_workers >> 16 (e.g., 176 on T1_BGM). +NPROC=$(nproc) +THREADS=$((NPROC > 16 ? NPROC / 2 : 16)) +MAXCONNS=$((NPROC * 8)) + +/usr/bin/memcached -u "$USER" -m "$MEMORY" -l "$LISTEN" -p "$PORT" -t "$THREADS" -c "$MAXCONNS" diff --git a/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch b/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch new file mode 100644 index 00000000..c38f01eb --- /dev/null +++ b/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch @@ -0,0 +1,99 @@ +diff --git a/setup.py b/setup.py +index 9bea257..84b3aeb 100644 +--- a/setup.py ++++ b/setup.py +@@ -1,11 +1,6 @@ +-from __future__ import print_function + import os + import sys +-from distutils.core import setup, Extension +- +-# Need an 'open' function that supports the 'encoding' argument: +-if sys.version_info[0] < 3: +- from codecs import open ++from setuptools import setup, Extension + + ## Command-line argument parsing + +@@ -95,9 +90,9 @@ if cmd == "gen-setup": + s.write(line + "\n") + sys.exit(0) + +-with open("README.rst", "U", encoding="utf-8") as r: ++with open("README.rst", encoding="utf-8") as r: + readme_text = r.read() +-with open("src/pylibmc-version.h", "U", encoding="utf-8") as r: ++with open("src/pylibmc-version.h", encoding="utf-8") as r: + version = r.read().strip().split("\"")[1] + + setup( +diff --git a/src/_pylibmcmodule.c b/src/_pylibmcmodule.c +index 5324d1d..077f6e2 100644 +--- a/src/_pylibmcmodule.c ++++ b/src/_pylibmcmodule.c +@@ -1714,7 +1714,7 @@ memcached_return pylibmc_memcached_fetch_multi(memcached_st *mc, pylibmc_mget_re + + /* Allocate as much as could possibly be needed, and an extra because of + * how libmemcached signals EOF. */ +- *req.results = PyMem_New(memcached_result_st, req.nkeys + 1); ++ *req.results = PyMem_RawNew(memcached_result_st, req.nkeys + 1); + + /* Note that nresults will not be off by one with this because the loops + * runs a half pass after the last key has been fetched, thus bumping the +@@ -1742,7 +1742,7 @@ memcached_return pylibmc_memcached_fetch_multi(memcached_st *mc, pylibmc_mget_re + memcached_result_free(*req.results + *req.nresults); + } while ((*req.nresults)--); + +- PyMem_Free(*req.results); ++ PyMem_RawDel(*req.results); + *req.results = NULL; + *req.nresults = 0; + +@@ -1942,7 +1942,7 @@ memory_cleanup: + for (i = 0; i < nresults && results != NULL; i++) { + memcached_result_free(results + i); + } +- PyMem_Free(results); ++ PyMem_RawDel(results); + } + + /* Not INCREFing because the only two outcomes are NULL and a new dict. +@@ -2167,7 +2167,7 @@ static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, + char *key; + + for (b = PylibMC_behaviors; b->name != NULL; b++) { +- if (!PyMapping_HasKeyString(behaviors, b->name)) { ++ if (behaviors == Py_None || !PyMapping_HasKeyString(behaviors, b->name)) { + continue; + } else if ((py_v = PyMapping_GetItemString(behaviors, b->name)) == NULL) { + goto error; +@@ -2197,7 +2197,7 @@ static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, + } + + for (b = PylibMC_callbacks; b->name != NULL; b++) { +- if (!PyMapping_HasKeyString(behaviors, b->name)) { ++ if (behaviors == Py_None || !PyMapping_HasKeyString(behaviors, b->name)) { + continue; + } else if ((py_v = PyMapping_GetItemString(behaviors, b->name)) == NULL) { + goto error; +diff --git a/src/_pylibmcmodule.h b/src/_pylibmcmodule.h +index cee9772..d28dec6 100644 +--- a/src/_pylibmcmodule.h ++++ b/src/_pylibmcmodule.h +@@ -52,6 +52,16 @@ + typedef ssize_t Py_ssize_t; + #endif + ++/* Python 3.4+ PyMem_New/PyMem_Free require the GIL. Use Raw variants for ++ * allocations in GIL-free sections (e.g. inside Py_BEGIN_ALLOW_THREADS). */ ++#if PY_VERSION_HEX >= 0x03040000 ++# define PyMem_RawNew(type, nelem) ((type *)PyMem_RawMalloc((nelem)*sizeof(type))) ++# define PyMem_RawDel(ptr) PyMem_RawFree(ptr) ++#else ++# define PyMem_RawNew PyMem_New ++# define PyMem_RawDel(ptr) PyMem_Del(ptr) ++#endif ++ + /* Server types. */ + #define PYLIBMC_SERVER_TCP (1 << 0) + #define PYLIBMC_SERVER_UDP (1 << 1) diff --git a/packages/django_workload/srcs/proxygen_binding/pyproject.toml b/packages/django_workload/srcs/proxygen_binding/pyproject.toml new file mode 100644 index 00000000..0650a122 --- /dev/null +++ b/packages/django_workload/srcs/proxygen_binding/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "pybind11>=2.6.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "proxygen_binding" +version = "0.1.0" +description = "Python bindings for Proxygen HTTP server" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} +authors = [ + {name = "Meta Platforms, Inc."} +] +dependencies = [ + "pybind11>=2.6.0", +] + +[project.optional-dependencies] +django = ["django>=3.0"] diff --git a/packages/django_workload/srcs/proxygen_binding/setup.py b/packages/django_workload/srcs/proxygen_binding/setup.py index 54f843d9..61f2c43f 100644 --- a/packages/django_workload/srcs/proxygen_binding/setup.py +++ b/packages/django_workload/srcs/proxygen_binding/setup.py @@ -235,6 +235,7 @@ def get_proxygen_paths(custom_args): "django_asgi_adapter", "example_server", "django_server", + "event_loop_manager", ], cmdclass={"build_ext": build_ext}, zip_safe=False, From 4626218ecb96cb4a40a42107dcec6e403031a5bc Mon Sep 17 00:00:00 2001 From: Ahmad El Youssef Date: Tue, 10 Mar 2026 13:29:31 -0700 Subject: [PATCH 2/2] testing django python 3.14 (#514) Summary: Pull Request resolved: https://github.com/facebookresearch/DCPerf/pull/514 Differential Revision: D95994832 --- .../install_django_workload_aarch64_ubuntu22.sh | 15 ++++++++++++++- .../install_django_workload_x86_64_ubuntu22.sh | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh index 836b69e5..464c5cf7 100755 --- a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh @@ -39,7 +39,20 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl ninja-build clang libev4 libev-dev + perl ninja-build libev4 libev-dev cmake \ + software-properties-common + +# Install Clang 17 for CinderX compatibility +# CinderX requires GCC 13+ or Clang 15+ for C23 attribute support ([[clang::always_inline]]) +# Ubuntu 22.04's default Clang 14 doesn't support this, so we install Clang 17 from LLVM repos +wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc +add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" +apt update +apt install -y clang-17 + +# Set Clang 17 as the default clang +update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 +update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 echo "System dependencies installed successfully" diff --git a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh index 80f7a74b..b122ddf0 100755 --- a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh @@ -39,7 +39,20 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl liburing-dev ninja-build clang libev4 libev-dev + perl liburing-dev ninja-build libev4 libev-dev cmake \ + software-properties-common + +# Install Clang 17 for CinderX compatibility +# CinderX requires GCC 13+ or Clang 15+ for C23 attribute support ([[clang::always_inline]]) +# Ubuntu 22.04's default Clang 14 doesn't support this, so we install Clang 17 from LLVM repos +wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc +add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" +apt update +apt install -y clang-17 + +# Set Clang 17 as the default clang +update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 +update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 echo "System dependencies installed successfully"