diff --git a/benchpress/config/benchmarks.yml b/benchpress/config/benchmarks.yml index fd1d7430..3f52554e 100644 --- a/benchpress/config/benchmarks.yml +++ b/benchpress/config/benchmarks.yml @@ -28,8 +28,8 @@ django_workload: - ./benchmarks/django_workload/proxygen_binding/proxygen_binding*.so - ./benchmarks/django_workload/django-workload/django-workload/django_workload/thrift/build/gen-py3/mock_services/MockAdsService-remote - ./benchmarks/django_workload/django-workload/django-workload/django_workload/feed_timeline.py - - ./benchmarks/django_workload/django-workload/django-workload/cinder/cinder-build/bin/python3.10 - - ./benchmarks/django_workload/django-workload/django-workload/Python-3.10.2/python-build/bin/python3.10 + - ./benchmarks/django_workload/django-workload/django-workload/cinder/cinder-build/bin/python3.14 + - ./benchmarks/django_workload/django-workload/django-workload/Python-3.14.2/python-build/bin/python3.14 - ./benchmarks/django_workload/wrk/wrk - ./benchmarks/django_workload/apache-cassandra/bin/cassandra path: ./benchmarks/django_workload/bin/run.sh diff --git a/packages/django_workload/install_django_workload_aarch64.sh b/packages/django_workload/install_django_workload_aarch64.sh index fc64ca5b..755d81d9 100755 --- a/packages/django_workload/install_django_workload_aarch64.sh +++ b/packages/django_workload/install_django_workload_aarch64.sh @@ -40,7 +40,7 @@ dnf groupinstall "Development Tools" -y --exclude="texlive*" dnf install -y memcached libmemcached-awesome-devel zlib-devel screen \ openssl-devel bzip2-devel libffi-devel wget make xz-devel haproxy \ xxhash-devel perl-FindBin perl-JSON perl-core liburing-devel \ - ninja-build + ninja-build clang libev libev-devel echo "System dependencies installed successfully" @@ -70,29 +70,21 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra_driver-3.29.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -wget "https://files.pythonhosted.org/packages/cc/60/f8de88175937481be98da88eb88b4fd704093e284e5907775293c496df32/cassandra_driver-3.29.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" -# Removed Cython download as it's not needed # Django-5.2.tar.gz wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django_statsd_mozilla-0.4.0-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -wget "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1-cp36-cp36m-manylinux1_x86_64.whl -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -219,45 +211,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Install python3.10 -if ! [ -d Python-3.10.2 ]; then - wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz - tar -xzf Python-3.10.2.tgz - cd Python-3.10.2 +# Install python3.14 +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install cd ../ fi -CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 built successfully" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -266,7 +261,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -278,11 +273,57 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 @@ -291,12 +332,35 @@ set -u export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -311,16 +375,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -513,6 +628,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -521,6 +637,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -535,6 +652,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh index f248a64e..464c5cf7 100755 --- a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh @@ -39,7 +39,20 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl ninja-build + perl ninja-build libev4 libev-dev cmake \ + software-properties-common + +# Install Clang 17 for CinderX compatibility +# CinderX requires GCC 13+ or Clang 15+ for C23 attribute support ([[clang::always_inline]]) +# Ubuntu 22.04's default Clang 14 doesn't support this, so we install Clang 17 from LLVM repos +wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc +add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" +apt update +apt install -y clang-17 + +# Set Clang 17 as the default clang +update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 +update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 echo "System dependencies installed successfully" @@ -69,31 +82,23 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.26_aarch64.whl -wget "https://files.pythonhosted.org/packages/b5/5e/54c58c98a4eeea12a2fee7220e7ac9e8b021ea5c3d84c84adb9106c4ed43/cassandra_driver-3.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" -# Removed Cython download as it's not needed # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" # Dulwich 0.21.2.tar.gz wget "https://files.pythonhosted.org/packages/14/a5/cf61f9209d48abf47d48086e0a0388f1030bb5f7cf2661972eee56ccee3d/dulwich-0.21.2.tar.gz" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django-statsd-mozilla-0.4.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.22.1-aarch64.whl -wget "https://files.pythonhosted.org/packages/d6/ec/a8b5f1b6d00bc4fd1bc91043d5dfb029536ec5c7769588d3f4c982240008/numpy-1.22.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1.tar.gz -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd-3.3.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -220,34 +225,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Ubuntu 22 comes with python3.10, so we use system python -echo "Using system Python 3.10" +# Install python3.14 from source +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 + ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" + make install + cd ../ +fi + +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 ready" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -256,7 +275,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -268,22 +287,96 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder # Create CPython virtual env -python3.10 -m venv venv_cpython +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython # Create Cinder virtual env export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 source ./venv_cpython/bin/activate set -u +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -298,16 +391,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -500,6 +644,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -508,6 +653,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -522,6 +668,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_x86_64_centos9.sh b/packages/django_workload/install_django_workload_x86_64_centos9.sh index dbb64348..150884b6 100755 --- a/packages/django_workload/install_django_workload_x86_64_centos9.sh +++ b/packages/django_workload/install_django_workload_x86_64_centos9.sh @@ -39,7 +39,8 @@ echo "=====================================================================" dnf groupinstall "Development Tools" -y --exclude="texlive*" dnf install -y memcached libmemcached-awesome-devel zlib-devel screen \ openssl-devel bzip2-devel libffi-devel wget make haproxy xxhash-devel \ - perl-FindBin perl-JSON perl-core liburing-devel ninja-build + perl-FindBin perl-JSON perl-core liburing-devel ninja-build clang \ + libev libev-devel echo "System dependencies installed successfully" @@ -67,21 +68,14 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.29.1_x86_64.whl -wget "https://files.pythonhosted.org/packages/eb/d5/e437271aea182e33db32e0990703b4e0d7025e4fba67829c5fd65dba926a/cassandra_driver-3.29.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" -# Removed Cython download as it's not needed # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django_statsd_mozilla-0.4.0-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-x86_64.whl -wget "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1-cp36-cp36m-manylinux1_x86_64.whl -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl @@ -89,8 +83,7 @@ wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2d # statsd # wget "https://files.pythonhosted.org/packages/2c/a8/714954464435178017e8d2f39ff418e0c9ad4411a416d4acc315298b9cea/statsd-2.1.2.tar.gz" wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -217,45 +210,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Install python3.10 -if ! [ -d Python-3.10.2 ]; then - wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz - tar -xzf Python-3.10.2.tgz - cd Python-3.10.2 +# Install python3.14 +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install cd ../ fi -CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 built successfully" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -264,7 +260,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -276,11 +272,57 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 @@ -289,12 +331,35 @@ set -u export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -308,16 +373,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -510,6 +626,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -518,6 +635,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -532,6 +650,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh index 6885f2ad..b122ddf0 100755 --- a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh @@ -39,7 +39,20 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl ninja-build + perl liburing-dev ninja-build libev4 libev-dev cmake \ + software-properties-common + +# Install Clang 17 for CinderX compatibility +# CinderX requires GCC 13+ or Clang 15+ for C23 attribute support ([[clang::always_inline]]) +# Ubuntu 22.04's default Clang 14 doesn't support this, so we install Clang 17 from LLVM repos +wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc +add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" +apt update +apt install -y clang-17 + +# Set Clang 17 as the default clang +update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 +update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 echo "System dependencies installed successfully" @@ -69,30 +82,23 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.29.1_x86_64.whl -wget "https://files.pythonhosted.org/packages/eb/d5/e437271aea182e33db32e0990703b4e0d7025e4fba67829c5fd65dba926a/cassandra_driver-3.29.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" # Dulwich 0.21.2.tar.gz wget "https://files.pythonhosted.org/packages/14/a5/cf61f9209d48abf47d48086e0a0388f1030bb5f7cf2661972eee56ccee3d/dulwich-0.21.2.tar.gz" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django-statsd-mozilla-0.4.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-x86_64.whl -wget "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1.tar.gz -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd-3.3.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -217,34 +223,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Ubuntu 22 comes with python3.10, so we use system python -echo "Using system Python 3.10" +# Install python3.14 from source +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 + ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" + make install + cd ../ +fi + +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 ready" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -253,7 +273,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -265,22 +285,96 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder # Create CPython virtual env -python3.10 -m venv venv_cpython +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython # Create Cinder virtual env export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 source ./venv_cpython/bin/activate set -u +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -295,16 +389,67 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc first (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -497,6 +642,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -505,6 +651,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -519,6 +666,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/srcs/bin/run.sh b/packages/django_workload/srcs/bin/run.sh index 19770105..cffc973b 100755 --- a/packages/django_workload/srcs/bin/run.sh +++ b/packages/django_workload/srcs/bin/run.sh @@ -10,7 +10,7 @@ SCRIPT_ROOT="$(dirname "$(readlink -f "$0")")" BENCHPRESS_ROOT="$(readlink -f "${SCRIPT_ROOT}/../../..")" DJANGO_PKG_SRC_ROOT="${BENCHPRESS_ROOT}/packages/django_workload/srcs" DJANGO_SERVER_ROOT="$(readlink -f "${SCRIPT_ROOT}/../django-workload/django-workload")" -CPYTHON_PATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_PATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" CINDER_PATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build" MEMCACHED_PID= CASSANDRA_PID= @@ -142,8 +142,27 @@ cleanup() { # echo "$HAPROXY_PIDS" | xargs -r kill -9 2>/dev/null || true # fi - # Stop memcached - [ -n "$MEMCACHED_PID" ] && { echo "Stopping memcached"; kill "$MEMCACHED_PID" 2>/dev/null || true; } + # Stop memcached using PID file + if [ -f memcached.pid ]; then + MEMCACHED_PID="$(cat memcached.pid)" + echo "Stopping memcached (PID: $MEMCACHED_PID)" + kill "$MEMCACHED_PID" 2>/dev/null || true + + # Wait briefly for graceful shutdown + sleep 1 + + # Force kill if still running + if kill -0 "$MEMCACHED_PID" 2>/dev/null; then + echo "Force killing memcached (PID: $MEMCACHED_PID)" + kill -9 "$MEMCACHED_PID" 2>/dev/null || true + fi + + rm -f memcached.pid + elif [ -n "$MEMCACHED_PID" ]; then + # Fallback to variable if PID file doesn't exist + echo "Stopping memcached (PID: $MEMCACHED_PID)" + kill "$MEMCACHED_PID" 2>/dev/null || true + fi # Stop Cassandra using PID file if [ -f cassandra.pid ]; then @@ -515,15 +534,12 @@ start_django_server() { # Export FBTHRIFT_PREFIX for thrift Python bindings export FBTHRIFT_PREFIX="${SCRIPT_ROOT}/../proxygen/proxygen/_build/deps" - # Enable JIT if requested (Cinder on x86 only) - if [ "${use_jit}" -gt 0 ] && [ "${interpreter}" = "cinder" ]; then - echo "Enabling Cinder JIT..." - export PYTHONJIT=1 - export PYTHONJITWRITEPROFILE=/tmp/cinder-jit.profile - export PYTHONJITPROFILEINTERP=1 - export PYTHONJITPROFILEINTERPPERIOD=10 - #export PYTHONJITDUMPSTATS=1 - export PYTHONJITALLSTATICFUNCTIONS=1 + # Enable JIT if requested (Cinder only) + if [ "${use_jit}" -ge 0 ] && [ "${interpreter}" = "cinder" ]; then + echo "Enabling Cinder JIT (CinderX)..." + export PYTHONJITALL=1 + export PYTHONJITLOGFILE=/tmp/cinder-jit.log + export PYTHONJITDUMPSTATS=1 fi # Export FBTHRIFT_PREFIX for thrift Python bindings @@ -534,6 +550,8 @@ start_django_server() { ./django-workload/services/memcached/run-memcached > memcached.log 2>&1 & MEMCACHED_PID=$! + echo "$MEMCACHED_PID" > memcached.pid + echo "Started memcached with PID: $MEMCACHED_PID" # Start django-workload # Set the cassandra ip in django config file diff --git a/packages/django_workload/srcs/django-workload/client/run-siege b/packages/django_workload/srcs/django-workload/client/run-siege index 85c43f81..d5207400 100755 --- a/packages/django_workload/srcs/django-workload/client/run-siege +++ b/packages/django_workload/srcs/django-workload/client/run-siege @@ -8,7 +8,7 @@ import optparse import os import subprocess import re -import numpy as np +import statistics # Concurrent worker count WORKERS = 185 @@ -274,8 +274,8 @@ def parse_results(iterations, url_dict, url_target): if iterations > 1: del url_dict[url][second_idx] del url_dict[url][first_idx] - arr = np.array(url_dict[url]) - arr_mean = np.mean(arr) * 100 + arr = url_dict[url] + arr_mean = statistics.mean(arr) * 100 if arr else 0 print(padding(url, 3), end="") print(str(arr_mean) + "%, expected " + str(url_target[url]) + "%") print() @@ -287,13 +287,13 @@ def parse_results(iterations, url_dict, url_target): print(padding(metric, 5), end="") - arr = np.array(results[metric]) - arr_mean = np.mean(arr) + arr = results[metric] + arr_mean = statistics.mean(arr) if arr else -1 if arr_mean >= 0: - if arr_mean == 0: + if arr_mean == 0 or len(arr) < 2: arr_rsd = 0 else: - arr_rsd = np.std(arr) / arr_mean + arr_rsd = statistics.stdev(arr) / arr_mean print(str(arr_mean) + " " + unit_measures[metric], end="") print(" ---- RSD " + str(arr_rsd)) else: diff --git a/packages/django_workload/srcs/django-workload/client/run-wrk b/packages/django_workload/srcs/django-workload/client/run-wrk index 8cd44cd1..ba62c315 100755 --- a/packages/django_workload/srcs/django-workload/client/run-wrk +++ b/packages/django_workload/srcs/django-workload/client/run-wrk @@ -8,7 +8,7 @@ import optparse import os import subprocess import re -import numpy as np +import statistics # Concurrent worker count WORKERS = 185 @@ -482,8 +482,8 @@ def parse_results(iterations, url_dict, url_target): if iterations > 1: del base_url_dict[url][second_idx] del base_url_dict[url][first_idx] - arr = np.array(base_url_dict[url]) - arr_mean = np.mean(arr) * 100 + arr = base_url_dict[url] + arr_mean = statistics.mean(arr) * 100 if arr else 0 expected_perc = (base_url_target[url] / total_expected_weight) * 100 print(padding(url, 3), end="") print(str(arr_mean) + "%, expected " + str(expected_perc) + "%") @@ -497,13 +497,13 @@ def parse_results(iterations, url_dict, url_target): print(padding(metric, 5), end="") - arr = np.array(results[metric]) - arr_mean = np.mean(arr) + arr = results[metric] + arr_mean = statistics.mean(arr) if arr else -1 if arr_mean >= 0: - if arr_mean == 0: + if arr_mean == 0 or len(arr) < 2: arr_rsd = 0 else: - arr_rsd = np.std(arr) / arr_mean + arr_rsd = statistics.stdev(arr) / arr_mean print(str(arr_mean) + " " + unit_measures[metric], end="") print(" ---- RSD " + str(arr_rsd)) else: diff --git a/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py b/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py new file mode 100644 index 00000000..b79faa7b --- /dev/null +++ b/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py @@ -0,0 +1,53 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Fault-tolerant PyLibMCCache backend. + +Wraps Django's PyLibMCCache to catch pylibmc connection errors (e.g. ENOTCONN) +and treat them as cache misses instead of crashing the request. This is +necessary because DjangoBench V2's per-worker-port architecture creates 176 +independent pylibmc connections to a single memcached instance, and transient +connection failures are expected under high concurrency. +""" + +import logging + +from django.core.cache.backends.memcached import PyLibMCCache + +logger = logging.getLogger(__name__) + + +class FaultTolerantPyLibMCCache(PyLibMCCache): + """PyLibMCCache that treats connection errors as cache misses.""" + + def get(self, key, default=None, version=None): + try: + return super().get(key, default, version) + except Exception: + return default + + def set(self, key, value, timeout=None, version=None): + try: + return super().set(key, value, timeout, version) + except Exception: + return False + + def delete(self, key, version=None): + try: + return super().delete(key, version) + except Exception: + return False + + def get_many(self, keys, version=None): + try: + return super().get_many(keys, version) + except Exception: + return {} + + def set_many(self, mapping, timeout=None, version=None): + try: + return super().set_many(mapping, timeout, version) + except Exception: + return list(mapping.keys()) diff --git a/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py b/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py index f043a103..9dc27ed5 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py +++ b/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py @@ -140,9 +140,13 @@ STATSD_IPV6 = False # Cache configuration +# Uses FaultTolerantPyLibMCCache to handle transient pylibmc connection errors +# (ENOTCONN, CONNECTION FAILURE) gracefully as cache misses instead of crashing +# requests. This is needed because V2's per-worker-port architecture creates +# 176 independent pylibmc connections to a single memcached instance. CACHES = { "default": { - "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", + "BACKEND": "django_workload.cache_backend.FaultTolerantPyLibMCCache", "LOCATION": "127.0.0.1:11811", } } diff --git a/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py b/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py index c3d64cd9..24feb3b6 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py +++ b/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py @@ -121,6 +121,12 @@ def start_uwsgi(self) -> None: env["PROXYGEN_THREADS"] = str(self.threads_per_worker) env["PYTHONUNBUFFERED"] = "1" + # Ensure LD_LIBRARY_PATH is set for Python 3.14 shared library + # This is critical for uWSGI workers to find libpython3.14.so + if "LD_LIBRARY_PATH" in os.environ: + env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + logger.info(f"Using LD_LIBRARY_PATH: {env['LD_LIBRARY_PATH']}") + # Start uWSGI with config overrides # Use --set to override config variables (standard uWSGI way) self.uwsgi_process = subprocess.Popen( @@ -775,6 +781,12 @@ def start_worker(self, worker_id: int, port: int) -> subprocess.Popen: env = os.environ.copy() env["PYTHONUNBUFFERED"] = "1" # Disable output buffering + # Ensure LD_LIBRARY_PATH is set for Python 3.14 shared library + # This is critical for uWSGI workers to find libpython3.14.so + if "LD_LIBRARY_PATH" in os.environ: + env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + logger.info(f"Using LD_LIBRARY_PATH: {env['LD_LIBRARY_PATH']}") + # Start worker process process = subprocess.Popen( [ diff --git a/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini b/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini index 30e1ab05..f22d6387 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini +++ b/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini @@ -42,3 +42,7 @@ thunder-lock = True stats = 127.0.0.1:9191 logger = file:django-uwsgi.log disable-logging = True + +# Python 3.14 shared library path - required for built-in modules (math, _socket, etc.) +# This must point to the Cinder/CPython lib directory containing libpython3.14.so +# The actual path is set by the run script via environment variable diff --git a/packages/django_workload/srcs/django-workload/services/memcached/run-memcached b/packages/django_workload/srcs/django-workload/services/memcached/run-memcached index f5c898e1..16f690f1 100755 --- a/packages/django_workload/srcs/django-workload/services/memcached/run-memcached +++ b/packages/django_workload/srcs/django-workload/services/memcached/run-memcached @@ -15,4 +15,12 @@ PORT="${PORT:-11811}" # User to run under USER="${USER:-memcache}" -/usr/bin/memcached -u "$USER" -m "$MEMORY" -l "$LISTEN" -p "$PORT" -t 16 +# Scale threads to match uWSGI worker count (1 worker = 1 connection). +# V2 architecture uses per-worker ports, so each worker has an independent +# memcached connection. Default memcached -t 16 is insufficient when +# num_workers >> 16 (e.g., 176 on T1_BGM). +NPROC=$(nproc) +THREADS=$((NPROC > 16 ? NPROC / 2 : 16)) +MAXCONNS=$((NPROC * 8)) + +/usr/bin/memcached -u "$USER" -m "$MEMORY" -l "$LISTEN" -p "$PORT" -t "$THREADS" -c "$MAXCONNS" diff --git a/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch b/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch new file mode 100644 index 00000000..c38f01eb --- /dev/null +++ b/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch @@ -0,0 +1,99 @@ +diff --git a/setup.py b/setup.py +index 9bea257..84b3aeb 100644 +--- a/setup.py ++++ b/setup.py +@@ -1,11 +1,6 @@ +-from __future__ import print_function + import os + import sys +-from distutils.core import setup, Extension +- +-# Need an 'open' function that supports the 'encoding' argument: +-if sys.version_info[0] < 3: +- from codecs import open ++from setuptools import setup, Extension + + ## Command-line argument parsing + +@@ -95,9 +90,9 @@ if cmd == "gen-setup": + s.write(line + "\n") + sys.exit(0) + +-with open("README.rst", "U", encoding="utf-8") as r: ++with open("README.rst", encoding="utf-8") as r: + readme_text = r.read() +-with open("src/pylibmc-version.h", "U", encoding="utf-8") as r: ++with open("src/pylibmc-version.h", encoding="utf-8") as r: + version = r.read().strip().split("\"")[1] + + setup( +diff --git a/src/_pylibmcmodule.c b/src/_pylibmcmodule.c +index 5324d1d..077f6e2 100644 +--- a/src/_pylibmcmodule.c ++++ b/src/_pylibmcmodule.c +@@ -1714,7 +1714,7 @@ memcached_return pylibmc_memcached_fetch_multi(memcached_st *mc, pylibmc_mget_re + + /* Allocate as much as could possibly be needed, and an extra because of + * how libmemcached signals EOF. */ +- *req.results = PyMem_New(memcached_result_st, req.nkeys + 1); ++ *req.results = PyMem_RawNew(memcached_result_st, req.nkeys + 1); + + /* Note that nresults will not be off by one with this because the loops + * runs a half pass after the last key has been fetched, thus bumping the +@@ -1742,7 +1742,7 @@ memcached_return pylibmc_memcached_fetch_multi(memcached_st *mc, pylibmc_mget_re + memcached_result_free(*req.results + *req.nresults); + } while ((*req.nresults)--); + +- PyMem_Free(*req.results); ++ PyMem_RawDel(*req.results); + *req.results = NULL; + *req.nresults = 0; + +@@ -1942,7 +1942,7 @@ memory_cleanup: + for (i = 0; i < nresults && results != NULL; i++) { + memcached_result_free(results + i); + } +- PyMem_Free(results); ++ PyMem_RawDel(results); + } + + /* Not INCREFing because the only two outcomes are NULL and a new dict. +@@ -2167,7 +2167,7 @@ static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, + char *key; + + for (b = PylibMC_behaviors; b->name != NULL; b++) { +- if (!PyMapping_HasKeyString(behaviors, b->name)) { ++ if (behaviors == Py_None || !PyMapping_HasKeyString(behaviors, b->name)) { + continue; + } else if ((py_v = PyMapping_GetItemString(behaviors, b->name)) == NULL) { + goto error; +@@ -2197,7 +2197,7 @@ static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, + } + + for (b = PylibMC_callbacks; b->name != NULL; b++) { +- if (!PyMapping_HasKeyString(behaviors, b->name)) { ++ if (behaviors == Py_None || !PyMapping_HasKeyString(behaviors, b->name)) { + continue; + } else if ((py_v = PyMapping_GetItemString(behaviors, b->name)) == NULL) { + goto error; +diff --git a/src/_pylibmcmodule.h b/src/_pylibmcmodule.h +index cee9772..d28dec6 100644 +--- a/src/_pylibmcmodule.h ++++ b/src/_pylibmcmodule.h +@@ -52,6 +52,16 @@ + typedef ssize_t Py_ssize_t; + #endif + ++/* Python 3.4+ PyMem_New/PyMem_Free require the GIL. Use Raw variants for ++ * allocations in GIL-free sections (e.g. inside Py_BEGIN_ALLOW_THREADS). */ ++#if PY_VERSION_HEX >= 0x03040000 ++# define PyMem_RawNew(type, nelem) ((type *)PyMem_RawMalloc((nelem)*sizeof(type))) ++# define PyMem_RawDel(ptr) PyMem_RawFree(ptr) ++#else ++# define PyMem_RawNew PyMem_New ++# define PyMem_RawDel(ptr) PyMem_Del(ptr) ++#endif ++ + /* Server types. */ + #define PYLIBMC_SERVER_TCP (1 << 0) + #define PYLIBMC_SERVER_UDP (1 << 1) diff --git a/packages/django_workload/srcs/proxygen_binding/pyproject.toml b/packages/django_workload/srcs/proxygen_binding/pyproject.toml new file mode 100644 index 00000000..0650a122 --- /dev/null +++ b/packages/django_workload/srcs/proxygen_binding/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "pybind11>=2.6.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "proxygen_binding" +version = "0.1.0" +description = "Python bindings for Proxygen HTTP server" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} +authors = [ + {name = "Meta Platforms, Inc."} +] +dependencies = [ + "pybind11>=2.6.0", +] + +[project.optional-dependencies] +django = ["django>=3.0"] diff --git a/packages/django_workload/srcs/proxygen_binding/setup.py b/packages/django_workload/srcs/proxygen_binding/setup.py index 54f843d9..61f2c43f 100644 --- a/packages/django_workload/srcs/proxygen_binding/setup.py +++ b/packages/django_workload/srcs/proxygen_binding/setup.py @@ -235,6 +235,7 @@ def get_proxygen_paths(custom_args): "django_asgi_adapter", "example_server", "django_server", + "event_loop_manager", ], cmdclass={"build_ext": build_ext}, zip_safe=False,