From bf0f299cce516d0011e49e4fcf5c64d2ce533bb1 Mon Sep 17 00:00:00 2001 From: wsu Date: Mon, 30 Mar 2026 16:31:08 -0700 Subject: [PATCH 1/2] Upgrade to Python 3.14, Cinder 3.14, and add CinderX JIT Summary: Upgrade DjangoBench V2 to use Python 3.14 and Cinder 3.14 to take advantage of newer language features and performance improvements. This also adds CinderX for JIT compilation support in the Cinder runtime environment. Key changes: - CPython upgraded from 3.10 to 3.14 (built from source on all platforms) - Cinder upgraded from cinder/3.10 branch to meta/3.14 at pinned commit - CinderX installed for JIT functionality with validation - Removed numpy dependency in favor of Python's built-in statistics module - cassandra-driver now builds from PyPI source for Python 3.14 compatibility Differential Revision: D91604990 --- benchpress/cli/commands/run.py | 4 + benchpress/config/benchmarks.yml | 5 +- .../install_django_workload_aarch64.sh | 193 +++++++++--- ...nstall_django_workload_aarch64_ubuntu22.sh | 199 ++++++++++-- .../install_django_workload_x86_64_centos9.sh | 199 +++++++++--- ...install_django_workload_x86_64_ubuntu22.sh | 198 ++++++++++-- packages/django_workload/srcs/bin/run.sh | 88 +++--- .../srcs/django-workload/client/run-siege | 14 +- .../srcs/django-workload/client/run-wrk | 14 +- .../django_workload/cache_backend.py | 288 ++++++++++++++++++ .../django_workload/settings.py | 5 +- .../start_loadbalanced_server.py | 12 + .../django-workload/django-workload/uwsgi.ini | 4 + .../services/memcached/run-memcached | 10 +- .../patches/pylibmc-1.6.1-py314-compat.patch | 99 ++++++ .../srcs/proxygen_binding/pyproject.toml | 20 ++ .../srcs/proxygen_binding/setup.py | 1 + 17 files changed, 1165 insertions(+), 188 deletions(-) create mode 100644 packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py create mode 100644 packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch create mode 100644 packages/django_workload/srcs/proxygen_binding/pyproject.toml diff --git a/benchpress/cli/commands/run.py b/benchpress/cli/commands/run.py index 41787581..a733da35 100644 --- a/benchpress/cli/commands/run.py +++ b/benchpress/cli/commands/run.py @@ -227,6 +227,10 @@ def run(self, args, jobs) -> None: metrics_dir = os.path.join( get_artifacts_dir(), f"benchmark_metrics_{job.uuid}" ) + # Export metrics dir so benchmark scripts (e.g., run.sh) can write + # breakdown.csv directly there, avoiding race conditions when + # multiple runs share the same benchmarks/ directory. + os.environ["BENCHPRESS_METRICS_DIR"] = metrics_dir now = datetime.now() date = now.strftime("%Y%m%d_%H%M") symlink = job.name + "_timestamp:" + date + "_" + job.uuid diff --git a/benchpress/config/benchmarks.yml b/benchpress/config/benchmarks.yml index fd1d7430..35da75ea 100644 --- a/benchpress/config/benchmarks.yml +++ b/benchpress/config/benchmarks.yml @@ -21,6 +21,7 @@ oss_performance_mediawiki: django_workload: parser: django_workload + check_returncode: false install_script: ./packages/django_workload/install_django_workload.sh cleanup_script: ./packages/django_workload/cleanup_django_workload.sh install_markers: @@ -28,8 +29,8 @@ django_workload: - ./benchmarks/django_workload/proxygen_binding/proxygen_binding*.so - ./benchmarks/django_workload/django-workload/django-workload/django_workload/thrift/build/gen-py3/mock_services/MockAdsService-remote - ./benchmarks/django_workload/django-workload/django-workload/django_workload/feed_timeline.py - - ./benchmarks/django_workload/django-workload/django-workload/cinder/cinder-build/bin/python3.10 - - ./benchmarks/django_workload/django-workload/django-workload/Python-3.10.2/python-build/bin/python3.10 + - ./benchmarks/django_workload/django-workload/django-workload/cinder/cinder-build/bin/python3.14 + - ./benchmarks/django_workload/django-workload/django-workload/Python-3.14.2/python-build/bin/python3.14 - ./benchmarks/django_workload/wrk/wrk - ./benchmarks/django_workload/apache-cassandra/bin/cassandra path: ./benchmarks/django_workload/bin/run.sh diff --git a/packages/django_workload/install_django_workload_aarch64.sh b/packages/django_workload/install_django_workload_aarch64.sh index fc64ca5b..8306189c 100755 --- a/packages/django_workload/install_django_workload_aarch64.sh +++ b/packages/django_workload/install_django_workload_aarch64.sh @@ -40,7 +40,7 @@ dnf groupinstall "Development Tools" -y --exclude="texlive*" dnf install -y memcached libmemcached-awesome-devel zlib-devel screen \ openssl-devel bzip2-devel libffi-devel wget make xz-devel haproxy \ xxhash-devel perl-FindBin perl-JSON perl-core liburing-devel \ - ninja-build + ninja-build clang libev libev-devel echo "System dependencies installed successfully" @@ -70,29 +70,21 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra_driver-3.29.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -wget "https://files.pythonhosted.org/packages/cc/60/f8de88175937481be98da88eb88b4fd704093e284e5907775293c496df32/cassandra_driver-3.29.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" -# Removed Cython download as it's not needed # Django-5.2.tar.gz wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django_statsd_mozilla-0.4.0-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -wget "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1-cp36-cp36m-manylinux1_x86_64.whl -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -219,46 +211,49 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Install python3.10 -if ! [ -d Python-3.10.2 ]; then - wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz - tar -xzf Python-3.10.2.tgz - cd Python-3.10.2 +# Install python3.14 +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" - make -j"${NUM_BUILD_JOBS}" + make -j"${NUM_BUILD_JOBS}" PROFILE_TASK="-m test --pgo --ignore test_generators" make install cd ../ fi -CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 built successfully" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch — includes gh-145615 mimalloc page leak fix +CINDER_COMMIT="7627b90844073b94ef60415ed1f1248837bedef7" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" - make -j"${NUM_BUILD_JOBS}" + make -j"${NUM_BUILD_JOBS}" PROFILE_TASK="-m test --pgo --ignore test_generators" make install popd fi @@ -266,7 +261,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -278,11 +273,57 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 @@ -291,12 +332,37 @@ set -u export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" +pip3.14 install pymemcache +echo "pymemcache installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -311,16 +377,69 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" +pip3.14 install pymemcache +echo "pymemcache installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="6e0472215c9917c2b5c9777dd4702b5d84eb3111" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -513,6 +632,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -521,6 +641,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -535,6 +656,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh index a66334c9..1faa1533 100755 --- a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh @@ -39,7 +39,7 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl ninja-build + perl ninja-build clang libev4 libev-dev echo "System dependencies installed successfully" @@ -69,31 +69,23 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.26_aarch64.whl -wget "https://files.pythonhosted.org/packages/b5/5e/54c58c98a4eeea12a2fee7220e7ac9e8b021ea5c3d84c84adb9106c4ed43/cassandra_driver-3.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" -# Removed Cython download as it's not needed # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" # Dulwich 0.21.2.tar.gz wget "https://files.pythonhosted.org/packages/14/a5/cf61f9209d48abf47d48086e0a0388f1030bb5f7cf2661972eee56ccee3d/dulwich-0.21.2.tar.gz" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django-statsd-mozilla-0.4.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.22.1-aarch64.whl -wget "https://files.pythonhosted.org/packages/d6/ec/a8b5f1b6d00bc4fd1bc91043d5dfb029536ec5c7769588d3f4c982240008/numpy-1.22.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1.tar.gz -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd-3.3.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -223,38 +215,48 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Ubuntu 22 comes with python3.10, so we use system python -echo "Using system Python 3.10" +# Install python3.14 from source +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 + ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" + make install + cd ../ +fi -# Create directory and symlink so the install marker check passes -mkdir -p "${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/bin" -ln -sf /usr/bin/python3.10 "${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/bin/python3.10" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 ready" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" make -j"${NUM_BUILD_JOBS}" make install popd @@ -263,7 +265,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -275,22 +277,98 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder # Create CPython virtual env -python3.10 -m venv venv_cpython +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython # Create Cinder virtual env export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 source ./venv_cpython/bin/activate set -u +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" +pip3.14 install pymemcache +echo "pymemcache installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -305,16 +383,69 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" +pip3.14 install pymemcache +echo "pymemcache installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -507,6 +638,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -515,6 +647,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -529,6 +662,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_x86_64_centos9.sh b/packages/django_workload/install_django_workload_x86_64_centos9.sh index dbb64348..bc634277 100755 --- a/packages/django_workload/install_django_workload_x86_64_centos9.sh +++ b/packages/django_workload/install_django_workload_x86_64_centos9.sh @@ -38,8 +38,9 @@ echo "=====================================================================" dnf groupinstall "Development Tools" -y --exclude="texlive*" dnf install -y memcached libmemcached-awesome-devel zlib-devel screen \ - openssl-devel bzip2-devel libffi-devel wget make haproxy xxhash-devel \ - perl-FindBin perl-JSON perl-core liburing-devel ninja-build + openssl-devel bzip2-devel libffi-devel wget make xz-devel haproxy \ + xxhash-devel perl-FindBin perl-JSON perl-core liburing-devel \ + ninja-build clang libev libev-devel echo "System dependencies installed successfully" @@ -67,21 +68,14 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.29.1_x86_64.whl -wget "https://files.pythonhosted.org/packages/eb/d5/e437271aea182e33db32e0990703b4e0d7025e4fba67829c5fd65dba926a/cassandra_driver-3.29.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" -# Removed Cython download as it's not needed # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django_statsd_mozilla-0.4.0-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-x86_64.whl -wget "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1-cp36-cp36m-manylinux1_x86_64.whl -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl @@ -89,8 +83,7 @@ wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2d # statsd # wget "https://files.pythonhosted.org/packages/2c/a8/714954464435178017e8d2f39ff418e0c9ad4411a416d4acc315298b9cea/statsd-2.1.2.tar.gz" wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -217,46 +210,52 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Install python3.10 -if ! [ -d Python-3.10.2 ]; then - wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz - tar -xzf Python-3.10.2.tgz - cd Python-3.10.2 +# Install python3.14 +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" - make -j"${NUM_BUILD_JOBS}" + # Skip test_generators during PGO profiling — test_raise_and_yield_from + # fails on some x86 microarchitectures (Cooper Lake, Turin) due to + # signal handling timing differences. The test works fine at runtime. + make -j"${NUM_BUILD_JOBS}" PROFILE_TASK="-m test --pgo --ignore test_generators" make install cd ../ fi -CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -echo "CPython 3.10 built successfully" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch — includes gh-145615 mimalloc page leak fix +CINDER_COMMIT="7627b90844073b94ef60415ed1f1248837bedef7" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" - make -j"${NUM_BUILD_JOBS}" + make -j"${NUM_BUILD_JOBS}" PROFILE_TASK="-m test --pgo --ignore test_generators" make install popd fi @@ -264,7 +263,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -276,11 +275,57 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 @@ -289,12 +334,37 @@ set -u export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" -export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" +pip3.14 install pymemcache +echo "pymemcache installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -308,16 +378,69 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" +pip3.14 install pymemcache +echo "pymemcache installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="6e0472215c9917c2b5c9777dd4702b5d84eb3111" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -510,6 +633,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -518,6 +642,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -532,6 +657,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh index 700da8a3..caff5a19 100755 --- a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh @@ -39,7 +39,7 @@ echo "=====================================================================" apt install -y memcached libmemcached-dev zlib1g-dev screen \ python3 python3.10-dev python3.10-venv rpm libffi-dev \ libssl-dev libcrypt-dev haproxy libxxhash-dev \ - perl ninja-build + perl liburing-dev ninja-build clang libev4 libev-dev echo "System dependencies installed successfully" @@ -69,30 +69,23 @@ else alias wget='wget --no-clobber' fi pushd "${DJANGO_WORKLOAD_DEPS}" -# cassandra-driver-3.29.1_x86_64.whl -wget "https://files.pythonhosted.org/packages/eb/d5/e437271aea182e33db32e0990703b4e0d7025e4fba67829c5fd65dba926a/cassandra_driver-3.29.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # Django-5.2.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl" # Dulwich 0.21.2.tar.gz wget "https://files.pythonhosted.org/packages/14/a5/cf61f9209d48abf47d48086e0a0388f1030bb5f7cf2661972eee56ccee3d/dulwich-0.21.2.tar.gz" -# django-cassandra-engine-1.6.2.tar.gz -wget "https://files.pythonhosted.org/packages/1f/5e/438eb7f2d8b8e240701b721a43cb5a20cf970c8e9da8b3770df1de6d7c5b/django-cassandra-engine-1.6.2.tar.gz" +# Note: django-cassandra-engine is installed directly from PyPI (not pre-downloaded) to avoid poetry build issues # django-statsd-mozilla-0.4.3-py3-none-any.whl wget "https://files.pythonhosted.org/packages/ac/54/5fa99753dab7ced46129a4c95c777596a2e4094a8b0f65c8764d60d5cff4/django_statsd_mozilla-0.4.0-py3-none-any.whl" -# numpy-1.26.4-x86_64.whl -wget "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" # psutil-5.8.0.tar.gz wget "https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz" -# pylibmc-1.6.1.tar.gz -wget "https://files.pythonhosted.org/packages/a7/0c/f7a3af34b05c167a69ed1fc330b06b658dac4ab25b8632c52d1022dd5337/pylibmc-1.6.1.tar.gz" +# pylibmc - installed from GitHub source for Python 3.14 compatibility (see Step 6.3) # pytz-2021.1-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" # six-1.16.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" # statsd-3.3.0-py2.py3-none-any.whl wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc6c55c46a939f6e4250639134222d99eb/statsd-3.3.0-py2.py3-none-any.whl" -# uwsgi-2.0.22.tar.gz -wget "https://files.pythonhosted.org/packages/a7/4e/c4d5559b3504bb65175a759392b03cac04b8771e9a9b14811adf1151f02f/uwsgi-2.0.22.tar.gz" +# Note: uwsgi is installed directly from PyPI (not pre-downloaded) to ensure proper linking # geomet-0.2.1.post1-py3-none-any.whl wget "https://files.pythonhosted.org/packages/c9/81/156ca48f950f833ddc392f8e3677ca50a18cb9d5db38ccb4ecea55a9303f/geomet-0.2.1.post1-py3-none-any.whl" # click-7.1.2.tar.gz @@ -220,40 +213,54 @@ popd echo "JDK and Cassandra installed successfully" # ===================================================================== -# Step 4: Build CPython 3.10 +# Step 4: Build CPython 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 4: Building CPython 3.10" +echo "Step 4: Building CPython 3.14" echo "=====================================================================" pushd "${DJANGO_SERVER_ROOT}" -# Ubuntu 22 comes with python3.10, so we use system python -echo "Using system Python 3.10" +# Install python3.14 from source +if ! [ -d Python-3.14.2 ]; then + wget https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz + tar -xzf Python-3.14.2.tgz + cd Python-3.14.2 + ./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" PROFILE_TASK="-m test --pgo --ignore test_generators" + make install + cd ../ +fi + +CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" # Create directory and symlink so the install marker check passes mkdir -p "${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/bin" ln -sf /usr/bin/python3.10 "${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/bin/python3.10" -echo "CPython 3.10 ready" +echo "CPython 3.14 built successfully" # ===================================================================== -# Step 5: Build Cinder 3.10 +# Step 5: Build Cinder 3.14 # ===================================================================== echo "" echo "=====================================================================" -echo "Step 5: Building Cinder 3.10" +echo "Step 5: Building Cinder 3.14" echo "=====================================================================" # Download and build Cinder +# Using meta/3.14 branch at a known good commit for reproducibility +CINDER_COMMIT="04f91c3659d8d2dfe4331a47548316289d2fa3f0" if ! [ -d "cinder" ]; then - git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git + git clone https://github.com/facebookincubator/cinder.git pushd cinder + git checkout "${CINDER_COMMIT}" mkdir -p cinder-build - ./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s" - make -j"${NUM_BUILD_JOBS}" + ./configure --prefix="$(pwd)/cinder-build" --enable-profiling --enable-optimizations --enable-shared LN="ln -s" + make -j"${NUM_BUILD_JOBS}" PROFILE_TASK="-m test --pgo --ignore test_generators" make install popd fi @@ -261,7 +268,7 @@ fi CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" -echo "Cinder 3.10 built successfully" +echo "Cinder 3.14 built successfully" # ===================================================================== # Step 6: Install Python dependencies in virtual environments @@ -273,22 +280,98 @@ echo "=====================================================================" # Create virtual environments for both CPython and Cinder # Create CPython virtual env -python3.10 -m venv venv_cpython +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.14" -m venv venv_cpython # Create Cinder virtual env export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" [ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder +# ===================================================================== +# Step 6.3: Clone pylibmc from GitHub (Python 3.14 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.3: Cloning pylibmc from GitHub source" +echo "=====================================================================" + +# Clone pylibmc 1.6.1 from GitHub and patch for Python 3.14 compatibility +# pylibmc trunk has bugs in the refactored _fetch_multi function that cause +# memcached connection failures after ~2300 requests. Using stable 1.6.1 +# with a minimal setup.py patch (distutils→setuptools, remove "U" mode) instead. +PYLIBMC_DIR="${DJANGO_SERVER_ROOT}/pylibmc" +if ! [ -d "${PYLIBMC_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/lericson/pylibmc.git + cd "${PYLIBMC_DIR}" + git checkout 1.6.1 + # Apply Python 3.14 compatibility patch (setup.py: distutils removed in 3.12) + patch -p1 < "${BENCHPRESS_ROOT}/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch" + echo "Applied pylibmc 1.6.1 Python 3.14 compatibility patch" +fi + +# ===================================================================== +# Step 6.4: Clone cassandra-python-driver from GitHub (setuptools v82 compatible) +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.4: Cloning cassandra-python-driver from GitHub source" +echo "=====================================================================" + +# Clone cassandra-python-driver from GitHub +# PyPI version uses deprecated ez_setup which is incompatible with setuptools v82+ +# Latest trunk removed ez_setup for compatibility +CASSANDRA_DRIVER_COMMIT="7d8015e3c1cff543a5f64c70cff3e14216e58037" +CASSANDRA_DRIVER_DIR="${DJANGO_SERVER_ROOT}/cassandra-python-driver" +if ! [ -d "${CASSANDRA_DRIVER_DIR}" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/apache/cassandra-python-driver.git + cd cassandra-python-driver + git checkout "${CASSANDRA_DRIVER_COMMIT}" +fi + +# Return to DJANGO_SERVER_ROOT before activating virtual environments +cd "${DJANGO_SERVER_ROOT}" + # Install packages in CPython environment set +u # shellcheck disable=SC1091 source ./venv_cpython/bin/activate set -u +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export CPATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.14.2/Include" +# Set LIBRARY_PATH and LDFLAGS to ensure packages link against CPython's libpython +export LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CPYTHON_INSTALL_PREFIX}/lib -Wl,-rpath,${CPYTHON_INSTALL_PREFIX}/lib" + +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in CPython venv" +pip3.14 install pymemcache +echo "pymemcache installed in CPython venv" + +# Install uwsgi directly from PyPI to ensure proper linking with CPython's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in CPython venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in CPython venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in CPython venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in CPython venv" @@ -303,16 +386,69 @@ pushd "${DJANGO_SERVER_ROOT}" export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include" export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +# Critical: Set LIBRARY_PATH and LDFLAGS to ensure uwsgi links against Cinder's libpython +# Without these, the linker may find CPython's libpython3.14.so instead +export LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" +export LDFLAGS="-L${CINDER_INSTALL_PREFIX}/lib -Wl,-rpath,${CINDER_INSTALL_PREFIX}/lib" source ./venv_cinder/bin/activate set -u +# Install pylibmc (required by django-workload) +cd "${PYLIBMC_DIR}" +pip3.14 install -e . +echo "pylibmc installed in Cinder venv" +pip3.14 install pymemcache +echo "pymemcache installed in Cinder venv" + +# Install uwsgi directly from PyPI to ensure proper linking with Cinder's libpython +# Use --no-cache-dir to prevent reusing a wheel built for a different Python environment +pip3.14 install --no-cache-dir uwsgi +echo "uwsgi installed in Cinder venv" + +# Install django-cassandra-engine directly from PyPI to avoid poetry build issues +pip3.14 install django-cassandra-engine +echo "django-cassandra-engine installed in Cinder venv" + # Install dependencies using third_party pip dependencies -pip3 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" -pip3 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" +# Install cassandra-driver from GitHub source (setuptools v82+ compatible) +# Disable Cython extension to avoid build issues +cd "${CASSANDRA_DRIVER_DIR}" +CASS_DRIVER_NO_CYTHON=1 pip3.14 install -e . +echo "cassandra-driver installed in Cinder venv" +cd "${DJANGO_SERVER_ROOT}" +pip3.14 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}" echo "Dependencies installed in Cinder venv" +# ===================================================================== +# Step 6.5: Install CinderX for JIT Support +# ===================================================================== +echo "" +echo "=====================================================================" +echo "Step 6.5: Installing CinderX for JIT Support" +echo "=====================================================================" + +# Clone and install CinderX for JIT functionality +CINDERX_COMMIT="497b2b671a8084345a8288cfbd9995acfab9dfbf" +if ! [ -d "${DJANGO_SERVER_ROOT}/cinderx" ]; then + cd "${DJANGO_SERVER_ROOT}" + git clone https://github.com/facebookincubator/cinderx.git + cd cinderx + git checkout "${CINDERX_COMMIT}" +fi + +# Install CinderX in Cinder virtual environment (already activated) +cd "${DJANGO_SERVER_ROOT}/cinderx" +python -m pip install -e . + +# Validate CinderX installation +echo "Validating CinderX installation..." +python -c "import cinderx; cinderx.init(); assert cinderx.is_initialized(), 'CinderX failed to initialize'; print('CinderX initialized successfully')" + +echo "CinderX installed and validated successfully" + deactivate popd # ${DJANGO_SERVER_ROOT} @@ -505,6 +641,7 @@ export PROXYGEN_INSTALL_DIR="${DJANGO_WORKLOAD_ROOT}/proxygen/staging" # Build and install in venv_cpython echo "" echo "Installing proxygen_binding in venv_cpython..." +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install -e . @@ -513,6 +650,7 @@ echo "proxygen_binding installed in venv_cpython" # Build and install in venv_cinder echo "" echo "Installing proxygen_binding in venv_cinder..." +export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib" cd "${DJANGO_WORKLOAD_ROOT}/proxygen_binding" "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install pybind11 "${DJANGO_SERVER_ROOT}/venv_cinder/bin/python" -m pip install -e . @@ -527,6 +665,8 @@ echo "Step 9: Generating Code Variants for FeedFlow" echo "=====================================================================" # Install jinja2 in venv_cpython +# Set LD_LIBRARY_PATH for CPython shared library +export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib" echo "Installing jinja2 for code generation..." "${DJANGO_SERVER_ROOT}/venv_cpython/bin/python" -m pip install jinja2 diff --git a/packages/django_workload/srcs/bin/run.sh b/packages/django_workload/srcs/bin/run.sh index 19770105..5dbd4451 100755 --- a/packages/django_workload/srcs/bin/run.sh +++ b/packages/django_workload/srcs/bin/run.sh @@ -10,7 +10,7 @@ SCRIPT_ROOT="$(dirname "$(readlink -f "$0")")" BENCHPRESS_ROOT="$(readlink -f "${SCRIPT_ROOT}/../../..")" DJANGO_PKG_SRC_ROOT="${BENCHPRESS_ROOT}/packages/django_workload/srcs" DJANGO_SERVER_ROOT="$(readlink -f "${SCRIPT_ROOT}/../django-workload/django-workload")" -CPYTHON_PATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build" +CPYTHON_PATH="${DJANGO_SERVER_ROOT}/Python-3.14.2/python-build" CINDER_PATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build" MEMCACHED_PID= CASSANDRA_PID= @@ -142,8 +142,27 @@ cleanup() { # echo "$HAPROXY_PIDS" | xargs -r kill -9 2>/dev/null || true # fi - # Stop memcached - [ -n "$MEMCACHED_PID" ] && { echo "Stopping memcached"; kill "$MEMCACHED_PID" 2>/dev/null || true; } + # Stop memcached using PID file + if [ -f memcached.pid ]; then + MEMCACHED_PID="$(cat memcached.pid)" + echo "Stopping memcached (PID: $MEMCACHED_PID)" + kill "$MEMCACHED_PID" 2>/dev/null || true + + # Wait briefly for graceful shutdown + sleep 1 + + # Force kill if still running + if kill -0 "$MEMCACHED_PID" 2>/dev/null; then + echo "Force killing memcached (PID: $MEMCACHED_PID)" + kill -9 "$MEMCACHED_PID" 2>/dev/null || true + fi + + rm -f memcached.pid + elif [ -n "$MEMCACHED_PID" ]; then + # Fallback to variable if PID file doesn't exist + echo "Stopping memcached (PID: $MEMCACHED_PID)" + kill "$MEMCACHED_PID" 2>/dev/null || true + fi # Stop Cassandra using PID file if [ -f cassandra.pid ]; then @@ -185,36 +204,18 @@ check_port_available() { local port=$1 local port_name=$2 - # Check for LISTENING sockets only (TIME_WAIT won't appear here) - # With SO_REUSEADDR enabled, TIME_WAIT sockets won't prevent binding - if ss -tan | grep -q ":${port} "; then - echo "ERROR: Port ${port} (${port_name}) has an active LISTENING socket!" + # Only check for LISTENING sockets — use ss -tln (not ss -tan which + # also matches TIME_WAIT/CLOSE_WAIT sockets that don't block binding). + # With SO_REUSEADDR, TIME_WAIT sockets won't prevent binding. + local pid=$(ss -tlnp | grep ":${port} " | grep -oP 'pid=\K[0-9]+' | head -1) - # Check if there's a process associated with this port - local pid=$(ss -tlnp | grep ":${port} " | grep -oP 'pid=\K[0-9]+' | head -1) - - if [ -n "$pid" ]; then - echo "ERROR: Process with PID ${pid} is actively listening on port ${port}" - echo "Process details:" - ps -p "$pid" -o pid,cmd 2>/dev/null || echo " (Process information unavailable)" - echo "" - echo "To fix this issue:" - echo " 1. Kill the process: kill ${pid}" - echo " 2. Or choose a different port using -P (base port) or -T (stats port) options" - else - echo "WARNING: Port ${port} appears to be listening but no process ID found." - echo "This is unusual and may indicate a kernel-level issue." - echo "" - echo "To fix this issue:" - echo " 1. Try running the benchmark anyway (SO_REUSEADDR may allow binding)" - echo " 2. Or choose a different port using -P (base port) or -T (stats port) options" - echo " 3. Or check for zombie processes: ps aux | grep defunct" - fi + if [ -n "$pid" ]; then + echo "ERROR: Port ${port} (${port_name}) has an active LISTENING socket!" + echo "ERROR: Process with PID ${pid} is actively listening on port ${port}" + ps -p "$pid" -o pid,cmd 2>/dev/null || true return 1 fi - # With SO_REUSEADDR enabled, we don't need to check for TIME_WAIT sockets - # The kernel will allow us to bind even if old connections are in TIME_WAIT return 0 } @@ -515,15 +516,12 @@ start_django_server() { # Export FBTHRIFT_PREFIX for thrift Python bindings export FBTHRIFT_PREFIX="${SCRIPT_ROOT}/../proxygen/proxygen/_build/deps" - # Enable JIT if requested (Cinder on x86 only) - if [ "${use_jit}" -gt 0 ] && [ "${interpreter}" = "cinder" ]; then - echo "Enabling Cinder JIT..." - export PYTHONJIT=1 - export PYTHONJITWRITEPROFILE=/tmp/cinder-jit.profile - export PYTHONJITPROFILEINTERP=1 - export PYTHONJITPROFILEINTERPPERIOD=10 - #export PYTHONJITDUMPSTATS=1 - export PYTHONJITALLSTATICFUNCTIONS=1 + # Enable JIT if requested (Cinder only) + if [ "${use_jit}" -ge 1 ] && [ "${interpreter}" = "cinder" ]; then + echo "Enabling Cinder JIT (CinderX)..." + export PYTHONJITALL=1 + export PYTHONJITLOGFILE=/tmp/cinder-jit.log + export PYTHONJITDUMPSTATS=1 fi # Export FBTHRIFT_PREFIX for thrift Python bindings @@ -534,6 +532,8 @@ start_django_server() { ./django-workload/services/memcached/run-memcached > memcached.log 2>&1 & MEMCACHED_PID=$! + echo "$MEMCACHED_PID" > memcached.pid + echo "Started memcached with PID: $MEMCACHED_PID" # Start django-workload # Set the cassandra ip in django config file @@ -750,6 +750,18 @@ start_clientserver() { echo "Interpreter: ${interpreter}" log_postprocessing_end "$BREAKDOWN_FOLDER" "$$" + + # Copy breakdown.csv directly to the metrics directory to avoid race + # conditions when multiple runs share the same benchmarks/ directory. + # BENCHPRESS_METRICS_DIR is set by benchpress before launching run.sh. + if [ -n "${BENCHPRESS_METRICS_DIR:-}" ]; then + local breakdown_src="${BREAKDOWN_FOLDER}/breakdown.csv" + if [ -f "$breakdown_src" ]; then + mkdir -p "$BENCHPRESS_METRICS_DIR" + cp "$breakdown_src" "$BENCHPRESS_METRICS_DIR/" + echo "Copied breakdown.csv to ${BENCHPRESS_METRICS_DIR}/" + fi + fi } main() { diff --git a/packages/django_workload/srcs/django-workload/client/run-siege b/packages/django_workload/srcs/django-workload/client/run-siege index 85c43f81..d5207400 100755 --- a/packages/django_workload/srcs/django-workload/client/run-siege +++ b/packages/django_workload/srcs/django-workload/client/run-siege @@ -8,7 +8,7 @@ import optparse import os import subprocess import re -import numpy as np +import statistics # Concurrent worker count WORKERS = 185 @@ -274,8 +274,8 @@ def parse_results(iterations, url_dict, url_target): if iterations > 1: del url_dict[url][second_idx] del url_dict[url][first_idx] - arr = np.array(url_dict[url]) - arr_mean = np.mean(arr) * 100 + arr = url_dict[url] + arr_mean = statistics.mean(arr) * 100 if arr else 0 print(padding(url, 3), end="") print(str(arr_mean) + "%, expected " + str(url_target[url]) + "%") print() @@ -287,13 +287,13 @@ def parse_results(iterations, url_dict, url_target): print(padding(metric, 5), end="") - arr = np.array(results[metric]) - arr_mean = np.mean(arr) + arr = results[metric] + arr_mean = statistics.mean(arr) if arr else -1 if arr_mean >= 0: - if arr_mean == 0: + if arr_mean == 0 or len(arr) < 2: arr_rsd = 0 else: - arr_rsd = np.std(arr) / arr_mean + arr_rsd = statistics.stdev(arr) / arr_mean print(str(arr_mean) + " " + unit_measures[metric], end="") print(" ---- RSD " + str(arr_rsd)) else: diff --git a/packages/django_workload/srcs/django-workload/client/run-wrk b/packages/django_workload/srcs/django-workload/client/run-wrk index 8cd44cd1..ba62c315 100755 --- a/packages/django_workload/srcs/django-workload/client/run-wrk +++ b/packages/django_workload/srcs/django-workload/client/run-wrk @@ -8,7 +8,7 @@ import optparse import os import subprocess import re -import numpy as np +import statistics # Concurrent worker count WORKERS = 185 @@ -482,8 +482,8 @@ def parse_results(iterations, url_dict, url_target): if iterations > 1: del base_url_dict[url][second_idx] del base_url_dict[url][first_idx] - arr = np.array(base_url_dict[url]) - arr_mean = np.mean(arr) * 100 + arr = base_url_dict[url] + arr_mean = statistics.mean(arr) * 100 if arr else 0 expected_perc = (base_url_target[url] / total_expected_weight) * 100 print(padding(url, 3), end="") print(str(arr_mean) + "%, expected " + str(expected_perc) + "%") @@ -497,13 +497,13 @@ def parse_results(iterations, url_dict, url_target): print(padding(metric, 5), end="") - arr = np.array(results[metric]) - arr_mean = np.mean(arr) + arr = results[metric] + arr_mean = statistics.mean(arr) if arr else -1 if arr_mean >= 0: - if arr_mean == 0: + if arr_mean == 0 or len(arr) < 2: arr_rsd = 0 else: - arr_rsd = np.std(arr) / arr_mean + arr_rsd = statistics.stdev(arr) / arr_mean print(str(arr_mean) + " " + unit_measures[metric], end="") print(" ---- RSD " + str(arr_rsd)) else: diff --git a/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py b/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py new file mode 100644 index 00000000..47098cac --- /dev/null +++ b/packages/django_workload/srcs/django-workload/django-workload/django_workload/cache_backend.py @@ -0,0 +1,288 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Thread-safe, fault-tolerant memcached cache backend for Django using pylibmc. + +pylibmc.Client is NOT thread-safe — concurrent access from multiple threads +corrupts the memcached protocol stream (interleaved request/response bytes), +causing ~50% ConnectionError/SocketCreateError/Failure rates. + +Django's CacheHandler creates per-thread cache backend instances, so +instance-level locks don't provide cross-thread protection. This backend +uses module-level (global) state: one lock and one pylibmc.Client per +process, shared across all FaultTolerantPyLibMCCache instances. This +ensures only one thread accesses memcached at a time while keeping +memory usage minimal (one client per process). + +Errors are swallowed and treated as cache misses to prevent cascading +failures in DjangoBench's multi-threaded Proxygen architecture. +""" + +from __future__ import annotations + +import logging +import os +import re +import threading +import time + +import pylibmc +from django.core.cache.backends.base import BaseCache, DEFAULT_TIMEOUT + +logger = logging.getLogger(__name__) + +# How often (seconds) each worker logs its cache stats summary +_STATS_LOG_INTERVAL = 30 + +# Module-level globals shared across ALL cache backend instances in a process. +# Django's CacheHandler creates per-thread instances, but we need ONE lock +# and ONE client per process to prevent pylibmc protocol corruption. +_global_lock = threading.Lock() +_global_client = None +_global_client_pid = None + + +def _get_client(servers, options): + """Get or create the process-wide pylibmc client. + + Must be called while holding _global_lock. + """ + global _global_client, _global_client_pid + pid = os.getpid() + if _global_client_pid != pid: + _global_client = pylibmc.Client(servers, **options) + _global_client_pid = pid + return _global_client + + +class _CacheStats: + """Per-process cache operation counters with periodic logging.""" + + def __init__(self): + self._lock = threading.Lock() + self._pid = os.getpid() + self._reset_counters() + + def _reset_counters(self): + self._ops = 0 + self._errors = 0 + self._error_types = {} # exception class name -> count + self._last_log = time.monotonic() + + def record_success(self): + with self._lock: + self._maybe_reset_pid() + self._ops += 1 + self._maybe_log() + + def record_error(self, exc): + with self._lock: + self._maybe_reset_pid() + self._ops += 1 + self._errors += 1 + name = type(exc).__name__ + self._error_types[name] = self._error_types.get(name, 0) + 1 + self._maybe_log() + + def _maybe_reset_pid(self): + pid = os.getpid() + if self._pid != pid: + self._pid = pid + self._reset_counters() + + def _maybe_log(self): + now = time.monotonic() + if now - self._last_log >= _STATS_LOG_INTERVAL: + elapsed = now - self._last_log + ops, errors = self._ops, self._errors + error_types = dict(self._error_types) + self._ops = 0 + self._errors = 0 + self._error_types = {} + self._last_log = now + rate = ops / elapsed if elapsed > 0 else 0 + if errors > 0: + logger.warning( + "cache stats [pid=%d] %.0fs: %d ops (%.1f/s), " + "%d errors (%.1f%%) — %s", + self._pid, + elapsed, + ops, + rate, + errors, + 100.0 * errors / ops if ops else 0, + ", ".join(f"{k}={v}" for k, v in error_types.items()), + ) + else: + logger.info( + "cache stats [pid=%d] %.0fs: %d ops (%.1f/s), 0 errors", + self._pid, + elapsed, + ops, + rate, + ) + + +_stats = _CacheStats() + + +class FaultTolerantPyLibMCCache(BaseCache): + """pylibmc backend that swallows errors and treats them as cache misses. + + Uses module-level lock and client shared across all instances in a + process. Django creates per-thread instances, but we bypass that by + using global state, ensuring one pylibmc.Client per process. + """ + + def __init__(self, server, params): + super().__init__(params) + if isinstance(server, str): + self._servers = [s.removeprefix("unix:") for s in re.split("[;,]", server)] + else: + self._servers = [s.removeprefix("unix:") for s in server] + self._options = params.get("OPTIONS") or {} + + @property + def _cache(self): + return _get_client(self._servers, self._options) + + def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): + if timeout == DEFAULT_TIMEOUT: + timeout = self.default_timeout + if timeout is None: + return 0 + elif int(timeout) == 0: + timeout = -1 + if timeout > 2592000: + import time + + timeout += int(time.time()) + return int(timeout) + + def make_and_validate_key(self, key, version=None): + key = self.make_key(key, version=version) + self.validate_key(key) + return key + + def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): + key = self.make_and_validate_key(key, version=version) + try: + with _global_lock: + result = self._cache.add(key, value, self.get_backend_timeout(timeout)) + _stats.record_success() + return result + except Exception as e: + _stats.record_error(e) + return False + + def get(self, key, default=None, version=None): + key = self.make_and_validate_key(key, version=version) + try: + with _global_lock: + val = self._cache.get(key) + _stats.record_success() + except Exception as e: + _stats.record_error(e) + val = None + return val if val is not None else default + + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): + key = self.make_and_validate_key(key, version=version) + try: + with _global_lock: + if not self._cache.set(key, value, self.get_backend_timeout(timeout)): + self._cache.delete(key) + _stats.record_success() + except Exception as e: + _stats.record_error(e) + + def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None): + key = self.make_and_validate_key(key, version=version) + try: + with _global_lock: + if timeout == 0: + result = self._cache.delete(key) + else: + result = self._cache.touch(key, self.get_backend_timeout(timeout)) + _stats.record_success() + return result + except Exception as e: + _stats.record_error(e) + return False + + def delete(self, key, version=None): + key = self.make_and_validate_key(key, version=version) + try: + with _global_lock: + result = bool(self._cache.delete(key)) + _stats.record_success() + return result + except Exception as e: + _stats.record_error(e) + return False + + def get_many(self, keys, version=None): + key_map = { + self.make_and_validate_key(key, version=version): key for key in keys + } + try: + with _global_lock: + ret = self._cache.get_multi(key_map.keys()) + _stats.record_success() + return {key_map[k]: v for k, v in ret.items()} + except Exception as e: + _stats.record_error(e) + return {} + + def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): + safe_data = {} + original_keys = {} + for key, value in data.items(): + safe_key = self.make_and_validate_key(key, version=version) + safe_data[safe_key] = value + original_keys[safe_key] = key + try: + with _global_lock: + failed_keys = self._cache.set_multi( + safe_data, self.get_backend_timeout(timeout) + ) + _stats.record_success() + return [original_keys[k] for k in failed_keys] + except Exception as e: + _stats.record_error(e) + return list(data.keys()) + + def delete_many(self, keys, version=None): + keys = [self.make_and_validate_key(key, version=version) for key in keys] + try: + with _global_lock: + self._cache.delete_multi(keys) + _stats.record_success() + except Exception as e: + _stats.record_error(e) + + def incr(self, key, delta=1, version=None): + key = self.make_and_validate_key(key, version=version) + try: + with _global_lock: + if delta < 0: + val = self._cache.decr(key, -delta) + else: + val = self._cache.incr(key, delta) + except Exception: + val = None + if val is None: + raise ValueError("Key '%s' not found" % key) + return val + + def clear(self): + try: + with _global_lock: + self._cache.flush_all() + except Exception: + pass + + def close(self, **kwargs): + pass diff --git a/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py b/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py index f043a103..0b5dffa5 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py +++ b/packages/django_workload/srcs/django-workload/django-workload/django_workload/settings.py @@ -140,9 +140,12 @@ STATSD_IPV6 = False # Cache configuration +# Uses ThreadSafePyLibMCCache — one shared pylibmc.Client per worker process, +# created lazily post-fork via os.getpid(), with a threading.Lock for ASGI +# thread safety. This avoids both per-thread OOM and pre-fork connection issues. CACHES = { "default": { - "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", + "BACKEND": "django_workload.cache_backend.FaultTolerantPyLibMCCache", "LOCATION": "127.0.0.1:11811", } } diff --git a/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py b/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py index c3d64cd9..24feb3b6 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py +++ b/packages/django_workload/srcs/django-workload/django-workload/start_loadbalanced_server.py @@ -121,6 +121,12 @@ def start_uwsgi(self) -> None: env["PROXYGEN_THREADS"] = str(self.threads_per_worker) env["PYTHONUNBUFFERED"] = "1" + # Ensure LD_LIBRARY_PATH is set for Python 3.14 shared library + # This is critical for uWSGI workers to find libpython3.14.so + if "LD_LIBRARY_PATH" in os.environ: + env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + logger.info(f"Using LD_LIBRARY_PATH: {env['LD_LIBRARY_PATH']}") + # Start uWSGI with config overrides # Use --set to override config variables (standard uWSGI way) self.uwsgi_process = subprocess.Popen( @@ -775,6 +781,12 @@ def start_worker(self, worker_id: int, port: int) -> subprocess.Popen: env = os.environ.copy() env["PYTHONUNBUFFERED"] = "1" # Disable output buffering + # Ensure LD_LIBRARY_PATH is set for Python 3.14 shared library + # This is critical for uWSGI workers to find libpython3.14.so + if "LD_LIBRARY_PATH" in os.environ: + env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + logger.info(f"Using LD_LIBRARY_PATH: {env['LD_LIBRARY_PATH']}") + # Start worker process process = subprocess.Popen( [ diff --git a/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini b/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini index 30e1ab05..f22d6387 100644 --- a/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini +++ b/packages/django_workload/srcs/django-workload/django-workload/uwsgi.ini @@ -42,3 +42,7 @@ thunder-lock = True stats = 127.0.0.1:9191 logger = file:django-uwsgi.log disable-logging = True + +# Python 3.14 shared library path - required for built-in modules (math, _socket, etc.) +# This must point to the Cinder/CPython lib directory containing libpython3.14.so +# The actual path is set by the run script via environment variable diff --git a/packages/django_workload/srcs/django-workload/services/memcached/run-memcached b/packages/django_workload/srcs/django-workload/services/memcached/run-memcached index f5c898e1..16f690f1 100755 --- a/packages/django_workload/srcs/django-workload/services/memcached/run-memcached +++ b/packages/django_workload/srcs/django-workload/services/memcached/run-memcached @@ -15,4 +15,12 @@ PORT="${PORT:-11811}" # User to run under USER="${USER:-memcache}" -/usr/bin/memcached -u "$USER" -m "$MEMORY" -l "$LISTEN" -p "$PORT" -t 16 +# Scale threads to match uWSGI worker count (1 worker = 1 connection). +# V2 architecture uses per-worker ports, so each worker has an independent +# memcached connection. Default memcached -t 16 is insufficient when +# num_workers >> 16 (e.g., 176 on T1_BGM). +NPROC=$(nproc) +THREADS=$((NPROC > 16 ? NPROC / 2 : 16)) +MAXCONNS=$((NPROC * 8)) + +/usr/bin/memcached -u "$USER" -m "$MEMORY" -l "$LISTEN" -p "$PORT" -t "$THREADS" -c "$MAXCONNS" diff --git a/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch b/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch new file mode 100644 index 00000000..c38f01eb --- /dev/null +++ b/packages/django_workload/srcs/patches/pylibmc-1.6.1-py314-compat.patch @@ -0,0 +1,99 @@ +diff --git a/setup.py b/setup.py +index 9bea257..84b3aeb 100644 +--- a/setup.py ++++ b/setup.py +@@ -1,11 +1,6 @@ +-from __future__ import print_function + import os + import sys +-from distutils.core import setup, Extension +- +-# Need an 'open' function that supports the 'encoding' argument: +-if sys.version_info[0] < 3: +- from codecs import open ++from setuptools import setup, Extension + + ## Command-line argument parsing + +@@ -95,9 +90,9 @@ if cmd == "gen-setup": + s.write(line + "\n") + sys.exit(0) + +-with open("README.rst", "U", encoding="utf-8") as r: ++with open("README.rst", encoding="utf-8") as r: + readme_text = r.read() +-with open("src/pylibmc-version.h", "U", encoding="utf-8") as r: ++with open("src/pylibmc-version.h", encoding="utf-8") as r: + version = r.read().strip().split("\"")[1] + + setup( +diff --git a/src/_pylibmcmodule.c b/src/_pylibmcmodule.c +index 5324d1d..077f6e2 100644 +--- a/src/_pylibmcmodule.c ++++ b/src/_pylibmcmodule.c +@@ -1714,7 +1714,7 @@ memcached_return pylibmc_memcached_fetch_multi(memcached_st *mc, pylibmc_mget_re + + /* Allocate as much as could possibly be needed, and an extra because of + * how libmemcached signals EOF. */ +- *req.results = PyMem_New(memcached_result_st, req.nkeys + 1); ++ *req.results = PyMem_RawNew(memcached_result_st, req.nkeys + 1); + + /* Note that nresults will not be off by one with this because the loops + * runs a half pass after the last key has been fetched, thus bumping the +@@ -1742,7 +1742,7 @@ memcached_return pylibmc_memcached_fetch_multi(memcached_st *mc, pylibmc_mget_re + memcached_result_free(*req.results + *req.nresults); + } while ((*req.nresults)--); + +- PyMem_Free(*req.results); ++ PyMem_RawDel(*req.results); + *req.results = NULL; + *req.nresults = 0; + +@@ -1942,7 +1942,7 @@ memory_cleanup: + for (i = 0; i < nresults && results != NULL; i++) { + memcached_result_free(results + i); + } +- PyMem_Free(results); ++ PyMem_RawDel(results); + } + + /* Not INCREFing because the only two outcomes are NULL and a new dict. +@@ -2167,7 +2167,7 @@ static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, + char *key; + + for (b = PylibMC_behaviors; b->name != NULL; b++) { +- if (!PyMapping_HasKeyString(behaviors, b->name)) { ++ if (behaviors == Py_None || !PyMapping_HasKeyString(behaviors, b->name)) { + continue; + } else if ((py_v = PyMapping_GetItemString(behaviors, b->name)) == NULL) { + goto error; +@@ -2197,7 +2197,7 @@ static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, + } + + for (b = PylibMC_callbacks; b->name != NULL; b++) { +- if (!PyMapping_HasKeyString(behaviors, b->name)) { ++ if (behaviors == Py_None || !PyMapping_HasKeyString(behaviors, b->name)) { + continue; + } else if ((py_v = PyMapping_GetItemString(behaviors, b->name)) == NULL) { + goto error; +diff --git a/src/_pylibmcmodule.h b/src/_pylibmcmodule.h +index cee9772..d28dec6 100644 +--- a/src/_pylibmcmodule.h ++++ b/src/_pylibmcmodule.h +@@ -52,6 +52,16 @@ + typedef ssize_t Py_ssize_t; + #endif + ++/* Python 3.4+ PyMem_New/PyMem_Free require the GIL. Use Raw variants for ++ * allocations in GIL-free sections (e.g. inside Py_BEGIN_ALLOW_THREADS). */ ++#if PY_VERSION_HEX >= 0x03040000 ++# define PyMem_RawNew(type, nelem) ((type *)PyMem_RawMalloc((nelem)*sizeof(type))) ++# define PyMem_RawDel(ptr) PyMem_RawFree(ptr) ++#else ++# define PyMem_RawNew PyMem_New ++# define PyMem_RawDel(ptr) PyMem_Del(ptr) ++#endif ++ + /* Server types. */ + #define PYLIBMC_SERVER_TCP (1 << 0) + #define PYLIBMC_SERVER_UDP (1 << 1) diff --git a/packages/django_workload/srcs/proxygen_binding/pyproject.toml b/packages/django_workload/srcs/proxygen_binding/pyproject.toml new file mode 100644 index 00000000..0650a122 --- /dev/null +++ b/packages/django_workload/srcs/proxygen_binding/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "pybind11>=2.6.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "proxygen_binding" +version = "0.1.0" +description = "Python bindings for Proxygen HTTP server" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} +authors = [ + {name = "Meta Platforms, Inc."} +] +dependencies = [ + "pybind11>=2.6.0", +] + +[project.optional-dependencies] +django = ["django>=3.0"] diff --git a/packages/django_workload/srcs/proxygen_binding/setup.py b/packages/django_workload/srcs/proxygen_binding/setup.py index 54f843d9..61f2c43f 100644 --- a/packages/django_workload/srcs/proxygen_binding/setup.py +++ b/packages/django_workload/srcs/proxygen_binding/setup.py @@ -235,6 +235,7 @@ def get_proxygen_paths(custom_args): "django_asgi_adapter", "example_server", "django_server", + "event_loop_manager", ], cmdclass={"build_ext": build_ext}, zip_safe=False, From 37e74141b5a0257cfae8ad146e45c4d9b54aa979 Mon Sep 17 00:00:00 2001 From: Wei Su Date: Tue, 31 Mar 2026 10:32:28 -0700 Subject: [PATCH 2/2] Fix uArch metrics sampling for relative-seconds timestamp formats Summary: PerfPub failed to correctly sample uArch time-series CSVs (amd-perf-collector, amd-zen4/zen5-perf-collector, nv-perf-collector, arm-perf-collector, topdown-intel) when using breakdown.csv for timestamp alignment. These CSVs use numeric relative-second timestamps (e.g. Timestamp_Secs starting from 0) rather than time-of-day strings (e.g. "04:33:45 PM"). The existing code only handled the time-of-day format, causing a silent fallback to last_secs=300 which sampled the wrong time window and produced incorrect metrics (e.g. 8 GB/s memory BW on BGM instead of >100 GB/s). Fix: - Add TIMESTAMP_COLUMN_MAP routing each CSV filename to its timestamp column name and format type (time_of_day vs relative_secs) - Compute relative-second offsets from bm_metrics["timestamp"] epoch to align breakdown.csv absolute datetimes with uArch CSV relative timestamps - Add relative_secs matching path in get_start_end_index() that finds the closest rows by numeric offset instead of time-of-day string parsing Differential Revision: D98835676 --- perfpub/utils.py | 222 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 187 insertions(+), 35 deletions(-) diff --git a/perfpub/utils.py b/perfpub/utils.py index e37eafd6..b5e1c9c9 100644 --- a/perfpub/utils.py +++ b/perfpub/utils.py @@ -14,6 +14,25 @@ default_last_secs = 300 default_skip_last_secs = 0 +# Maps CSV filenames to (timestamp_column_name, timestamp_format) +# "time_of_day": absolute time strings like "04:33:45 PM" (mpstat, memstat, etc.) +# "relative_secs": numeric seconds relative to benchmark start (uArch collectors) +# "epoch_secs": absolute Unix epoch timestamps (Intel PerfSpect) +TIMESTAMP_COLUMN_MAP = { + "mpstat.csv": ("timestamp", "time_of_day"), + "mem-stat.csv": ("timestamp", "time_of_day"), + "cpufreq_scaling.csv": ("timestamp", "time_of_day"), + "cpufreq_cpuinfo.csv": ("timestamp", "time_of_day"), + "net-stat.csv": ("timestamp", "time_of_day"), + "perf-stat.csv": ("timestamp", "time_of_day"), + "amd-perf-collector-timeseries.csv": ("Timestamp_Secs", "relative_secs"), + "amd-zen4-perf-collector-timeseries.csv": ("Timestamp_Secs", "relative_secs"), + "amd-zen5-perf-collector-timeseries.csv": ("Timestamp_Secs", "relative_secs"), + "nv-perf-collector-timeseries.csv": ("Timestamp_Secs", "relative_secs"), + "arm-perf-collector-transposed.csv": ("time", "relative_secs"), + "topdown-intel.sys.csv": ("TS", "epoch_secs"), +} + def parse_breakdown_csv(breakdown_file, operation_name="main_benchmark"): """Parse breakdown.csv to extract operation start and end times. @@ -209,20 +228,34 @@ def get_bios_version(system_specs): def get_start_end_index( - df, interval, last_secs, skip_last_secs, start_time=None, end_time=None + df, + interval, + last_secs, + skip_last_secs, + start_time=None, + end_time=None, + ts_column=None, + ts_format=None, + start_offset_secs=None, + end_offset_secs=None, ): """Calculate start and end indices for sampling data from a dataframe. - If last_secs and skip_last_secs are None, attempts to calculate them from breakdown.csv - by finding the closest timestamps in the dataframe's timestamp column. + Supports two timestamp formats: + - "time_of_day": Absolute time strings (e.g., "04:33:45 PM") in a "timestamp" column + - "relative_secs": Numeric seconds relative to benchmark start (e.g., Timestamp_Secs) Args: - df: DataFrame with a 'timestamp' column + df: DataFrame with timestamp data interval: Metrics collection interval in seconds last_secs: Last N seconds to process, or None skip_last_secs: Last N seconds to skip, or None start_time: Start time in datetime format, or None end_time: End time in datetime format, or None + ts_column: Name of the timestamp column in the CSV + ts_format: Format type ("time_of_day" or "relative_secs") + start_offset_secs: Start offset in seconds from benchmark epoch (for relative_secs) + end_offset_secs: End offset in seconds from benchmark epoch (for relative_secs) Returns: Tuple of (start_index, end_index) @@ -230,9 +263,23 @@ def get_start_end_index( # If both parameters are None, use start and end times from breakdown.csv if last_secs is None and skip_last_secs is None: if start_time is not None and end_time is not None: - # Check if timestamp column exists + # Handle relative_secs/epoch_secs format (uArch collector CSVs) + if ( + ts_format in ("relative_secs", "epoch_secs") + and start_offset_secs is not None + and end_offset_secs is not None + and ts_column + and ts_column in df.columns + ): + ts_values = pd.to_numeric(df[ts_column], errors="coerce") + valid = ts_values.dropna() + if len(valid) > 0: + start_idx = int((valid - start_offset_secs).abs().idxmin()) + end_idx = int((valid - end_offset_secs).abs().idxmin()) + return start_idx, end_idx + + # Handle time_of_day format (mpstat, memstat, etc.) if "timestamp" in df.columns: - # Get timestamps from the current dataframe df_timestamps = [] for ts_str in df["timestamp"]: try: @@ -246,7 +293,6 @@ def get_start_end_index( continue if df_timestamps: - # Find closest timestamps in this dataframe start_idx = find_closest_timestamp_index(df_timestamps, start_time) end_idx = find_closest_timestamp_index(df_timestamps, end_time) @@ -284,13 +330,40 @@ def sample_avg_from_csv( sanitizer=None, start_time=None, end_time=None, + bm_epoch=None, ): try: df_mpstat = pd.read_csv(filename, index_col=False) except FileNotFoundError: return "" + + # Look up timestamp column info for this CSV + ts_info = TIMESTAMP_COLUMN_MAP.get(os.path.basename(filename)) + ts_column = None + ts_format = None + start_offset_secs = None + end_offset_secs = None + if ts_info: + ts_column, ts_format = ts_info + if start_time is not None and end_time is not None: + if ts_format == "relative_secs" and bm_epoch is not None: + start_offset_secs = (start_time - bm_epoch).total_seconds() + end_offset_secs = (end_time - bm_epoch).total_seconds() + elif ts_format == "epoch_secs": + start_offset_secs = start_time.timestamp() + end_offset_secs = end_time.timestamp() + start, end = get_start_end_index( - df_mpstat, interval, last_secs, skip_last_secs, start_time, end_time + df_mpstat, + interval, + last_secs, + skip_last_secs, + start_time, + end_time, + ts_column=ts_column, + ts_format=ts_format, + start_offset_secs=start_offset_secs, + end_offset_secs=end_offset_secs, ) print(f"Sampling {filename} from {start} to {end}") samples = df_mpstat.iloc[start:end] @@ -315,7 +388,9 @@ def sample_avg_from_csv( return res.to_csv(header=False) -def read_mpstat(interval, last_secs, skip_last_secs, start_time=None, end_time=None): +def read_mpstat( + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None +): metrics = [ "%gnice", "%guest", @@ -336,10 +411,13 @@ def read_mpstat(interval, last_secs, skip_last_secs, start_time=None, end_time=N metrics, start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) -def read_memstat(interval, last_secs, skip_last_secs, start_time=None, end_time=None): +def read_memstat( + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None +): return sample_avg_from_csv( "mem-stat.csv", interval, @@ -350,11 +428,12 @@ def read_memstat(interval, last_secs, skip_last_secs, start_time=None, end_time= key_suffix="_GB", start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_cpufreq_scaling( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): return sample_avg_from_csv( "cpufreq_scaling.csv", @@ -364,11 +443,12 @@ def read_cpufreq_scaling( exclude_columns=("index", "timestamp"), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_cpufreq_cpuinfo( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): return sample_avg_from_csv( "cpufreq_cpuinfo.csv", @@ -378,10 +458,13 @@ def read_cpufreq_cpuinfo( exclude_columns=("index", "timestamp"), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) -def read_netstat(interval, last_secs, skip_last_secs, start_time=None, end_time=None): +def read_netstat( + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None +): return sample_avg_from_csv( "net-stat.csv", interval, @@ -390,10 +473,13 @@ def read_netstat(interval, last_secs, skip_last_secs, start_time=None, end_time= exclude_columns=("index", "timestamp"), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) -def read_perfstat(interval, last_secs, skip_last_secs, start_time=None, end_time=None): +def read_perfstat( + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None +): return sample_avg_from_csv( "perf-stat.csv", interval, @@ -402,11 +488,12 @@ def read_perfstat(interval, last_secs, skip_last_secs, start_time=None, end_time exclude_columns=("index", "timestamp"), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_amd_perf_collector( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): def sanitize_metrics(series): if series.loc["Total Memory Read BW (MB/s)"] < 0: @@ -424,11 +511,12 @@ def sanitize_metrics(series): sanitizer=sanitize_metrics, start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_amd_zen4_perf_collector( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): return sample_avg_from_csv( "amd-zen4-perf-collector-timeseries.csv", @@ -438,11 +526,12 @@ def read_amd_zen4_perf_collector( exclude_columns=("index", "Timestamp_Secs"), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_amd_zen5_perf_collector( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): return sample_avg_from_csv( "amd-zen5-perf-collector-timeseries.csv", @@ -452,11 +541,12 @@ def read_amd_zen5_perf_collector( exclude_columns=("index", "Timestamp_Secs"), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_nv_perf_collector( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): return sample_avg_from_csv( "nv-perf-collector-timeseries.csv", @@ -466,11 +556,12 @@ def read_nv_perf_collector( exclude_columns=("index", "Timestamp_Secs"), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_arm_perf_collector( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): return sample_avg_from_csv( "arm-perf-collector-transposed.csv", @@ -480,11 +571,12 @@ def read_arm_perf_collector( exclude_columns=("time",), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) def read_intel_perfspect( - interval, last_secs, skip_last_secs, start_time=None, end_time=None + interval, last_secs, skip_last_secs, start_time=None, end_time=None, bm_epoch=None ): return sample_avg_from_csv( "topdown-intel.sys.csv", @@ -494,6 +586,7 @@ def read_intel_perfspect( exclude_columns=("TS",), start_time=start_time, end_time=end_time, + bm_epoch=bm_epoch, ) @@ -538,9 +631,8 @@ def process_metrics( return "" bm_name = bm_metrics["benchmark_name"] db_fields["benchmark_name"] = f'"{bm_name}"' - timestamp = datetime.strftime( - datetime.fromtimestamp(bm_metrics["timestamp"]), "%Y-%m-%d %H:%M:%S" - ) + bm_epoch = datetime.fromtimestamp(bm_metrics["timestamp"]) + timestamp = datetime.strftime(bm_epoch, "%Y-%m-%d %H:%M:%S") db_fields["bm_datetime"] = f'"{timestamp}"' db_fields["run_id"] = f'"{bm_metrics["run_id"]}"' @@ -582,7 +674,12 @@ def process_metrics( res += unfold_json(bm_metrics["metrics"]) # mpstat mpstat = read_mpstat( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in mpstat.splitlines(): if line: @@ -594,7 +691,12 @@ def process_metrics( # memstat memstat = read_memstat( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in memstat.splitlines(): if line: @@ -603,18 +705,33 @@ def process_metrics( res += memstat # cpufreq cpufreq_scaling = read_cpufreq_scaling( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) put_value(db_fields, "cpufreq_mhz_scaling", cpufreq_scaling) res += cpufreq_scaling cpufreq_cpuinfo = read_cpufreq_cpuinfo( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) put_value(db_fields, "cpufreq_mhz_cpuinfo", cpufreq_cpuinfo) res += cpufreq_cpuinfo # netstat netstat = read_netstat( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in netstat.splitlines(): if line: @@ -625,7 +742,12 @@ def process_metrics( res += netstat # perfstat perfstat = read_perfstat( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) res += perfstat # override cpufreq_mhz_cpuinfo if CPU_CYCLES and CNT_CYCLES exist @@ -695,7 +817,12 @@ def process_metrics( } amd_perf_collector = read_amd_perf_collector( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in amd_perf_collector.splitlines(): key, value = line.split(",") @@ -703,7 +830,12 @@ def process_metrics( db_fields[f"{Map[key]}"] = value res += amd_perf_collector amd_zen4_perf_collector = read_amd_zen4_perf_collector( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in amd_zen4_perf_collector.splitlines(): key, value = line.split(",") @@ -712,7 +844,12 @@ def process_metrics( res += amd_zen4_perf_collector amd_zen5_perf_collector = read_amd_zen5_perf_collector( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in amd_zen5_perf_collector.splitlines(): key, value = line.split(",") @@ -721,7 +858,12 @@ def process_metrics( res += amd_zen5_perf_collector nv_perf_collector = read_nv_perf_collector( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in nv_perf_collector.splitlines(): key, value = line.split(",") @@ -729,7 +871,12 @@ def process_metrics( db_fields[f"{Map[key]}"] = value res += nv_perf_collector arm_perf_collector = read_arm_perf_collector( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) for line in arm_perf_collector.splitlines(): key, value = line.split(",") @@ -737,7 +884,12 @@ def process_metrics( db_fields[f"{Map[key]}"] = value res += arm_perf_collector intel_perfspect = read_intel_perfspect( - args.interval, args.last_secs, args.skip_last_secs, start_time, end_time + args.interval, + args.last_secs, + args.skip_last_secs, + start_time, + end_time, + bm_epoch, ) res += intel_perfspect for line in intel_perfspect.splitlines():