From 3b55fb9094aaebc22ab958e5a6fce64538fd36f9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:35:20 +0100 Subject: [PATCH 01/34] Windows makefile --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 233 ++++++++++-------- 1 file changed, 137 insertions(+), 96 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 35d3d06f8..5ecc23864 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -38,44 +38,44 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") ) set( - f4_library_file_names - libavutil.so.56 - libavcodec.so.58 - libavformat.so.58 - libavdevice.so.58 - libavfilter.so.7 - libswscale.so.5 - libswresample.so.3 - ) - set( - f5_library_file_names - libavutil.so.57 - libavcodec.so.59 - libavformat.so.59 - libavdevice.so.59 - libavfilter.so.8 - libswscale.so.6 - libswresample.so.4 - ) - set( - f6_library_file_names - libavutil.so.58 - libavcodec.so.60 - libavformat.so.60 - libavdevice.so.60 - libavfilter.so.9 - libswscale.so.7 - libswresample.so.4 - ) - set( - f7_library_file_names - libavutil.so.59 - libavcodec.so.61 - libavformat.so.61 - libavdevice.so.61 - libavfilter.so.10 - libswscale.so.8 - libswresample.so.5 + f4_library_paths + ${f4_SOURCE_DIR}/lib/libavutil.so.56 + ${f4_SOURCE_DIR}/lib/libavcodec.so.58 + ${f4_SOURCE_DIR}/lib/libavformat.so.58 + ${f4_SOURCE_DIR}/lib/libavdevice.so.58 + ${f4_SOURCE_DIR}/lib/libavfilter.so.7 + ${f4_SOURCE_DIR}/lib/libswscale.so.5 + ${f4_SOURCE_DIR}/lib/libswresample.so.3 + ) + set( + f5_library_paths + ${f5_SOURCE_DIR}/lib/libavutil.so.57 + ${f5_SOURCE_DIR}/lib/libavcodec.so.59 + ${f5_SOURCE_DIR}/lib/libavformat.so.59 + ${f5_SOURCE_DIR}/lib/libavdevice.so.59 + ${f5_SOURCE_DIR}/lib/libavfilter.so.8 + ${f5_SOURCE_DIR}/lib/libswscale.so.6 + ${f5_SOURCE_DIR}/lib/libswresample.so.4 + ) + set( + f6_library_paths + ${f6_SOURCE_DIR}/lib/libavutil.so.58 + ${f6_SOURCE_DIR}/lib/libavcodec.so.60 + ${f6_SOURCE_DIR}/lib/libavformat.so.60 + ${f6_SOURCE_DIR}/lib/libavdevice.so.60 + ${f6_SOURCE_DIR}/lib/libavfilter.so.9 + ${f6_SOURCE_DIR}/lib/libswscale.so.7 + ${f6_SOURCE_DIR}/lib/libswresample.so.4 + ) + set( + f7_library_paths + ${f7_SOURCE_DIR}/lib/libavutil.so.59 + ${f7_SOURCE_DIR}/lib/libavcodec.so.61 + ${f7_SOURCE_DIR}/lib/libavformat.so.61 + ${f7_SOURCE_DIR}/lib/libavdevice.so.61 + ${f7_SOURCE_DIR}/lib/libavfilter.so.10 + ${f7_SOURCE_DIR}/lib/libswscale.so.8 + ${f7_SOURCE_DIR}/lib/libswresample.so.5 ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") set( @@ -99,44 +99,105 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a ) set( - f4_library_file_names - libavutil.56.dylib - libavcodec.58.dylib - libavformat.58.dylib - libavdevice.58.dylib - libavfilter.7.dylib - libswscale.5.dylib - libswresample.3.dylib - ) - set( - f5_library_file_names - libavutil.57.dylib - libavcodec.59.dylib - libavformat.59.dylib - libavdevice.59.dylib - libavfilter.8.dylib - libswscale.6.dylib - libswresample.4.dylib - ) - set( - f6_library_file_names - libavutil.58.dylib - libavcodec.60.dylib - libavformat.60.dylib - libavdevice.60.dylib - libavfilter.9.dylib - libswscale.7.dylib - libswresample.4.dylib - ) - set( - f7_library_file_names - libavutil.59.dylib - libavcodec.61.dylib - libavformat.61.dylib - libavdevice.61.dylib - libavfilter.10.dylib - libswscale.8.dylib - libswresample.5.dylib + f4_library_paths + ${f4_SOURCE_DIR}/lib/libavutil.56.dylib + ${f4_SOURCE_DIR}/lib/libavcodec.58.dylib + ${f4_SOURCE_DIR}/lib/libavformat.58.dylib + ${f4_SOURCE_DIR}/lib/libavdevice.58.dylib + ${f4_SOURCE_DIR}/lib/libavfilter.7.dylib + ${f4_SOURCE_DIR}/lib/libswscale.5.dylib + ${f4_SOURCE_DIR}/lib/libswresample.3.dylib + ) + set( + f5_library_paths + ${f6_SOURCE_DIR}/lib/libavutil.57.dylib + ${f6_SOURCE_DIR}/lib/libavcodec.59.dylib + ${f6_SOURCE_DIR}/lib/libavformat.59.dylib + ${f6_SOURCE_DIR}/lib/libavdevice.59.dylib + ${f6_SOURCE_DIR}/lib/libavfilter.8.dylib + ${f6_SOURCE_DIR}/lib/libswscale.6.dylib + ${f6_SOURCE_DIR}/lib/libswresample.4.dylib + ) + set( + f6_library_paths + ${f6_SOURCE_DIR}/lib/libavutil.58.dylib + ${f6_SOURCE_DIR}/lib/libavcodec.60.dylib + ${f6_SOURCE_DIR}/lib/libavformat.60.dylib + ${f6_SOURCE_DIR}/lib/libavdevice.60.dylib + ${f6_SOURCE_DIR}/lib/libavfilter.9.dylib + ${f6_SOURCE_DIR}/lib/libswscale.7.dylib + ${f6_SOURCE_DIR}/lib/libswresample.4.dylib + ) + set( + f7_library_paths + ${f7_SOURCE_DIR}/lib/libavutil.59.dylib + ${f7_SOURCE_DIR}/lib/libavcodec.61.dylib + ${f7_SOURCE_DIR}/lib/libavformat.61.dylib + ${f7_SOURCE_DIR}/lib/libavdevice.61.dylib + ${f7_SOURCE_DIR}/lib/libavfilter.10.dylib + ${f7_SOURCE_DIR}/lib/libswscale.8.dylib + ${f7_SOURCE_DIR}/lib/libswresample.5.dylib + ) +elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set( + platform_url + ${base_url}/windows_x86_64 + ) + set( + f4_sha256 + 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb + ) + set( + f5_sha256 + b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 + ) + set( + f6_sha256 + 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a + ) + set( + f7_sha256 + ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 + ) + set( + f4_library_paths + ${f4_SOURCE_DIR}/bin/avutil.lib + ${f4_SOURCE_DIR}/bin/avcodec.lib + ${f4_SOURCE_DIR}/bin/avformat.lib + ${f4_SOURCE_DIR}/bin/avdevice.lib + ${f4_SOURCE_DIR}/bin/avfilter.lib + ${f4_SOURCE_DIR}/bin/swscale.lib + ${f4_SOURCE_DIR}/bin/swresample.lib + ) + set( + f5_library_paths + ${f5_SOURCE_DIR}/bin/avutil.lib + ${f5_SOURCE_DIR}/bin/avcodec.lib + ${f5_SOURCE_DIR}/bin/avformat.lib + ${f5_SOURCE_DIR}/bin/avdevice.lib + ${f5_SOURCE_DIR}/bin/avfilter.lib + ${f5_SOURCE_DIR}/bin/swscale.lib + ${f5_SOURCE_DIR}/bin/swresample.lib + ) + set( + f6_library_paths + ${f6_SOURCE_DIR}/bin/avutil.lib + ${f6_SOURCE_DIR}/bin/avcodec.lib + ${f6_SOURCE_DIR}/bin/avformat.lib + ${f6_SOURCE_DIR}/bin/avdevice.lib + ${f6_SOURCE_DIR}/bin/avfilter.lib + ${f6_SOURCE_DIR}/bin/swscale.lib + ${f6_SOURCE_DIR}/bin/swresample.lib + ) + set( + f7_library_paths + ${f7_SOURCE_DIR}/bin/avutil.lib + ${f7_SOURCE_DIR}/bin/avcodec.lib + ${f7_SOURCE_DIR}/bin/avformat.lib + ${f7_SOURCE_DIR}/bin/avdevice.lib + ${f7_SOURCE_DIR}/bin/avfilter.lib + ${f7_SOURCE_DIR}/bin/swscale.lib + ${f7_SOURCE_DIR}/bin/swresample.lib ) else() message( @@ -183,26 +244,6 @@ target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) target_include_directories(ffmpeg7 INTERFACE ${f7_SOURCE_DIR}/include) -list( - TRANSFORM f4_library_file_names - PREPEND ${f4_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f4_library_paths -) -list( - TRANSFORM f5_library_file_names - PREPEND ${f5_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f5_library_paths -) -list( - TRANSFORM f6_library_file_names - PREPEND ${f6_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f6_library_paths -) -list( - TRANSFORM f7_library_file_names - PREPEND ${f7_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f7_library_paths -) target_link_libraries( ffmpeg4 From 3a20aceca5d26fbae4b06415096c1cda6d4b6add Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:49:58 +0100 Subject: [PATCH 02/34] Fix --- .../fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 5ecc23864..1b3ae6b4d 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -110,13 +110,13 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") ) set( f5_library_paths - ${f6_SOURCE_DIR}/lib/libavutil.57.dylib - ${f6_SOURCE_DIR}/lib/libavcodec.59.dylib - ${f6_SOURCE_DIR}/lib/libavformat.59.dylib - ${f6_SOURCE_DIR}/lib/libavdevice.59.dylib - ${f6_SOURCE_DIR}/lib/libavfilter.8.dylib - ${f6_SOURCE_DIR}/lib/libswscale.6.dylib - ${f6_SOURCE_DIR}/lib/libswresample.4.dylib + ${f5_SOURCE_DIR}/lib/libavutil.57.dylib + ${f5_SOURCE_DIR}/lib/libavcodec.59.dylib + ${f5_SOURCE_DIR}/lib/libavformat.59.dylib + ${f5_SOURCE_DIR}/lib/libavdevice.59.dylib + ${f5_SOURCE_DIR}/lib/libavfilter.8.dylib + ${f5_SOURCE_DIR}/lib/libswscale.6.dylib + ${f5_SOURCE_DIR}/lib/libswresample.4.dylib ) set( f6_library_paths From 02a998df229c3b4bc66e584cdd7e431ceb2833be Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:52:01 +0100 Subject: [PATCH 03/34] Add Windows workflow --- .github/workflows/windows_wheel.yaml | 127 +++++++++++++++++++++++++++ packaging/vc_env_helper.bat | 41 +++++++++ 2 files changed, 168 insertions(+) create mode 100644 .github/workflows/windows_wheel.yaml create mode 100644 packaging/vc_env_helper.bat diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml new file mode 100644 index 000000000..b29d2ea0f --- /dev/null +++ b/.github/workflows/windows_wheel.yaml @@ -0,0 +1,127 @@ +name: Build and test Windows wheel + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +permissions: + id-token: write + contents: write + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: disable + with-rocm: disable + with-cuda: disable + build-python-only: "disable" + + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload Windows wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + env-script: packaging/vc_env_helper.bat + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + + install-and-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_windows_cpu_x86_64 + path: pytorch/torchcodec/dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + source packaging/helpers.sh + assert_ffmpeg_not_installed + + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version + + - name: Install test dependencies + run: | + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow + + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Run Python tests + run: | + pytest test -vvv diff --git a/packaging/vc_env_helper.bat b/packaging/vc_env_helper.bat new file mode 100644 index 000000000..1f50b1b05 --- /dev/null +++ b/packaging/vc_env_helper.bat @@ -0,0 +1,41 @@ +@echo on + +set VC_VERSION_LOWER=17 +set VC_VERSION_UPPER=18 + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere +if "%VSDEVCMD_ARGS%" == "" ( + call "%VS15VCVARSALL%" x64 || exit /b 1 +) else ( + call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 +) + +@echo on + +if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + +set DISTUTILS_USE_SDK=1 + +set args=%1 +shift +:start +if [%1] == [] goto done +set args=%args% %1 +shift +goto start + +:done +if "%args%" == "" ( + echo Usage: vc_env_helper.bat [command] [args] + echo e.g. vc_env_helper.bat cl /c test.cpp +) + +%args% || exit /b 1 From 4af56adf14735428275d46d56d48033c162a8083 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:57:35 +0100 Subject: [PATCH 04/34] Fix ? --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 1b3ae6b4d..80fc9fca8 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -14,6 +14,42 @@ set( https://pytorch.s3.amazonaws.com/torchcodec/ffmpeg/2025-03-14 ) + + +FetchContent_Declare( + f4 + URL ${platform_url}/4.4.4.tar.gz + URL_HASH + SHA256=${f4_sha256} +) +FetchContent_Declare( + f5 + URL ${platform_url}/5.1.4.tar.gz + URL_HASH + SHA256=${f5_sha256} +) +FetchContent_Declare( + f6 + URL ${platform_url}/6.1.1.tar.gz + URL_HASH + SHA256=${f6_sha256} +) +FetchContent_Declare( + f7 + URL ${platform_url}/7.0.1.tar.gz + URL_HASH + SHA256=${f7_sha256} +) + +FetchContent_MakeAvailable(f4 f5 f6 f7) + +add_library(ffmpeg4 INTERFACE) +add_library(ffmpeg5 INTERFACE) +add_library(ffmpeg6 INTERFACE) +add_library(ffmpeg7 INTERFACE) + +# Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable + if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") set( platform_url @@ -205,40 +241,6 @@ else() "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" ) endif() - -FetchContent_Declare( - f4 - URL ${platform_url}/4.4.4.tar.gz - URL_HASH - SHA256=${f4_sha256} -) -FetchContent_Declare( - f5 - URL ${platform_url}/5.1.4.tar.gz - URL_HASH - SHA256=${f5_sha256} -) -FetchContent_Declare( - f6 - URL ${platform_url}/6.1.1.tar.gz - URL_HASH - SHA256=${f6_sha256} -) -FetchContent_Declare( - f7 - URL ${platform_url}/7.0.1.tar.gz - URL_HASH - SHA256=${f7_sha256} -) - -FetchContent_MakeAvailable(f4 f5 f6 f7) - -add_library(ffmpeg4 INTERFACE) -add_library(ffmpeg5 INTERFACE) -add_library(ffmpeg6 INTERFACE) -add_library(ffmpeg7 INTERFACE) - -# Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include) target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) From 95bd9b21bdf0784d1140a8b60a1304cdd45d9901 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:59:35 +0100 Subject: [PATCH 05/34] tmp --- .github/workflows/windows_wheel.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index b29d2ea0f..89e949851 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -55,8 +55,9 @@ jobs: smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + # TODO Probably needs to enable this + # build-platform: "python-build-package" + # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest From 9b3d6e4471acba573847957733a38525e81b3107 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:13:38 +0100 Subject: [PATCH 06/34] Try to use custom test-infra branch --- .github/workflows/windows_wheel.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 89e949851..a5613b43b 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -42,12 +42,12 @@ jobs: strategy: fail-fast: false name: Build and Upload Windows wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows with: repository: pytorch/torchcodec ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main + test-infra-repository: nicolashug/test-infra + test-infra-ref: build-platform-windows build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh @@ -55,9 +55,8 @@ jobs: smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} - # TODO Probably needs to enable this - # build-platform: "python-build-package" - # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest From 18f78ab0201699e65609543214cfa5a5ee985c5b Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:35:59 +0100 Subject: [PATCH 07/34] dont use env helper? --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index a5613b43b..854540b74 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -51,7 +51,7 @@ jobs: build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh - env-script: packaging/vc_env_helper.bat + # env-script: packaging/vc_env_helper.bat smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} From c814edf0fdf2bc5be8fef0caae1c9f9c9cff5e4c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:40:55 +0100 Subject: [PATCH 08/34] Hmm? --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 854540b74..1f581d5a6 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -56,7 +56,7 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + build-command: "set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest From a25882e1e8ede1bac5750f8c2fafa680a5a7af93 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:45:49 +0100 Subject: [PATCH 09/34] reorder --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 138 +++++++++--------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 80fc9fca8..81f115ff4 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -15,6 +15,77 @@ set( ) +if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + set( + platform_url + ${base_url}/linux_x86_64 + ) + + set( + f4_sha256 + 1a083f1922443bedb5243d04896383b8c606778a7ddb9d886c8303e55339fe0c + ) + set( + f5_sha256 + 65d6ad54082d94dcb3f801d73df2265e0e1bb303c7afbce7723e3b77ccd0e207 + ) + set( + f6_sha256 + 8bd5939c2f4a4b072e837e7870c13fe7d13824e5ff087ab534e4db4e90b7be9c + ) + set( + f7_sha256 + 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 + ) +elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + set( + platform_url + ${base_url}/macos_arm64 + ) + set( + f4_sha256 + f0335434529d9e19359eae0fe912dd9e747667534a1c92e662f5219a55dfad8c + ) + set( + f5_sha256 + cfc3449c9af6863731a431ce89e32c08c5f8ece94b306fb6b695828502a76166 + ) + set( + f6_sha256 + ec47b4783c342038e720e33b2fdfa55a9a490afb1cf37a26467733983688647e + ) + set( + f7_sha256 + 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a + ) + +elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set( + platform_url + ${base_url}/windows_x86_64 + ) + set( + f4_sha256 + 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb + ) + set( + f5_sha256 + b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 + ) + set( + f6_sha256 + 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a + ) + set( + f7_sha256 + ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 + ) +else() + message( + FATAL_ERROR + "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" + ) +endif() FetchContent_Declare( f4 @@ -51,28 +122,6 @@ add_library(ffmpeg7 INTERFACE) # Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") - set( - platform_url - ${base_url}/linux_x86_64 - ) - - set( - f4_sha256 - 1a083f1922443bedb5243d04896383b8c606778a7ddb9d886c8303e55339fe0c - ) - set( - f5_sha256 - 65d6ad54082d94dcb3f801d73df2265e0e1bb303c7afbce7723e3b77ccd0e207 - ) - set( - f6_sha256 - 8bd5939c2f4a4b072e837e7870c13fe7d13824e5ff087ab534e4db4e90b7be9c - ) - set( - f7_sha256 - 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 - ) - set( f4_library_paths ${f4_SOURCE_DIR}/lib/libavutil.so.56 @@ -114,26 +163,6 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") ${f7_SOURCE_DIR}/lib/libswresample.so.5 ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") - set( - platform_url - ${base_url}/macos_arm64 - ) - set( - f4_sha256 - f0335434529d9e19359eae0fe912dd9e747667534a1c92e662f5219a55dfad8c - ) - set( - f5_sha256 - cfc3449c9af6863731a431ce89e32c08c5f8ece94b306fb6b695828502a76166 - ) - set( - f6_sha256 - ec47b4783c342038e720e33b2fdfa55a9a490afb1cf37a26467733983688647e - ) - set( - f7_sha256 - 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a - ) set( f4_library_paths ${f4_SOURCE_DIR}/lib/libavutil.56.dylib @@ -175,26 +204,6 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") ${f7_SOURCE_DIR}/lib/libswresample.5.dylib ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - set( - platform_url - ${base_url}/windows_x86_64 - ) - set( - f4_sha256 - 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb - ) - set( - f5_sha256 - b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 - ) - set( - f6_sha256 - 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a - ) - set( - f7_sha256 - ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 - ) set( f4_library_paths ${f4_SOURCE_DIR}/bin/avutil.lib @@ -235,11 +244,6 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") ${f7_SOURCE_DIR}/bin/swscale.lib ${f7_SOURCE_DIR}/bin/swresample.lib ) -else() - message( - FATAL_ERROR - "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" - ) endif() target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include) target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) From b46e5ac3eba2d3f9c3a418ef8dbcbcf5979e421f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:47:58 +0100 Subject: [PATCH 10/34] Try something --- .github/workflows/windows_wheel.yaml | 5 +++-- setup.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 1f581d5a6..8297688f3 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -51,12 +51,13 @@ jobs: build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh - # env-script: packaging/vc_env_helper.bat + env-script: packaging/vc_env_helper.bat smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation" + # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + build-command: "python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest diff --git a/setup.py b/setup.py index 5074ef478..8321a8262 100644 --- a/setup.py +++ b/setup.py @@ -169,14 +169,14 @@ def copy_extensions_to_source(self): build_against_all_ffmpeg_from_s3 = ( os.getenv(BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR) is not None ) -if "bdist_wheel" in sys.argv and not ( - build_against_all_ffmpeg_from_s3 or not_a_license_violation -): - raise ValueError( - "It looks like you're trying to build a wheel. " - f"You probably want to set {BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR}. " - f"If you have a good reason *not* to, then set {NOT_A_LICENSE_VIOLATION_VAR}." - ) +# if "bdist_wheel" in sys.argv and not ( +# build_against_all_ffmpeg_from_s3 or not_a_license_violation +# ): +# raise ValueError( +# "It looks like you're trying to build a wheel. " +# f"You probably want to set {BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR}. " +# f"If you have a good reason *not* to, then set {NOT_A_LICENSE_VIOLATION_VAR}." +# ) # See `CMakeBuild.build_extension()`. fake_extension = Extension(name="FAKE_NAME", sources=[]) From c6c30cefb7df58185020e3d8de3c5bbdde16755f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:01:49 +0100 Subject: [PATCH 11/34] Comment-out some workflows --- .github/workflows/cpp_tests.yaml | 136 +++++++-------- .github/workflows/docs.yaml | 218 ++++++++++++------------ .github/workflows/lint.yaml | 142 +++++++-------- .github/workflows/linux_cuda_wheel.yaml | 162 +++++++++--------- .github/workflows/linux_wheel.yaml | 126 +++++++------- .github/workflows/macos_wheel.yaml | 130 +++++++------- 6 files changed, 457 insertions(+), 457 deletions(-) diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index e08d90754..44dd288d6 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -1,73 +1,73 @@ -name: CPP tests +# name: CPP tests -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} +# cancel-in-progress: true -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - Cpp-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v3 - with: - auto-update-conda: true - # Using miniforge instead of miniconda ensures that the default - # conda channel is conda-forge instead of main/default. This ensures - # ABI consistency between dependencies: - # https://conda-forge.org/docs/user/transitioning_from_defaults/ - miniforge-version: latest - activate-environment: test - python-version: '3.12' - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install torch dependencies - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install ffmpeg, pkg-config and pybind11 - run: | - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge - ffmpeg -version - - name: Build and run C++ tests - run: | - # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so - # we're building libtorchcodec against the installed FFmpeg version - # (from conda-forge) instead of building against our pre-built non-GPL - # FFmpeg libraries. - # The reason we need this is because the C++ tests decode x264 files. - # x264 support is not LGPL, os it is not supported by our - # pre-built non-GPL FFmpeg libraries. And if we were to build against - # those, this is also what the tests would be loading at run time, - # then failing when we try to decode x264. - # To remediate that, we build against the FFmpeg that we installed - # from conda-forge (which is able to decode x264), and that's also - # what gets loaded at run time. - # The Python tests are also decoding x264 files, and are built against - # our non-GPL FFmpeg. And yet they pass. This is because in Python - # we're able to distinguish between build-time (non-GPL FFmpeg) and - # run time (conda-forge FFmpeg). +# jobs: +# Cpp-tests: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v3 +# with: +# auto-update-conda: true +# # Using miniforge instead of miniconda ensures that the default +# # conda channel is conda-forge instead of main/default. This ensures +# # ABI consistency between dependencies: +# # https://conda-forge.org/docs/user/transitioning_from_defaults/ +# miniforge-version: latest +# activate-environment: test +# python-version: '3.12' +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install torch dependencies +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# - name: Install ffmpeg, pkg-config and pybind11 +# run: | +# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge +# ffmpeg -version +# - name: Build and run C++ tests +# run: | +# # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so +# # we're building libtorchcodec against the installed FFmpeg version +# # (from conda-forge) instead of building against our pre-built non-GPL +# # FFmpeg libraries. +# # The reason we need this is because the C++ tests decode x264 files. +# # x264 support is not LGPL, os it is not supported by our +# # pre-built non-GPL FFmpeg libraries. And if we were to build against +# # those, this is also what the tests would be loading at run time, +# # then failing when we try to decode x264. +# # To remediate that, we build against the FFmpeg that we installed +# # from conda-forge (which is able to decode x264), and that's also +# # what gets loaded at run time. +# # The Python tests are also decoding x264 files, and are built against +# # our non-GPL FFmpeg. And yet they pass. This is because in Python +# # we're able to distinguish between build-time (non-GPL FFmpeg) and +# # run time (conda-forge FFmpeg). - build_tests_dir="${PWD}/build_tests" - mkdir $build_tests_dir - pushd $build_tests_dir - TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") - Torch_DIR="${TORCH_PATH}/share/cmake/Torch" - cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON - cmake --build . - ctest --output-on-failure - popd +# build_tests_dir="${PWD}/build_tests" +# mkdir $build_tests_dir +# pushd $build_tests_dir +# TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") +# Torch_DIR="${TORCH_PATH}/share/cmake/Torch" +# cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON +# cmake --build . +# ctest --output-on-failure +# popd diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 60bfbfa2e..39aaec1a1 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,116 +1,116 @@ -name: Docs +# name: Docs -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -permissions: - id-token: write - contents: write +# permissions: +# id-token: write +# contents: write -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-cpu: disable - with-xpu: disable - with-rocm: disable - with-cuda: enable - build-python-only: "disable" - build: - needs: generate-matrix - strategy: - fail-fast: false - name: Build and Upload wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: pytorch/torchcodec - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" +# jobs: +# generate-matrix: +# uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main +# with: +# package-type: wheel +# os: linux +# test-infra-repository: pytorch/test-infra +# test-infra-ref: main +# with-cpu: disable +# with-xpu: disable +# with-rocm: disable +# with-cuda: enable +# build-python-only: "disable" +# build: +# needs: generate-matrix +# strategy: +# fail-fast: false +# name: Build and Upload wheel +# uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main +# with: +# repository: pytorch/torchcodec +# ref: "" +# test-infra-repository: pytorch/test-infra +# test-infra-ref: main +# build-matrix: ${{ needs.generate-matrix.outputs.matrix }} +# pre-script: packaging/pre_build_script.sh +# post-script: packaging/post_build_script.sh +# smoke-test-script: packaging/fake_smoke_test.py +# package-name: torchcodec +# trigger-event: ${{ github.event_name }} +# build-platform: "python-build-package" +# build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" - build-docs: - runs-on: linux.4xlarge.nvidia.gpu - strategy: - fail-fast: false - matrix: - # 3.9 corresponds to the minimum python version for which we build - # the wheel unless the label cliflow/binaries/all is present in the - # PR. - python-version: ['3.9'] - cuda-version: ['12.6'] - ffmpeg-version-for-tests: ['7'] - container: - image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ matrix.python-version }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" - - name: Check env - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv +# build-docs: +# runs-on: linux.4xlarge.nvidia.gpu +# strategy: +# fail-fast: false +# matrix: +# # 3.9 corresponds to the minimum python version for which we build +# # the wheel unless the label cliflow/binaries/all is present in the +# # PR. +# python-version: ['3.9'] +# cuda-version: ['12.6'] +# ffmpeg-version-for-tests: ['7'] +# container: +# image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" +# options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" +# needs: build +# steps: +# - name: Setup env vars +# run: | +# cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') +# echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV +# python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') +# echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV +# - uses: actions/download-artifact@v4 +# with: +# name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 +# path: pytorch/torchcodec/dist/ +# - name: Setup miniconda using test-infra +# uses: pytorch/test-infra/.github/actions/setup-miniconda@main +# with: +# python-version: ${{ matrix.python-version }} +# # We install conda packages at the start because otherwise conda may have conflicts with dependencies. +# default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" +# - name: Check env +# run: | +# ${CONDA_RUN} env +# ${CONDA_RUN} conda info +# ${CONDA_RUN} nvidia-smi +# ${CONDA_RUN} conda list +# - name: Assert ffmpeg exists +# run: | +# ${CONDA_RUN} ffmpeg -buildconf +# - name: Update pip +# run: ${CONDA_RUN} python -m pip install --upgrade pip +# - name: Install PyTorch +# run: | +# ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} +# ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' +# - name: Install torchcodec from the wheel +# run: | +# wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` +# echo Installing $wheel_path +# ${CONDA_RUN} python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 +# - name: Check out repo +# uses: actions/checkout@v3 - - name: Install doc dependencies - run: | - cd docs - ${CONDA_RUN} python -m pip install -r requirements.txt - - name: Build docs - run: | - cd docs - ${CONDA_RUN} make html - - uses: actions/upload-artifact@v4 - with: - name: Built-Docs - path: docs/build/html/ +# - name: Install doc dependencies +# run: | +# cd docs +# ${CONDA_RUN} python -m pip install -r requirements.txt +# - name: Build docs +# run: | +# cd docs +# ${CONDA_RUN} make html +# - uses: actions/upload-artifact@v4 +# with: +# name: Built-Docs +# path: docs/build/html/ diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c156a833c..6ed79d88c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,76 +1,76 @@ -name: Lint +# name: Lint -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} +# cancel-in-progress: true -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - pre-commit-checks: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install pre-commit - run: | - python -m pip install pre-commit - - name: Run pre-commit checks - run: | - pre-commit run --all-files - - name: Check to see what files pre-commit modified - run: | - git diff +# jobs: +# pre-commit-checks: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# python-version: ['3.12'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: ${{ matrix.python-version }} +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install pre-commit +# run: | +# python -m pip install pre-commit +# - name: Run pre-commit checks +# run: | +# pre-commit run --all-files +# - name: Check to see what files pre-commit modified +# run: | +# git diff - mypy: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install dependencies and FFmpeg - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge - ffmpeg -version - - name: Build and install torchcodec - run: | - python -m pip install -e ".[dev]" --no-build-isolation -vvv - - name: Install mypy - run: | - python -m pip install mypy - - name: Run mypy - run: | - mypy --install-types --non-interactive --config-file mypy.ini +# mypy: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# python-version: ['3.12'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: ${{ matrix.python-version }} +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install dependencies and FFmpeg +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge +# ffmpeg -version +# - name: Build and install torchcodec +# run: | +# python -m pip install -e ".[dev]" --no-build-isolation -vvv +# - name: Install mypy +# run: | +# python -m pip install mypy +# - name: Run mypy +# run: | +# mypy --install-types --non-interactive --config-file mypy.ini diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index bd57cac5e..ffc21ab46 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -56,88 +56,88 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" - install-and-test: - runs-on: linux.g5.4xlarge.nvidia.gpu - strategy: - fail-fast: false - matrix: - # 3.9 corresponds to the minimum python version for which we build - # the wheel unless the label cliflow/binaries/all is present in the - # PR. - # For the actual release we should add that label and change this to - # include more python versions. - python-version: ['3.9'] - cuda-version: ['12.6', '12.8'] - # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 - ffmpeg-version-for-tests: ['4.4.2', '6', '7'] + # install-and-test: + # runs-on: linux.g5.4xlarge.nvidia.gpu + # strategy: + # fail-fast: false + # matrix: + # # 3.9 corresponds to the minimum python version for which we build + # # the wheel unless the label cliflow/binaries/all is present in the + # # PR. + # # For the actual release we should add that label and change this to + # # include more python versions. + # python-version: ['3.9'] + # cuda-version: ['12.6', '12.8'] + # # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 + # ffmpeg-version-for-tests: ['4.4.2', '6', '7'] - container: - image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ matrix.python-version }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" - - name: Check env - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv + # container: + # image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" + # options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + # needs: build + # steps: + # - name: Setup env vars + # run: | + # cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') + # echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV + # python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') + # echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV + # - uses: actions/download-artifact@v4 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 + # path: pytorch/torchcodec/dist/ + # - name: Setup miniconda using test-infra + # uses: pytorch/test-infra/.github/actions/setup-miniconda@main + # with: + # python-version: ${{ matrix.python-version }} + # # We install conda packages at the start because otherwise conda may have conflicts with dependencies. + # default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" + # - name: Check env + # run: | + # ${CONDA_RUN} env + # ${CONDA_RUN} conda info + # ${CONDA_RUN} nvidia-smi + # ${CONDA_RUN} conda list + # - name: Assert ffmpeg exists + # run: | + # ${CONDA_RUN} ffmpeg -buildconf + # - name: Update pip + # run: ${CONDA_RUN} python -m pip install --upgrade pip + # - name: Install PyTorch + # run: | + # ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} + # ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` + # echo Installing $wheel_path + # ${CONDA_RUN} python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 + # - name: Check out repo + # uses: actions/checkout@v3 - - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - ${CONDA_RUN} python -m pip install numpy pytest pillow + # - name: Install test dependencies + # run: | + # # Ideally we would find a way to get those dependencies from pyproject.toml + # ${CONDA_RUN} python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests - run: | - ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short - - name: Run Python benchmark - run: | - ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls + # - name: Run Python tests + # run: | + # ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short + # - name: Run Python benchmark + # run: | + # ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 1855e904e..58086b0bc 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -57,70 +57,70 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" - install-and-test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.9'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - needs: build - steps: - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv + # install-and-test: + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: + # python-version: ['3.9'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # needs: build + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 + # path: pytorch/torchcodec/dist/ + # - name: Setup conda env + # uses: conda-incubator/setup-miniconda@v2 + # with: + # auto-update-conda: true + # miniconda-version: "latest" + # activate-environment: test + # python-version: ${{ matrix.python-version }} + # - name: Update pip + # run: python -m pip install --upgrade pip + # - name: Install PyTorch + # run: | + # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + # echo Installing $wheel_path + # python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed + # - name: Check out repo + # uses: actions/checkout@v3 + # - name: Install ffmpeg, post build + # run: | + # # Ideally we would have checked for that before installing the wheel, + # # but we need to checkout the repo to access this file, and we don't + # # want to checkout the repo before installing the wheel to avoid any + # # side-effect. It's OK. + # source packaging/helpers.sh + # assert_ffmpeg_not_installed - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version + # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + # ffmpeg -version - - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow + # - name: Install test dependencies + # run: | + # # Ideally we would find a way to get those dependencies from pyproject.toml + # python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests - run: | - pytest test -vvv + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls + # - name: Run Python tests + # run: | + # pytest test -vvv diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index ee436b7a4..ad6c00fee 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -58,68 +58,68 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" - install-and-test: - runs-on: macos-14-xlarge - strategy: - fail-fast: false - matrix: - python-version: ['3.9'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - needs: build - steps: - - name: Download wheel - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ - path: pytorch/torchcodec/dist/ - - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v3 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv - - - name: Check out torchcodec repo - uses: actions/checkout@v3 - - - name: Install ffmpeg - run: | - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version - - - name: Install test dependencies - run: | - python -m pip install numpy pytest pillow - - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls -lh - - - name: Run Python tests - run: | - pytest test -vvv + # install-and-test: + # runs-on: macos-14-xlarge + # strategy: + # fail-fast: false + # matrix: + # python-version: ['3.9'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # needs: build + # steps: + # - name: Download wheel + # uses: actions/download-artifact@v4 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ + # path: pytorch/torchcodec/dist/ + + # - name: Setup conda env + # uses: conda-incubator/setup-miniconda@v3 + # with: + # auto-update-conda: true + # miniconda-version: "latest" + # activate-environment: test + # python-version: ${{ matrix.python-version }} + # - name: Update pip + # run: python -m pip install --upgrade pip + + # - name: Install PyTorch + # run: | + # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + # echo Installing $wheel_path + # python -m pip install $wheel_path -vvv + + # - name: Check out torchcodec repo + # uses: actions/checkout@v3 + + # - name: Install ffmpeg + # run: | + # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + # ffmpeg -version + + # - name: Install test dependencies + # run: | + # python -m pip install numpy pytest pillow + + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls -lh + + # - name: Run Python tests + # run: | + # pytest test -vvv From 8930df5214d7f8a849a7803dae5b919bd3f141f1 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:11:07 +0100 Subject: [PATCH 12/34] Remove compilation flags --- src/torchcodec/_core/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 7196d0487..cce8e4a2a 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -14,7 +14,8 @@ else() set(TORCHCODEC_WERROR_OPTION "-Werror") endif() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") function(make_torchcodec_sublibrary library_name From db0d342310fc1e5274835479f7f5ac3ff0a6f4d9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:35:35 +0100 Subject: [PATCH 13/34] rework flags --- src/torchcodec/_core/CMakeLists.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index cce8e4a2a..803323b62 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -14,8 +14,19 @@ else() set(TORCHCODEC_WERROR_OPTION "-Werror") endif() +if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + # Avoid warnings about non-ASCII characters in source files. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4819") + # Important for when we add Windows CUDA: exporting all symbols is aparently + # limited to 65535 symbols, which (aparently) will not work for CUDA. + # https://github.com/pytorch/pytorch/pull/3650 + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +endif() + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +# TODO put back previous line and set proper flags for windows. e.g. Wall is W4. +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") + function(make_torchcodec_sublibrary library_name From 059c925bc2a51788e94d9809f02d6fa4d3cc707c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:54:36 +0100 Subject: [PATCH 14/34] Remove some flags --- src/torchcodec/_core/CMakeLists.txt | 30 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 803323b62..2cf44280e 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -158,19 +158,23 @@ function(make_torchcodec_libraries # stray initialization of py::objects. The rest of the object code must # match. See: # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes - target_compile_options( - ${pybind_ops_library_name} - PUBLIC - "-fvisibility=hidden" - ) - # If we don't make sure this flag is set, we run into segfauls at import - # time on Mac. See: - # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 - target_link_options( - ${pybind_ops_library_name} - PUBLIC - "LINKER:-undefined,dynamic_lookup" - ) + if(NOT WIN32) # TODO unify WIN detection + target_compile_options( + ${pybind_ops_library_name} + PUBLIC + "-fvisibility=hidden" + ) + endif() + if(APPLE) # TODO unify APPLE detection + # If we don't make sure this flag is set, we run into segfauls at import + # time on Mac. See: + # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 + target_link_options( + ${pybind_ops_library_name} + PUBLIC + "LINKER:-undefined,dynamic_lookup" + ) + endif() # Install all libraries. set( From 46d4df54910ef87d750e81f26d8fa3cfba4a1f40 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 11:30:29 +0100 Subject: [PATCH 15/34] Add RUNTIME destination for windows --- src/torchcodec/_core/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 2cf44280e..516ffa950 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -192,6 +192,7 @@ function(make_torchcodec_libraries install( TARGETS ${all_libraries} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} + RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs ) endfunction() From 4ebd78a2ac755519795598a3ccae3c3363ca27f8 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 11:47:29 +0100 Subject: [PATCH 16/34] Set output dirs to CMAKE_CURRENT_BINARY_DIR for Windows --- src/torchcodec/_core/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 516ffa950..e7394d020 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -46,6 +46,18 @@ function(make_torchcodec_sublibrary # Avoid adding the "lib" prefix which we already add explicitly. set_target_properties(${library_name} PROPERTIES PREFIX "") + # On Windows, avoid the Release/Debug subdirectories in output + if(WIN32) + set_target_properties(${library_name} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + ) + endif() + target_link_libraries( ${library_name} PUBLIC From 43c9e4f672c6a0bba6c9d9a1dd9dcd01093ffbad Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 11:59:56 +0100 Subject: [PATCH 17/34] Add comment --- src/torchcodec/_core/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index e7394d020..1a665b955 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -46,8 +46,10 @@ function(make_torchcodec_sublibrary # Avoid adding the "lib" prefix which we already add explicitly. set_target_properties(${library_name} PROPERTIES PREFIX "") - # On Windows, avoid the Release/Debug subdirectories in output - if(WIN32) + if(WIN32) # TODO unify WIN detection + # On Windows, the built artifacts are put in Release/Debug + # subdirectories by default. We want to avoid that, otherwise our + # install() step would not know where to find those. set_target_properties(${library_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} From 41b90f4a1cebb79d5d72ea30d9489ba26dd51209 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:02:01 +0100 Subject: [PATCH 18/34] Disable post-build-script for Windows --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 8297688f3..562eddfec 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -50,7 +50,7 @@ jobs: test-infra-ref: build-platform-windows build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + # post-script: packaging/post_build_script.sh # TODO enable post-build checks? env-script: packaging/vc_env_helper.bat smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec From 8c0bc380c8c496f8e66dbb64e5d4586fdd481f31 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:16:12 +0100 Subject: [PATCH 19/34] Fix expected wheel artifact name --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 562eddfec..b51de3671 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -70,7 +70,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: pytorch_torchcodec__${{ matrix.python-version }}_windows_cpu_x86_64 + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_64 path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 From cbbd34f1c1f16f01f0e09d76290b83d27dd29174 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:31:40 +0100 Subject: [PATCH 20/34] Update .github/workflows/windows_wheel.yaml --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index b51de3671..6bfaae2de 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -70,7 +70,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_64 + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 From 2ccb7ee9b831e0ca35a118aa83b7851f9a2f15f4 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:51:14 +0100 Subject: [PATCH 21/34] Update .github/workflows/windows_wheel.yaml --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 6bfaae2de..c8d8d85ff 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -60,7 +60,7 @@ jobs: build-command: "python -m build --wheel -vvv --no-isolation" install-and-test: - runs-on: ubuntu-latest + runs-on: windows-latest strategy: fail-fast: false matrix: From a06060ad9afd0ab1ebdff921d28a6b61102edf39 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 13:13:37 +0100 Subject: [PATCH 22/34] fix extension finder --- src/torchcodec/_internally_replaced_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_internally_replaced_utils.py b/src/torchcodec/_internally_replaced_utils.py index a5a3ffa39..39b04d5c1 100644 --- a/src/torchcodec/_internally_replaced_utils.py +++ b/src/torchcodec/_internally_replaced_utils.py @@ -18,9 +18,11 @@ def _get_extension_path(lib_name: str) -> str: extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES elif sys.platform == "darwin": extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dylib"] + elif sys.platform in ("win32", "cygwin"): + extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dll"] else: raise NotImplementedError( - "Platforms other than linux/darwin are not supported yet" + f"{sys.platform = } is not not supported" ) loader_details = ( importlib.machinery.ExtensionFileLoader, From 7fb6536e4f96e991c4afa87c95344148350b8b15 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:06:12 +0100 Subject: [PATCH 23/34] maybe fix something --- .github/workflows/windows_wheel.yaml | 3 +++ setup.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index c8d8d85ff..a96f4be27 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -42,6 +42,8 @@ jobs: strategy: fail-fast: false name: Build and Upload Windows wheel + # TODO + # We shouldn't use this, but this is needed for now. Note that this is where we are setting the BUILD_AGAINST_ALL_FFMPEG_FROM_S3 variable currently, we'll have to find a way around that. uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows with: repository: pytorch/torchcodec @@ -57,6 +59,7 @@ jobs: trigger-event: ${{ github.event_name }} build-platform: "python-build-package" # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + # TODO see comment above build-command: "python -m build --wheel -vvv --no-isolation" install-and-test: diff --git a/setup.py b/setup.py index 8321a8262..8a583b3b5 100644 --- a/setup.py +++ b/setup.py @@ -150,9 +150,11 @@ def copy_extensions_to_source(self): # dynamically loaded module. For more, see: # https://stackoverflow.com/a/2339910 extensions = ["dylib", "so"] + elif sys.platform in ("win32", "cygwin"): + extensions = ["dll"] else: raise NotImplementedError( - "Platforms other than linux/darwin are not supported yet" + f"Platform {sys.platform} is not supported" ) for ext in extensions: From 0be6336d0b55855f6ecef8596729e59615989336 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:17:17 +0100 Subject: [PATCH 24/34] try to add Library/bin to PATH?? --- .github/workflows/windows_wheel.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index a96f4be27..e4c0bc35e 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -107,6 +107,26 @@ jobs: conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version + # On Windows, ensure the conda Library/bin directory is in PATH + # This is needed for Windows DLL loading to find FFmpeg DLLs + if [[ "$RUNNER_OS" == "Windows" ]]; then + conda_env_path=$(conda info --base)/envs/test + library_bin_path="$conda_env_path/Library/bin" + echo "Adding conda Library/bin to PATH: $library_bin_path" + echo "$library_bin_path" >> $GITHUB_PATH + + # Verify FFmpeg DLLs are accessible + echo "Checking if FFmpeg DLLs are in PATH:" + where avutil.dll || echo "avutil.dll not found in PATH" + where avcodec.dll || echo "avcodec.dll not found in PATH" + where avformat.dll || echo "avformat.dll not found in PATH" + fi + + - name: Test torchcodec import after FFmpeg installation + run: | + echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." + python -c "import torchcodec; print('TorchCodec import successful!')" + - name: Install test dependencies run: | # Ideally we would find a way to get those dependencies from pyproject.toml From 5b9827599914a47c536b1318ccbd7c42a7c249b3 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:27:28 +0100 Subject: [PATCH 25/34] Add build and test from source workflow --- .github/workflows/windows_source_build.yaml | 103 ++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 .github/workflows/windows_source_build.yaml diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml new file mode 100644 index 000000000..13b72b2b2 --- /dev/null +++ b/.github/workflows/windows_source_build.yaml @@ -0,0 +1,103 @@ +name: Build and test Windows from source + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + build-and-test: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + ffmpeg-version: ['4.4.4', '5.1.4', '6.1.1', '7.0.1'] + steps: + - name: Check out repo + uses: actions/checkout@v3 + + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: build + python-version: ${{ matrix.python-version }} + + - name: Install FFmpeg + run: | + conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge + ffmpeg -version + + # On Windows, ensure the conda Library/bin directory is in PATH + # This is needed for both pkg-config to find FFmpeg and for DLL loading + conda_env_path=$(conda info --base)/envs/build + library_bin_path="$conda_env_path/Library/bin" + echo "Adding conda Library/bin to PATH: $library_bin_path" + echo "$library_bin_path" >> $GITHUB_PATH + + # Also add Library/lib/pkgconfig to PKG_CONFIG_PATH for cmake to find FFmpeg + pkg_config_path="$conda_env_path/Library/lib/pkgconfig" + echo "Adding to PKG_CONFIG_PATH: $pkg_config_path" + echo "PKG_CONFIG_PATH=$pkg_config_path" >> $GITHUB_ENV + + # Verify FFmpeg DLLs are accessible + echo "Checking if FFmpeg DLLs are in PATH:" + where avutil.dll || echo "avutil.dll not found in PATH" + where avcodec.dll || echo "avcodec.dll not found in PATH" + where avformat.dll || echo "avformat.dll not found in PATH" + + - name: Install build dependencies + run: | + # Install pkg-config and pybind11 which are needed for Windows builds + conda install -y pkg-config pybind11 -c conda-forge + + # Verify pkg-config can find FFmpeg + echo "Testing pkg-config for FFmpeg libraries:" + pkg-config --exists libavcodec && echo "libavcodec found" || echo "libavcodec NOT found" + pkg-config --exists libavformat && echo "libavformat found" || echo "libavformat NOT found" + pkg-config --exists libavutil && echo "libavutil found" || echo "libavutil NOT found" + + - name: Update pip and install PyTorch + run: | + python -m pip install --upgrade pip + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + + - name: Build torchcodec from source + run: | + # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg + echo "Building torchcodec from source using system FFmpeg..." + python -m pip install -e . --no-build-isolation -v + + - name: Test torchcodec import + run: | + echo "Testing torchcodec import..." + python -c "import torchcodec; print('TorchCodec import successful!')" + python -c "import torchcodec; print('FFmpeg versions:', torchcodec.get_ffmpeg_library_versions())" + + - name: Install test dependencies + run: | + python -m pip install numpy pytest pillow + + - name: Run Python tests + run: | + pytest test -v From a1ba873f50095ddbabad33776d20b968858d7470 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:33:26 +0100 Subject: [PATCH 26/34] Fix FFmpeg versions for source build --- .github/workflows/windows_source_build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 13b72b2b2..15c3ae84b 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -30,7 +30,7 @@ jobs: fail-fast: false matrix: python-version: ['3.9'] - ffmpeg-version: ['4.4.4', '5.1.4', '6.1.1', '7.0.1'] + ffmpeg-version: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] steps: - name: Check out repo uses: actions/checkout@v3 From f61971b0d4554f0791950fc9e035a28b454c2a07 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:43:11 +0100 Subject: [PATCH 27/34] Try random things --- .github/workflows/windows_source_build.yaml | 35 ++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 15c3ae84b..1061c7abf 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -45,14 +45,26 @@ jobs: - name: Install FFmpeg run: | + # Install FFmpeg and its development headers conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge + + # Try to install FFmpeg development package if available + conda install ffmpeg-dev -c conda-forge || echo "ffmpeg-dev not available, continuing..." + ffmpeg -version # On Windows, ensure the conda Library/bin directory is in PATH # This is needed for both pkg-config to find FFmpeg and for DLL loading conda_env_path=$(conda info --base)/envs/build library_bin_path="$conda_env_path/Library/bin" - echo "Adding conda Library/bin to PATH: $library_bin_path" + library_lib_path="$conda_env_path/Library/lib" + library_include_path="$conda_env_path/Library/include" + + echo "Adding conda Library paths:" + echo " bin: $library_bin_path" + echo " lib: $library_lib_path" + echo " include: $library_include_path" + echo "$library_bin_path" >> $GITHUB_PATH # Also add Library/lib/pkgconfig to PKG_CONFIG_PATH for cmake to find FFmpeg @@ -60,6 +72,11 @@ jobs: echo "Adding to PKG_CONFIG_PATH: $pkg_config_path" echo "PKG_CONFIG_PATH=$pkg_config_path" >> $GITHUB_ENV + # Set additional environment variables that CMake might need + echo "CMAKE_PREFIX_PATH=$conda_env_path/Library" >> $GITHUB_ENV + echo "LIBRARY_PATH=$library_lib_path" >> $GITHUB_ENV + echo "INCLUDE_PATH=$library_include_path" >> $GITHUB_ENV + # Verify FFmpeg DLLs are accessible echo "Checking if FFmpeg DLLs are in PATH:" where avutil.dll || echo "avutil.dll not found in PATH" @@ -71,11 +88,23 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge + # Install FFmpeg development dependencies that are needed for linking on Windows + conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge || true + + # Alternative: install common FFmpeg dependencies individually + conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge || true + # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" pkg-config --exists libavcodec && echo "libavcodec found" || echo "libavcodec NOT found" pkg-config --exists libavformat && echo "libavformat found" || echo "libavformat NOT found" pkg-config --exists libavutil && echo "libavutil found" || echo "libavutil NOT found" + + # Show what libraries pkg-config thinks we need + echo "pkg-config --libs libavcodec:" + pkg-config --libs libavcodec || true + echo "pkg-config --libs libavformat:" + pkg-config --libs libavformat || true - name: Update pip and install PyTorch run: | @@ -86,6 +115,10 @@ jobs: run: | # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg echo "Building torchcodec from source using system FFmpeg..." + + # On Windows, we may need to disable some CMake flags that cause linking issues + export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON" + python -m pip install -e . --no-build-isolation -v - name: Test torchcodec import From 2f2a38aaba2f3d70b7aa9f347a1dd37757b39407 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 10:59:53 +0100 Subject: [PATCH 28/34] Add quiet flags --- .github/workflows/windows_source_build.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 1061c7abf..295adf438 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -46,10 +46,10 @@ jobs: - name: Install FFmpeg run: | # Install FFmpeg and its development headers - conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge + conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge --quiet # Try to install FFmpeg development package if available - conda install ffmpeg-dev -c conda-forge || echo "ffmpeg-dev not available, continuing..." + conda install ffmpeg-dev -c conda-forge --quiet || echo "ffmpeg-dev not available, continuing..." ffmpeg -version @@ -86,13 +86,13 @@ jobs: - name: Install build dependencies run: | # Install pkg-config and pybind11 which are needed for Windows builds - conda install -y pkg-config pybind11 -c conda-forge + conda install -y pkg-config pybind11 -c conda-forge --quiet # Install FFmpeg development dependencies that are needed for linking on Windows - conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge || true + conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge --quiet || true # Alternative: install common FFmpeg dependencies individually - conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge || true + conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge --quiet || true # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" From c570edbeb53c41a85cc4c985536e4e596791b236 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 11:08:43 +0100 Subject: [PATCH 29/34] print dependencies of all libs --- .github/workflows/windows_source_build.yaml | 24 ++++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 295adf438..d8570e879 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -48,9 +48,6 @@ jobs: # Install FFmpeg and its development headers conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge --quiet - # Try to install FFmpeg development package if available - conda install ffmpeg-dev -c conda-forge --quiet || echo "ffmpeg-dev not available, continuing..." - ffmpeg -version # On Windows, ensure the conda Library/bin directory is in PATH @@ -88,11 +85,6 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge --quiet - # Install FFmpeg development dependencies that are needed for linking on Windows - conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge --quiet || true - - # Alternative: install common FFmpeg dependencies individually - conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge --quiet || true # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" @@ -103,9 +95,25 @@ jobs: # Show what libraries pkg-config thinks we need echo "pkg-config --libs libavcodec:" pkg-config --libs libavcodec || true + echo "pkg-config --libs libavformat:" pkg-config --libs libavformat || true + echo "pkg-config --libs libavutil:" + pkg-config --libs libavutil || true + + echo "pkg-config --libs libswscale:" + pkg-config --libs libswscale|| true + + pkg-config --libs libswresample|| true + echo "pkg-config --libs libswresample:" + + echo "pkg-config --libs libavfilter:" + pkg-config --libs libavfilter || true + + echo "pkg-config --libs libavdevice:" + pkg-config --libs libavdevice || true + - name: Update pip and install PyTorch run: | python -m pip install --upgrade pip From 821982f9254af84175d370b5df7a5025eb456362 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 11:28:19 +0100 Subject: [PATCH 30/34] Crazy stuff --- .github/workflows/windows_source_build.yaml | 40 +++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index d8570e879..14f4c9ca7 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -45,7 +45,7 @@ jobs: - name: Install FFmpeg run: | - # Install FFmpeg and its development headers + # Install FFmpeg and try to get development packages conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge --quiet ffmpeg -version @@ -73,6 +73,10 @@ jobs: echo "CMAKE_PREFIX_PATH=$conda_env_path/Library" >> $GITHUB_ENV echo "LIBRARY_PATH=$library_lib_path" >> $GITHUB_ENV echo "INCLUDE_PATH=$library_include_path" >> $GITHUB_ENV + + # Add the Library/lib directory to the linker path for Visual Studio + echo "LIB=$library_lib_path;$LIB" >> $GITHUB_ENV + echo "LIBPATH=$library_lib_path;$LIBPATH" >> $GITHUB_ENV # Verify FFmpeg DLLs are accessible echo "Checking if FFmpeg DLLs are in PATH:" @@ -85,6 +89,23 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge --quiet + # Inspect what's actually in the Library directory + echo "Contents of Library/lib:" + ls -la "$CONDA_PREFIX/Library/lib/" | head -20 + echo "Looking for harfbuzz files:" + find "$CONDA_PREFIX/Library/" -name "*harfbuzz*" 2>/dev/null || echo "No harfbuzz files found" + echo "Looking for freetype files:" + find "$CONDA_PREFIX/Library/" -name "*freetype*" 2>/dev/null || echo "No freetype files found" + + # Check if the .lib files exist with the exact names pkg-config expects + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + + echo "Checking for specific .lib files in $library_lib_path:" + ls -la "$library_lib_path/harfbuzz.lib" 2>/dev/null || echo "harfbuzz.lib not found" + ls -la "$library_lib_path/freetype.lib" 2>/dev/null || echo "freetype.lib not found" + ls -la "$library_lib_path/"*harfbuzz* 2>/dev/null || echo "No harfbuzz files in lib" + ls -la "$library_lib_path/"*freetype* 2>/dev/null || echo "No freetype files in lib" # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" @@ -124,9 +145,24 @@ jobs: # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg echo "Building torchcodec from source using system FFmpeg..." - # On Windows, we may need to disable some CMake flags that cause linking issues + # On Windows, try to work around the dependency linking issues + # by setting CMAKE_ARGS to use dynamic linking where possible export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON" + # Try to create symlinks for missing .lib files if the DLLs exist + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + library_bin_path="$conda_env_path/Library/bin" + + # Check if we can create import libraries from DLLs for missing dependencies + if [ ! -f "$library_lib_path/harfbuzz.lib" ] && [ -f "$library_bin_path/harfbuzz.dll" ]; then + echo "Attempting to create harfbuzz.lib from harfbuzz.dll" + # This is a workaround - we'll try to use lib.exe to create an import library + # lib.exe /def:harfbuzz.def /out:harfbuzz.lib /machine:x64 + # But this requires a .def file which we don't have + echo "harfbuzz.lib missing but harfbuzz.dll exists" + fi + python -m pip install -e . --no-build-isolation -v - name: Test torchcodec import From af7a98af28c2bc9df3d7fc1e609179129deff9bc Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 11:52:53 +0100 Subject: [PATCH 31/34] crazy cmake stuff --- .github/workflows/windows_source_build.yaml | 7 ++- src/torchcodec/_core/CMakeLists.txt | 60 +++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 14f4c9ca7..868dd64f8 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -147,7 +147,12 @@ jobs: # On Windows, try to work around the dependency linking issues # by setting CMAKE_ARGS to use dynamic linking where possible - export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON" + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + + export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=$library_lib_path" + + echo "CMAKE_ARGS: $CMAKE_ARGS" # Try to create symlinks for missing .lib files if the DLLs exist conda_env_path=$(conda info --base)/envs/build diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 1a665b955..54411add3 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -65,6 +65,12 @@ function(make_torchcodec_sublibrary PUBLIC ${library_dependencies} ) + + # Debug output for Windows builds + if(WIN32) + message(STATUS "Created library: ${library_name}") + message(STATUS "Library dependencies: ${library_dependencies}") + endif() endfunction() function(make_torchcodec_libraries @@ -198,6 +204,45 @@ function(make_torchcodec_libraries ${pybind_ops_library_name} ) + # The install step is invoked within CMakeBuild.build_library() in + # setup.py and just copies the built files from the temp + # cmake/setuptools build folder into the CMAKE_INSTALL_PREFIX folder. We + # still need to manually pass "DESTINATION ..." for cmake to copy those + # files in CMAKE_INSTALL_PREFIX instead of CMAKE_INSTALL_PREFIX/lib. + install( + TARGETS ${all_libraries} + LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} + RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs +ibrary to prevent + # stray initialization of py::objects. The rest of the object code must + # match. See: + # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes + if(NOT WIN32) # TODO unify WIN detection + target_compile_options( + ${pybind_ops_library_name} + PUBLIC + "-fvisibility=hidden" + ) + endif() + if(APPLE) # TODO unify APPLE detection + # If we don't make sure this flag is set, we run into segfauls at import + # time on Mac. See: + # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 + target_link_options( + ${pybind_ops_library_name} + PUBLIC + "LINKER:-undefined,dynamic_lookup" + ) + endif() + + # Install all libraries. + set( + all_libraries + ${decoder_library_name} + ${custom_ops_library_name} + ${pybind_ops_library_name} + ) + # The install step is invoked within CMakeBuild.build_library() in # setup.py and just copies the built files from the temp # cmake/setuptools build folder into the CMAKE_INSTALL_PREFIX folder. We @@ -246,6 +291,21 @@ else() libswscale ) + # On Windows, explicitly add the conda library directory to the linker search path + if(WIN32 AND DEFINED ENV{CONDA_PREFIX}) + link_directories("$ENV{CONDA_PREFIX}/Library/lib") + message(STATUS "Added conda library directory to linker search path: $ENV{CONDA_PREFIX}/Library/lib") + + # Also add the library path to CMAKE_PREFIX_PATH for better library finding + list(APPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}/Library") + message(STATUS "Added conda prefix to CMAKE_PREFIX_PATH: $ENV{CONDA_PREFIX}/Library") + + # Debug: Show what LIBAV variables are set + message(STATUS "LIBAV_INCLUDE_DIRS: ${LIBAV_INCLUDE_DIRS}") + message(STATUS "LIBAV_LIBRARY_DIRS: ${LIBAV_LIBRARY_DIRS}") + message(STATUS "LIBAV_LIBRARIES: ${LIBAV_LIBRARIES}") + endif() + # Split libavcodec's version string by '.' and convert it to a list string(REPLACE "." ";" libavcodec_version_list ${LIBAV_libavcodec_VERSION}) # Get the first element of the list, which is the major version From 1413666da05ff8bf8a94b526782b8a8a017da6b9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 12:00:23 +0100 Subject: [PATCH 32/34] ???? --- .github/workflows/windows_source_build.yaml | 76 ++++++++++++++------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 868dd64f8..d70bcb077 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -89,6 +89,10 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge --quiet + # Try to install missing dependencies that FFmpeg might need + echo "Installing additional dependencies that might be missing..." + conda install -y harfbuzz freetype zlib bzip2 -c conda-forge --quiet || echo "Some dependencies might already be installed" + # Inspect what's actually in the Library directory echo "Contents of Library/lib:" ls -la "$CONDA_PREFIX/Library/lib/" | head -20 @@ -124,10 +128,10 @@ jobs: pkg-config --libs libavutil || true echo "pkg-config --libs libswscale:" - pkg-config --libs libswscale|| true + pkg-config --libs libswscale || true - pkg-config --libs libswresample|| true echo "pkg-config --libs libswresample:" + pkg-config --libs libswresample || true echo "pkg-config --libs libavfilter:" pkg-config --libs libavfilter || true @@ -140,35 +144,59 @@ jobs: python -m pip install --upgrade pip python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Build torchcodec from source + - name: Create Windows build script run: | - # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg - echo "Building torchcodec from source using system FFmpeg..." + # Create a build script that sets up the VS environment and runs the build + cat > build_with_vs.bat << 'EOF' + @echo off - # On Windows, try to work around the dependency linking issues - # by setting CMAKE_ARGS to use dynamic linking where possible - conda_env_path=$(conda info --base)/envs/build - library_lib_path="$conda_env_path/Library/lib" + :: Get conda environment info + for /f "tokens=*" %%i in ('conda info --base') do set CONDA_BASE=%%i + set CONDA_ENV_PATH=%CONDA_BASE%\envs\build + set LIBRARY_LIB_PATH=%CONDA_ENV_PATH%\Library\lib + set LIBRARY_BIN_PATH=%CONDA_ENV_PATH%\Library\bin + set LIBRARY_INCLUDE_PATH=%CONDA_ENV_PATH%\Library\include - export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=$library_lib_path" + :: Add conda Library/bin to PATH for DLL loading + set PATH=%LIBRARY_BIN_PATH%;%PATH% - echo "CMAKE_ARGS: $CMAKE_ARGS" + :: Set PKG_CONFIG_PATH + set PKG_CONFIG_PATH=%CONDA_ENV_PATH%\Library\lib\pkgconfig - # Try to create symlinks for missing .lib files if the DLLs exist - conda_env_path=$(conda info --base)/envs/build - library_lib_path="$conda_env_path/Library/lib" - library_bin_path="$conda_env_path/Library/bin" + :: Set additional paths for the linker + set LIB=%LIBRARY_LIB_PATH%;%LIB% + set LIBPATH=%LIBRARY_LIB_PATH%;%LIBPATH% + set INCLUDE=%LIBRARY_INCLUDE_PATH%;%INCLUDE% - # Check if we can create import libraries from DLLs for missing dependencies - if [ ! -f "$library_lib_path/harfbuzz.lib" ] && [ -f "$library_bin_path/harfbuzz.dll" ]; then - echo "Attempting to create harfbuzz.lib from harfbuzz.dll" - # This is a workaround - we'll try to use lib.exe to create an import library - # lib.exe /def:harfbuzz.def /out:harfbuzz.lib /machine:x64 - # But this requires a .def file which we don't have - echo "harfbuzz.lib missing but harfbuzz.dll exists" - fi + :: Set CMAKE_ARGS with more comprehensive settings + set CMAKE_ARGS=-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=%LIBRARY_LIB_PATH% -DCMAKE_INCLUDE_PATH=%LIBRARY_INCLUDE_PATH% -DCMAKE_PREFIX_PATH=%LIBRARY_LIB_PATH%;%LIBRARY_INCLUDE_PATH% -DCMAKE_VERBOSE_MAKEFILE=ON + + echo Building with CMAKE_ARGS: %CMAKE_ARGS% + echo LIB path: %LIB% + echo PKG_CONFIG_PATH: %PKG_CONFIG_PATH% + echo PATH (first few entries): + echo %PATH% | cut -d; -f1-3 + + :: Debug: Show what libraries are available + echo Available .lib files in conda lib directory: + dir "%LIBRARY_LIB_PATH%\*.lib" 2>nul || echo No .lib files found + + :: Debug: Test pkg-config within the script + echo Testing pkg-config from within build script: + pkg-config --exists libavcodec && echo libavcodec found || echo libavcodec NOT found + + :: Use vc_env_helper.bat to set up VS environment and build + call packaging\vc_env_helper.bat python -m pip install -e . --no-build-isolation -v + EOF + + - name: Verify Visual Studio environment and build + shell: cmd + run: | + echo Checking Visual Studio installation... + call packaging\vc_env_helper.bat echo Visual Studio environment setup complete - python -m pip install -e . --no-build-isolation -v + echo Running build with Visual Studio environment... + call build_with_vs.bat - name: Test torchcodec import run: | From eb8b31824c8850fc50f718a77d83b8cbad966b8c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 12:38:15 +0100 Subject: [PATCH 33/34] ??? --- .github/workflows/windows_source_build.yaml | 85 +++++++++------------ 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index d70bcb077..04c359251 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -144,59 +144,42 @@ jobs: python -m pip install --upgrade pip python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Create Windows build script - run: | - # Create a build script that sets up the VS environment and runs the build - cat > build_with_vs.bat << 'EOF' - @echo off - - :: Get conda environment info - for /f "tokens=*" %%i in ('conda info --base') do set CONDA_BASE=%%i - set CONDA_ENV_PATH=%CONDA_BASE%\envs\build - set LIBRARY_LIB_PATH=%CONDA_ENV_PATH%\Library\lib - set LIBRARY_BIN_PATH=%CONDA_ENV_PATH%\Library\bin - set LIBRARY_INCLUDE_PATH=%CONDA_ENV_PATH%\Library\include - - :: Add conda Library/bin to PATH for DLL loading - set PATH=%LIBRARY_BIN_PATH%;%PATH% - - :: Set PKG_CONFIG_PATH - set PKG_CONFIG_PATH=%CONDA_ENV_PATH%\Library\lib\pkgconfig - - :: Set additional paths for the linker - set LIB=%LIBRARY_LIB_PATH%;%LIB% - set LIBPATH=%LIBRARY_LIB_PATH%;%LIBPATH% - set INCLUDE=%LIBRARY_INCLUDE_PATH%;%INCLUDE% - - :: Set CMAKE_ARGS with more comprehensive settings - set CMAKE_ARGS=-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=%LIBRARY_LIB_PATH% -DCMAKE_INCLUDE_PATH=%LIBRARY_INCLUDE_PATH% -DCMAKE_PREFIX_PATH=%LIBRARY_LIB_PATH%;%LIBRARY_INCLUDE_PATH% -DCMAKE_VERBOSE_MAKEFILE=ON - - echo Building with CMAKE_ARGS: %CMAKE_ARGS% - echo LIB path: %LIB% - echo PKG_CONFIG_PATH: %PKG_CONFIG_PATH% - echo PATH (first few entries): - echo %PATH% | cut -d; -f1-3 - - :: Debug: Show what libraries are available - echo Available .lib files in conda lib directory: - dir "%LIBRARY_LIB_PATH%\*.lib" 2>nul || echo No .lib files found - - :: Debug: Test pkg-config within the script - echo Testing pkg-config from within build script: - pkg-config --exists libavcodec && echo libavcodec found || echo libavcodec NOT found - - :: Use vc_env_helper.bat to set up VS environment and build - call packaging\vc_env_helper.bat python -m pip install -e . --no-build-isolation -v - EOF - - - name: Verify Visual Studio environment and build - shell: cmd + - name: Setup Visual Studio environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Build torchcodec from source run: | - echo Checking Visual Studio installation... - call packaging\vc_env_helper.bat echo Visual Studio environment setup complete + # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg + echo "Building torchcodec from source using system FFmpeg..." + + # Verify that torch is available + python -c "import torch; print('PyTorch found:', torch.__version__)" + + # Set up environment variables for the build + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + library_bin_path="$conda_env_path/Library/bin" + library_include_path="$conda_env_path/Library/include" + + # Set CMAKE_ARGS with proper library paths + export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=$library_lib_path -DCMAKE_INCLUDE_PATH=$library_include_path -DCMAKE_PREFIX_PATH=$library_lib_path;$library_include_path -DCMAKE_VERBOSE_MAKEFILE=ON" + + echo "CMAKE_ARGS: $CMAKE_ARGS" + echo "Python executable: $(which python)" + echo "Conda environment: $CONDA_PREFIX" + + # Debug: Show what libraries are available + echo "Available .lib files in conda lib directory:" + ls -la "$library_lib_path"/*.lib 2>/dev/null || echo "No .lib files found" + + # Debug: Test that torch is still available + echo "Testing PyTorch availability:" + python -c "import torch; print('PyTorch version:', torch.__version__)" - echo Running build with Visual Studio environment... - call build_with_vs.bat + # Build using pip install with the current conda environment + python -m pip install -e . --no-build-isolation -v - name: Test torchcodec import run: | From edca0c371b7f9e53090692e88e82812140b3270a Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 12:56:25 +0100 Subject: [PATCH 34/34] typo --- src/torchcodec/_core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 54411add3..0facc37cf 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -213,7 +213,7 @@ function(make_torchcodec_libraries TARGETS ${all_libraries} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs -ibrary to prevent + ) # stray initialization of py::objects. The rest of the object code must # match. See: # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes