From ab509d2f9223d9889f0ace94d71bee729c00315d Mon Sep 17 00:00:00 2001 From: cxzhong Date: Fri, 7 Nov 2025 18:35:35 +0800 Subject: [PATCH 01/22] Add support for multi-platform wheels and enhance tarball handling - Introduced a new `tarballs_info` property in the Package class to manage multiple tarballs for platform-specific wheels. - Implemented `find_tarball_for_platform` method to select the appropriate tarball based on the current platform and Python version. - Enhanced the `Tarball` class to handle platform-specific wheels, including methods for downloading wheels using pip and checking for cached wheels. - Updated checksum verification to support multiple tarballs, ensuring integrity checks for downloaded files. - Added logic to fall back to traditional download methods if pip fails or if no cached wheels are found. - Created new dependencies and package version files for the `rpds_py` package. --- build/pkgs/rpds_py/SPKG.rst | 59 ++++ build/pkgs/rpds_py/checksums.ini | 208 ++++++++++++++ build/pkgs/rpds_py/dependencies | 1 + build/pkgs/rpds_py/package-version.txt | 1 + build/pkgs/rpds_py/type | 1 + build/sage_bootstrap/package.py | 254 ++++++++++++++++- build/sage_bootstrap/tarball.py | 368 ++++++++++++++++++++++++- 7 files changed, 880 insertions(+), 12 deletions(-) create mode 100644 build/pkgs/rpds_py/SPKG.rst create mode 100644 build/pkgs/rpds_py/checksums.ini create mode 100644 build/pkgs/rpds_py/dependencies create mode 100644 build/pkgs/rpds_py/package-version.txt create mode 100644 build/pkgs/rpds_py/type diff --git a/build/pkgs/rpds_py/SPKG.rst b/build/pkgs/rpds_py/SPKG.rst new file mode 100644 index 00000000000..4dc3bbfdc16 --- /dev/null +++ b/build/pkgs/rpds_py/SPKG.rst @@ -0,0 +1,59 @@ +rpds_py: Python bindings to Rust's persistent data structures +============================================================== + +Description +----------- + +Python bindings to the Rust rpds crate for persistent data structures. + +rpds-py provides efficient, immutable data structures including: + +* ``HashTrieMap`` - Persistent hash map +* ``HashTrieSet`` - Persistent hash set +* ``List`` - Persistent list with efficient operations + +These data structures are backed by Rust implementations for high performance +while maintaining a Pythonic API. They are particularly useful for functional +programming patterns and situations requiring immutable, persistent collections. + +The library is used by projects like the referencing library (part of the +Python JSON Schema ecosystem) as a faster alternative to pyrsistent. + +License +------- + +MIT License + +Upstream Contact +---------------- + +- Author: Julian Berman +- Home page: https://github.com/crate-py/rpds +- PyPI: https://pypi.org/project/rpds-py/ +- Documentation: https://rpds.readthedocs.io/ +- Upstream Rust crate: https://github.com/orium/rpds + +Dependencies +------------ + +Python (>= 3.10) + +Build dependencies: Rust toolchain (automatically handled by pip when +installing from source) + +Special Notes +------------- + +This package provides platform-specific binary wheels for multiple Python +versions and platforms: + +* Python 3.11, 3.12, 3.13, 3.14 (including free-threaded 3.13t and 3.14t) +* Linux (x86_64, aarch64, musllinux) +* macOS (x86_64, arm64) +* Windows (win32, win_amd64, win_arm64) + +The Sage build system automatically selects and downloads the appropriate +wheel for your platform and Python version using pip's auto-detection. + +When building from source, a Rust toolchain is required as rpds-py contains +Rust extensions for performance. diff --git a/build/pkgs/rpds_py/checksums.ini b/build/pkgs/rpds_py/checksums.ini new file mode 100644 index 00000000000..6ef643a3263 --- /dev/null +++ b/build/pkgs/rpds_py/checksums.ini @@ -0,0 +1,208 @@ +tarball=rpds_py-0.28.0-cp311-cp311-macosx_10_12_x86_64.whl +sha256=03065002fd2e287725d95fbc69688e0c6daf6c6314ba38bdbaa3895418e09296 +upstream_url=https://files.pythonhosted.org/packages/a6/34/058d0db5471c6be7bef82487ad5021ff8d1d1d27794be8730aad938649cf/rpds_py-0.28.0-cp311-cp311-macosx_10_12_x86_64.whl + +tarball=rpds_py-0.28.0-cp311-cp311-macosx_11_0_arm64.whl +sha256=28ea02215f262b6d078daec0b45344c89e161eab9526b0d898221d96fdda5f27 +upstream_url=https://files.pythonhosted.org/packages/5d/67/9503f0ec8c055a0782880f300c50a2b8e5e72eb1f94dfc2053da527444dd/rpds_py-0.28.0-cp311-cp311-macosx_11_0_arm64.whl + +tarball=rpds_py-0.28.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +sha256=25dbade8fbf30bcc551cb352376c0ad64b067e4fc56f90e22ba70c3ce205988c +upstream_url=https://files.pythonhosted.org/packages/68/2e/94223ee9b32332a41d75b6f94b37b4ce3e93878a556fc5f152cbd856a81f/rpds_py-0.28.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + +tarball=rpds_py-0.28.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sha256=ac9f83e7b326a3f9ec3ef84cda98fb0a74c7159f33e692032233046e7fd15da2 +upstream_url=https://files.pythonhosted.org/packages/1f/53/14e37ce83202c632c89b0691185dca9532288ff9d390eacae3d2ff771bae/rpds_py-0.28.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + +tarball=rpds_py-0.28.0-cp311-cp311-musllinux_1_2_aarch64.whl +sha256=c9a40040aa388b037eb39416710fbcce9443498d2eaab0b9b45ae988b53f5c67 +upstream_url=https://files.pythonhosted.org/packages/dd/f5/e1cec473d4bde6df1fd3738be8e82d64dd0600868e76e92dfeaebbc2d18f/rpds_py-0.28.0-cp311-cp311-musllinux_1_2_aarch64.whl + +tarball=rpds_py-0.28.0-cp311-cp311-musllinux_1_2_x86_64.whl +sha256=1571ae4292649100d743b26d5f9c63503bb1fedf538a8f29a98dce2d5ba6b4e6 +upstream_url=https://files.pythonhosted.org/packages/9c/9c/ffc6e9218cd1eb5c2c7dbd276c87cd10e8c2232c456b554169eb363381df/rpds_py-0.28.0-cp311-cp311-musllinux_1_2_x86_64.whl + +tarball=rpds_py-0.28.0-cp311-cp311-win32.whl +sha256=5cfa9af45e7c1140af7321fa0bef25b386ee9faa8928c80dc3a5360971a29e8c +upstream_url=https://files.pythonhosted.org/packages/5f/50/da8b6d33803a94df0149345ee33e5d91ed4d25fc6517de6a25587eae4133/rpds_py-0.28.0-cp311-cp311-win32.whl + +tarball=rpds_py-0.28.0-cp311-cp311-win_amd64.whl +sha256=dd8d86b5d29d1b74100982424ba53e56033dc47720a6de9ba0259cf81d7cecaa +upstream_url=https://files.pythonhosted.org/packages/12/fd/b0f48c4c320ee24c8c20df8b44acffb7353991ddf688af01eef5f93d7018/rpds_py-0.28.0-cp311-cp311-win_amd64.whl + +tarball=rpds_py-0.28.0-cp311-cp311-win_arm64.whl +sha256=4e27d3a5709cc2b3e013bf93679a849213c79ae0573f9b894b284b55e729e120 +upstream_url=https://files.pythonhosted.org/packages/b4/21/c8e77a2ac66e2ec4e21f18a04b4e9a0417ecf8e61b5eaeaa9360a91713b4/rpds_py-0.28.0-cp311-cp311-win_arm64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-macosx_10_12_x86_64.whl +sha256=6b4f28583a4f247ff60cd7bdda83db8c3f5b05a7a82ff20dd4b078571747708f +upstream_url=https://files.pythonhosted.org/packages/b8/5c/6c3936495003875fe7b14f90ea812841a08fca50ab26bd840e924097d9c8/rpds_py-0.28.0-cp312-cp312-macosx_10_12_x86_64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-macosx_11_0_arm64.whl +sha256=d678e91b610c29c4b3d52a2c148b641df2b4676ffe47c59f6388d58b99cdc424 +upstream_url=https://files.pythonhosted.org/packages/56/f9/a0f1ca194c50aa29895b442771f036a25b6c41a35e4f35b1a0ea713bedae/rpds_py-0.28.0-cp312-cp312-macosx_11_0_arm64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +sha256=e819e0e37a44a78e1383bf1970076e2ccc4dc8c2bbaa2f9bd1dc987e9afff628 +upstream_url=https://files.pythonhosted.org/packages/18/ea/42d243d3a586beb72c77fa5def0487daf827210069a95f36328e869599ea/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sha256=8455933b4bcd6e83fde3fefc987a023389c4b13f9a58c8d23e4b3f6d13f78c84 +upstream_url=https://files.pythonhosted.org/packages/3e/cd/49ce51767b879cde77e7ad9fae164ea15dce3616fe591d9ea1df51152706/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-musllinux_1_2_aarch64.whl +sha256=735f8495a13159ce6a0d533f01e8674cec0c57038c920495f87dcb20b3ddb48a +upstream_url=https://files.pythonhosted.org/packages/ff/6a/841337980ea253ec797eb084665436007a1aad0faac1ba097fb906c5f69c/rpds_py-0.28.0-cp312-cp312-musllinux_1_2_aarch64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-musllinux_1_2_x86_64.whl +sha256=2374e16cc9131022e7d9a8f8d65d261d9ba55048c78f3b6e017971a4f5e6353c +upstream_url=https://files.pythonhosted.org/packages/b6/ee/44d024b4843f8386a4eeaa4c171b3d31d55f7177c415545fd1a24c249b5d/rpds_py-0.28.0-cp312-cp312-musllinux_1_2_x86_64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-win32.whl +sha256=d15431e334fba488b081d47f30f091e5d03c18527c325386091f31718952fe08 +upstream_url=https://files.pythonhosted.org/packages/7d/89/33e675dccff11a06d4d85dbb4d1865f878d5020cbb69b2c1e7b2d3f82562/rpds_py-0.28.0-cp312-cp312-win32.whl + +tarball=rpds_py-0.28.0-cp312-cp312-win_amd64.whl +sha256=a410542d61fc54710f750d3764380b53bf09e8c4edbf2f9141a82aa774a04f7c +upstream_url=https://files.pythonhosted.org/packages/af/36/45f6ebb3210887e8ee6dbf1bc710ae8400bb417ce165aaf3024b8360d999/rpds_py-0.28.0-cp312-cp312-win_amd64.whl + +tarball=rpds_py-0.28.0-cp312-cp312-win_arm64.whl +sha256=1f0cfd1c69e2d14f8c892b893997fa9a60d890a0c8a603e88dca4955f26d1edd +upstream_url=https://files.pythonhosted.org/packages/57/91/f3fb250d7e73de71080f9a221d19bd6a1c1eb0d12a1ea26513f6c1052ad6/rpds_py-0.28.0-cp312-cp312-win_arm64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-macosx_10_12_x86_64.whl +sha256=e9e184408a0297086f880556b6168fa927d677716f83d3472ea333b42171ee3b +upstream_url=https://files.pythonhosted.org/packages/d3/03/ce566d92611dfac0085c2f4b048cd53ed7c274a5c05974b882a908d540a2/rpds_py-0.28.0-cp313-cp313-macosx_10_12_x86_64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-macosx_11_0_arm64.whl +sha256=edd267266a9b0448f33dc465a97cfc5d467594b600fe28e7fa2f36450e03053a +upstream_url=https://files.pythonhosted.org/packages/00/34/1c61da1b25592b86fd285bd7bd8422f4c9d748a7373b46126f9ae792a004/rpds_py-0.28.0-cp313-cp313-macosx_11_0_arm64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +sha256=85beb8b3f45e4e32f6802fb6cd6b17f615ef6c6a52f265371fb916fae02814aa +upstream_url=https://files.pythonhosted.org/packages/fc/00/ed1e28616848c61c493a067779633ebf4b569eccaacf9ccbdc0e7cba2b9d/rpds_py-0.28.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sha256=b8e1e9be4fa6305a16be628959188e4fd5cd6f1b0e724d63c6d8b2a8adf74ea6 +upstream_url=https://files.pythonhosted.org/packages/ca/ee/a324d3198da151820a326c1f988caaa4f37fc27955148a76fff7a2d787a9/rpds_py-0.28.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-musllinux_1_2_aarch64.whl +sha256=5b43c6a3726efd50f18d8120ec0551241c38785b68952d240c45ea553912ac41 +upstream_url=https://files.pythonhosted.org/packages/68/61/7c195b30d57f1b8d5970f600efee72a4fad79ec829057972e13a0370fd24/rpds_py-0.28.0-cp313-cp313-musllinux_1_2_aarch64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-musllinux_1_2_x86_64.whl +sha256=7a52a5169c664dfb495882adc75c304ae1d50df552fbd68e100fdc719dee4ff9 +upstream_url=https://files.pythonhosted.org/packages/66/df/62fc783781a121e77fee9a21ead0a926f1b652280a33f5956a5e7833ed30/rpds_py-0.28.0-cp313-cp313-musllinux_1_2_x86_64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-win32.whl +sha256=2e42456917b6687215b3e606ab46aa6bca040c77af7df9a08a6dcfe8a4d10ca5 +upstream_url=https://files.pythonhosted.org/packages/84/85/d34366e335140a4837902d3dea89b51f087bd6a63c993ebdff59e93ee61d/rpds_py-0.28.0-cp313-cp313-win32.whl + +tarball=rpds_py-0.28.0-cp313-cp313-win_amd64.whl +sha256=e0a0311caedc8069d68fc2bf4c9019b58a2d5ce3cd7cb656c845f1615b577e1e +upstream_url=https://files.pythonhosted.org/packages/3c/1c/f25a3f3752ad7601476e3eff395fe075e0f7813fbb9862bd67c82440e880/rpds_py-0.28.0-cp313-cp313-win_amd64.whl + +tarball=rpds_py-0.28.0-cp313-cp313-win_arm64.whl +sha256=04c1b207ab8b581108801528d59ad80aa83bb170b35b0ddffb29c20e411acdc1 +upstream_url=https://files.pythonhosted.org/packages/e0/d6/5f39b42b99615b5bc2f36ab90423ea404830bdfee1c706820943e9a645eb/rpds_py-0.28.0-cp313-cp313-win_arm64.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-macosx_10_12_x86_64.whl +sha256=f296ea3054e11fc58ad42e850e8b75c62d9a93a9f981ad04b2e5ae7d2186ff9c +upstream_url=https://files.pythonhosted.org/packages/5c/8b/0c69b72d1cee20a63db534be0df271effe715ef6c744fdf1ff23bb2b0b1c/rpds_py-0.28.0-cp313-cp313t-macosx_10_12_x86_64.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-macosx_11_0_arm64.whl +sha256=5a7306c19b19005ad98468fcefeb7100b19c79fc23a5f24a12e06d91181193fa +upstream_url=https://files.pythonhosted.org/packages/f7/6d/0c2ee773cfb55c31a8514d2cece856dd299170a49babd50dcffb15ddc749/rpds_py-0.28.0-cp313-cp313t-macosx_11_0_arm64.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +sha256=e5d9b86aa501fed9862a443c5c3116f6ead8bc9296185f369277c42542bd646b +upstream_url=https://files.pythonhosted.org/packages/e2/1c/22513ab25a27ea205144414724743e305e8153e6abe81833b5e678650f5a/rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sha256=8d252db6b1a78d0a3928b6190156042d54c93660ce4d98290d7b16b5296fb7cc +upstream_url=https://files.pythonhosted.org/packages/b1/2c/f30892f9e54bd02e5faca3f6a26d6933c51055e67d54818af90abed9748e/rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-musllinux_1_2_aarch64.whl +sha256=8aa23b6f0fc59b85b4c7d89ba2965af274346f738e8d9fc2455763602e62fd5f +upstream_url=https://files.pythonhosted.org/packages/42/b5/71e8777ac55e6af1f4f1c05b47542a1eaa6c33c1cf0d300dca6a1c6e159a/rpds_py-0.28.0-cp313-cp313t-musllinux_1_2_aarch64.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-musllinux_1_2_x86_64.whl +sha256=bcf1d210dfee61a6c86551d67ee1031899c0fdbae88b2d44a569995d43797712 +upstream_url=https://files.pythonhosted.org/packages/4a/d4/407ad9960ca7856d7b25c96dcbe019270b5ffdd83a561787bc682c797086/rpds_py-0.28.0-cp313-cp313t-musllinux_1_2_x86_64.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-win32.whl +sha256=3aa4dc0fdab4a7029ac63959a3ccf4ed605fee048ba67ce89ca3168da34a1342 +upstream_url=https://files.pythonhosted.org/packages/51/31/2f46fe0efcac23fbf5797c6b6b7e1c76f7d60773e525cb65fcbc582ee0f2/rpds_py-0.28.0-cp313-cp313t-win32.whl + +tarball=rpds_py-0.28.0-cp313-cp313t-win_amd64.whl +sha256=7b7d9d83c942855e4fdcfa75d4f96f6b9e272d42fffcb72cd4bb2577db2e2907 +upstream_url=https://files.pythonhosted.org/packages/92/e4/15947bda33cbedfc134490a41841ab8870a72a867a03d4969d886f6594a2/rpds_py-0.28.0-cp313-cp313t-win_amd64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-macosx_10_12_x86_64.whl +sha256=dcdcb890b3ada98a03f9f2bb108489cdc7580176cb73b4f2d789e9a1dac1d472 +upstream_url=https://files.pythonhosted.org/packages/08/47/ffe8cd7a6a02833b10623bf765fbb57ce977e9a4318ca0e8cf97e9c3d2b3/rpds_py-0.28.0-cp314-cp314-macosx_10_12_x86_64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-macosx_11_0_arm64.whl +sha256=f274f56a926ba2dc02976ca5b11c32855cbd5925534e57cfe1fda64e04d1add2 +upstream_url=https://files.pythonhosted.org/packages/f9/9f/890f36cbd83a58491d0d91ae0db1702639edb33fb48eeb356f80ecc6b000/rpds_py-0.28.0-cp314-cp314-macosx_11_0_arm64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +sha256=4fe0438ac4a29a520ea94c8c7f1754cdd8feb1bc490dfda1bfd990072363d527 +upstream_url=https://files.pythonhosted.org/packages/09/e3/921eb109f682aa24fb76207698fbbcf9418738f35a40c21652c29053f23d/rpds_py-0.28.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sha256=5ae8ee156d6b586e4292491e885d41483136ab994e719a13458055bec14cf370 +upstream_url=https://files.pythonhosted.org/packages/da/37/e84283b9e897e3adc46b4c88bb3f6ec92a43bd4d2f7ef5b13459963b2e9c/rpds_py-0.28.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-musllinux_1_2_aarch64.whl +sha256=6796079e5d24fdaba6d49bda28e2c47347e89834678f2bc2c1b4fc1489c0fb01 +upstream_url=https://files.pythonhosted.org/packages/74/ae/cab05ff08dfcc052afc73dcb38cbc765ffc86f94e966f3924cd17492293c/rpds_py-0.28.0-cp314-cp314-musllinux_1_2_aarch64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-musllinux_1_2_x86_64.whl +sha256=bbdc5640900a7dbf9dd707fe6388972f5bbd883633eb68b76591044cfe346f7e +upstream_url=https://files.pythonhosted.org/packages/ab/12/85a57d7a5855a3b188d024b099fd09c90db55d32a03626d0ed16352413ff/rpds_py-0.28.0-cp314-cp314-musllinux_1_2_x86_64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-win32.whl +sha256=adc8aa88486857d2b35d75f0640b949759f79dc105f50aa2c27816b2e0dd749f +upstream_url=https://files.pythonhosted.org/packages/6c/65/10643fb50179509150eb94d558e8837c57ca8b9adc04bd07b98e57b48f8c/rpds_py-0.28.0-cp314-cp314-win32.whl + +tarball=rpds_py-0.28.0-cp314-cp314-win_amd64.whl +sha256=66e6fa8e075b58946e76a78e69e1a124a21d9a48a5b4766d15ba5b06869d1fa1 +upstream_url=https://files.pythonhosted.org/packages/b4/84/0c11fe4d9aaea784ff4652499e365963222481ac647bcd0251c88af646eb/rpds_py-0.28.0-cp314-cp314-win_amd64.whl + +tarball=rpds_py-0.28.0-cp314-cp314-win_arm64.whl +sha256=a6fe887c2c5c59413353b7c0caff25d0e566623501ccfff88957fa438a69377d +upstream_url=https://files.pythonhosted.org/packages/0f/e0/3ab3b86ded7bb18478392dc3e835f7b754cd446f62f3fc96f4fe2aca78f6/rpds_py-0.28.0-cp314-cp314-win_arm64.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-macosx_10_12_x86_64.whl +sha256=7a69df082db13c7070f7b8b1f155fa9e687f1d6aefb7b0e3f7231653b79a067b +upstream_url=https://files.pythonhosted.org/packages/51/ec/d5681bb425226c3501eab50fc30e9d275de20c131869322c8a1729c7b61c/rpds_py-0.28.0-cp314-cp314t-macosx_10_12_x86_64.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-macosx_11_0_arm64.whl +sha256=b1cde22f2c30ebb049a9e74c5374994157b9b70a16147d332f89c99c5960737a +upstream_url=https://files.pythonhosted.org/packages/be/ec/568c5e689e1cfb1ea8b875cffea3649260955f677fdd7ddc6176902d04cd/rpds_py-0.28.0-cp314-cp314t-macosx_11_0_arm64.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +sha256=5338742f6ba7a51012ea470bd4dc600a8c713c0c72adaa0977a1b1f4327d6592 +upstream_url=https://files.pythonhosted.org/packages/32/fe/51ada84d1d2a1d9d8f2c902cfddd0133b4a5eb543196ab5161d1c07ed2ad/rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sha256=3114f4db69ac5a1f32e7e4d1cbbe7c8f9cf8217f78e6e002cedf2d54c2a548ed +upstream_url=https://files.pythonhosted.org/packages/ab/81/5d98cc0329bbb911ccecd0b9e19fbf7f3a5de8094b4cda5e71013b2dd77e/rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-musllinux_1_2_aarch64.whl +sha256=1e8ee6413cfc677ce8898d9cde18cc3a60fc2ba756b0dec5b71eb6eb21c49fa1 +upstream_url=https://files.pythonhosted.org/packages/39/4c/f08283a82ac141331a83a40652830edd3a4a92c34e07e2bbe00baaea2f5f/rpds_py-0.28.0-cp314-cp314t-musllinux_1_2_aarch64.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-musllinux_1_2_x86_64.whl +sha256=b670c30fd87a6aec281c3c9896d3bae4b205fd75d79d06dc87c2503717e46092 +upstream_url=https://files.pythonhosted.org/packages/d3/0c/5bafdd8ccf6aa9d3bfc630cfece457ff5b581af24f46a9f3590f790e3df2/rpds_py-0.28.0-cp314-cp314t-musllinux_1_2_x86_64.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-win32.whl +sha256=8014045a15b4d2b3476f0a287fcc93d4f823472d7d1308d47884ecac9e612be3 +upstream_url=https://files.pythonhosted.org/packages/2c/37/dcc5d8397caa924988693519069d0beea077a866128719351a4ad95e82fc/rpds_py-0.28.0-cp314-cp314t-win32.whl + +tarball=rpds_py-0.28.0-cp314-cp314t-win_amd64.whl +sha256=7a4e59c90d9c27c561eb3160323634a9ff50b04e4f7820600a2beb0ac90db578 +upstream_url=https://files.pythonhosted.org/packages/d7/69/64d43b21a10d72b45939a28961216baeb721cc2a430f5f7c3bfa21659a53/rpds_py-0.28.0-cp314-cp314t-win_amd64.whl + diff --git a/build/pkgs/rpds_py/dependencies b/build/pkgs/rpds_py/dependencies new file mode 100644 index 00000000000..a1b589e38a3 --- /dev/null +++ b/build/pkgs/rpds_py/dependencies @@ -0,0 +1 @@ +pip diff --git a/build/pkgs/rpds_py/package-version.txt b/build/pkgs/rpds_py/package-version.txt new file mode 100644 index 00000000000..022a0337400 --- /dev/null +++ b/build/pkgs/rpds_py/package-version.txt @@ -0,0 +1 @@ +0.28.0 \ No newline at end of file diff --git a/build/pkgs/rpds_py/type b/build/pkgs/rpds_py/type new file mode 100644 index 00000000000..aa0bc074b62 --- /dev/null +++ b/build/pkgs/rpds_py/type @@ -0,0 +1 @@ +standard \ No newline at end of file diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 2d40d4915df..9cf483cdbef 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -241,6 +241,194 @@ def tarball_upstream_url(self): else: return None + @property + def tarballs_info(self): + """ + Return information about all tarballs for this package. + + This supports packages with multiple platform-specific wheels. + + OUTPUT: + + List of dictionaries, each containing: + - 'tarball': tarball filename pattern + - 'sha256': SHA256 checksum + - 'sha1': SHA1 checksum (optional) + - 'upstream_url': upstream URL pattern + """ + return self.__tarballs_info + + def find_tarball_for_platform(self, python_version=None): + """ + Find the appropriate tarball for the current platform. + + For packages with multiple platform-specific wheels, this selects + the one matching the current platform and Python version. + + Properly handles wheel ABI tags: + - cp313 (CPython 3.13 specific) + - cp313t (CPython 3.13 free-threaded/nogil) + - abi3 (stable ABI, forward compatible) + - pp39 (PyPy 3.9) + + INPUT: + + - ``python_version`` -- Python version string (e.g., '3.11'), or None to auto-detect + + OUTPUT: + + Dictionary with tarball info, or None if no suitable tarball found. + The dictionary contains the same fields as tarballs_info entries. + """ + import sys + import platform + + if not self.__tarballs_info: + return None + + # If only one tarball, return it + if len(self.__tarballs_info) == 1: + return self.__tarballs_info[0] + + # Auto-detect Python version if not provided + if python_version is None: + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + + py_ver_tuple = tuple(int(x) for x in python_version.split('.')) + + # Check if running free-threaded Python (Python 3.13+) + is_free_threaded = False + try: + # Python 3.13+ has sys._is_gil_enabled() to check free-threading mode + if hasattr(sys, '_is_gil_enabled'): + is_free_threaded = not sys._is_gil_enabled() + except Exception: + pass + + # Get platform info + system = platform.system().lower() + machine = platform.machine().lower() + + # Build platform tags to match against + platform_tags = [] + exclude_tags = [] # Tags that should NOT be in the platform string + if system == 'linux': + if machine == 'x86_64': + platform_tags = ['manylinux', 'musllinux', 'linux_x86_64', 'x86_64'] + exclude_tags = ['aarch64', 'arm64', 'armv7l', 'ppc64', 's390x', 'i686', 'riscv64', 'macosx', 'win'] + elif machine in ['aarch64', 'arm64']: + platform_tags = ['manylinux', 'musllinux', 'linux_aarch64', 'aarch64', 'arm64'] + exclude_tags = ['x86_64', 'armv7l', 'ppc64', 's390x', 'i686', 'riscv64', 'macosx', 'win'] + elif system == 'darwin': + if machine in ['arm64', 'aarch64']: + platform_tags = ['macosx_', 'arm64'] + exclude_tags = ['x86_64', 'i386', 'manylinux', 'musllinux', 'win'] + else: + platform_tags = ['macosx_', 'x86_64'] + exclude_tags = ['arm64', 'i386', 'manylinux', 'musllinux', 'win'] + elif system == 'windows': + if machine == 'amd64' or machine == 'x86_64': + platform_tags = ['win_amd64'] + exclude_tags = ['win32', 'win_arm64', 'manylinux', 'musllinux', 'macosx'] + elif machine == 'arm64': + platform_tags = ['win_arm64'] + exclude_tags = ['win32', 'win_amd64', 'manylinux', 'musllinux', 'macosx'] + else: + platform_tags = ['win32'] + exclude_tags = ['win_amd64', 'win_arm64', 'manylinux', 'musllinux', 'macosx'] + + # Python version tag (e.g., 'cp311' for CPython 3.11) + py_ver_tag = f"cp{python_version.replace('.', '')}" + # Free-threaded ABI tag has 't' suffix (e.g., cp313t in cp313-cp313t) + if is_free_threaded: + abi_tag_free_threaded = py_ver_tag + 't' # e.g., cp313t + else: + abi_tag_free_threaded = None + + # Collect compatible wheels + compatible_wheels = [] + + for tarball_info in self.__tarballs_info: + tarball = tarball_info['tarball'] + + # Check if platform-independent wheel (-none-any.whl) + if tarball.endswith('-none-any.whl'): + compatible_wheels.append((tarball_info, 1000)) # Highest priority + continue + + # Check if not a wheel - use for non-wheel tarballs + if not tarball.endswith('.whl'): + matches_platform = any(tag in tarball.lower() for tag in platform_tags) + if matches_platform or not platform_tags: + compatible_wheels.append((tarball_info, 500)) + continue + + # Parse wheel filename: {name}-{version}-{python}-{abi}-{platform}.whl + # Extract tags from the wheel filename + parts = tarball.rsplit('-', 3) + if len(parts) != 4: + continue + + _, python_tag, abi_tag, platform_tag = parts + platform_tag = platform_tag[:-4] # Remove .whl extension + + # Check if matches current platform (must have a matching tag AND not have any excluded tags) + matches_platform = any(tag in platform_tag.lower() for tag in platform_tags) + has_excluded = any(tag in platform_tag.lower() for tag in exclude_tags) + if not matches_platform or has_excluded: + continue + + # Check Python/ABI compatibility + priority = 0 + + # Free-threaded wheel: python_tag=cp313, abi_tag=cp313t + if is_free_threaded and abi_tag_free_threaded and python_tag == py_ver_tag and abi_tag == abi_tag_free_threaded: + priority = 110 # Highest priority for free-threaded wheels on free-threaded Python + # Reject free-threaded ABI on standard Python + elif not is_free_threaded and abi_tag_free_threaded and abi_tag == abi_tag_free_threaded: + priority = 0 # Incompatible - free-threaded wheel on standard Python + # Exact Python version match (e.g., cp313-cp313) + elif py_ver_tag == python_tag and py_ver_tag == abi_tag: + # Standard Python wheel on free-threaded Python - lower priority + if is_free_threaded: + priority = 95 # Can work but prefer free-threaded wheels + else: + priority = 100 + # abi3 wheels - stable ABI, forward compatible + elif 'abi3' in abi_tag: + # Extract minimum Python version from abi3 wheel + # E.g., cp37-abi3 means Python 3.7+ + if python_tag.startswith('cp'): + min_version_str = python_tag[2:] # Remove 'cp' prefix + if len(min_version_str) >= 2: + min_major = int(min_version_str[0]) + min_minor = int(min_version_str[1:]) + min_version = (min_major, min_minor) + + # Check if current Python >= minimum required + if py_ver_tuple >= min_version: + # Lower priority than exact match, but still compatible + priority = 90 - (py_ver_tuple[1] - min_version[1]) # Prefer closer versions + # PyPy wheels (pp39, pp310, etc.) + elif python_tag.startswith('pp') and sys.implementation.name == 'pypy': + if python_tag == f"pp{python_version.replace('.', '')}": + priority = 100 + + if priority > 0: + compatible_wheels.append((tarball_info, priority)) + + # Sort by priority (highest first) and return the best match + if compatible_wheels: + compatible_wheels.sort(key=lambda x: x[1], reverse=True) + best_match = compatible_wheels[0][0] + threading_mode = "free-threaded" if is_free_threaded else "standard" + log.debug(f'Selected {best_match["tarball"]} for platform ({threading_mode} Python, priority: {compatible_wheels[0][1]})') + return best_match + + # If no match found, return the first one (backward compatibility) + log.warning(f'No exact platform match found for {self.name}, using first tarball') + return self.__tarballs_info[0] + @property def tarball_package(self): """ @@ -356,6 +544,23 @@ def source(self): return 'script' return 'none' + def is_platform_specific_wheel(self): + """ + Check if this package uses a platform-specific wheel. + + Platform-specific wheels need special handling during download + as they contain platform tags in the filename. + + Returns True if this is a wheel package with platform-specific binaries, + False otherwise. + """ + if self.source != 'wheel': + return False + if not self.tarball_filename: + return False + # Platform-independent wheels end with -none-any.whl + return not self.tarball_filename.endswith('-none-any.whl') + @property def trees(self): """ @@ -507,10 +712,24 @@ def line_count_file(self, filename): def _init_checksum(self): """ Load the checksums from the appropriate ``checksums.ini`` file + + Supports multiple tarballs with format: + tarball=package-VERSION-cp311-cp311-manylinux_2_17_x86_64.whl + sha256=abc123... + upstream_url=https://... + tarball=package-VERSION-cp311-cp311-macosx_11_0_arm64.whl + sha256=def456... + upstream_url=https://... """ checksums_ini = os.path.join(self.path, 'checksums.ini') assignment = re.compile('(?P[a-zA-Z0-9_]*)=(?P.*)') - result = dict() + + # Store all entries, supporting multiple values for tarball, sha256, upstream_url + tarballs = [] + sha256s = [] + sha1s = [] + upstream_urls = [] + try: with open(checksums_ini, 'rt') as f: for line in f.readlines(): @@ -518,13 +737,36 @@ def _init_checksum(self): if match is None: continue var, value = match.groups() - result[var] = value + + # Collect multiple entries + if var == 'tarball': + tarballs.append(value) + elif var == 'sha256': + sha256s.append(value) + elif var == 'sha1': + sha1s.append(value) + elif var == 'upstream_url': + upstream_urls.append(value) except IOError: pass - self.__sha1 = result.get('sha1', None) - self.__sha256 = result.get('sha256', None) - self.__tarball_pattern = result.get('tarball', None) - self.__tarball_upstream_url_pattern = result.get('upstream_url', None) + + # Store all tarballs info + self.__tarballs_info = [] + for i, tarball in enumerate(tarballs): + info = { + 'tarball': tarball, + 'sha256': sha256s[i] if i < len(sha256s) else None, + 'sha1': sha1s[i] if i < len(sha1s) else None, + 'upstream_url': upstream_urls[i] if i < len(upstream_urls) else None, + } + self.__tarballs_info.append(info) + + # For backward compatibility, set the first tarball as primary + self.__sha1 = sha1s[0] if sha1s else None + self.__sha256 = sha256s[0] if sha256s else None + self.__tarball_pattern = tarballs[0] if tarballs else None + self.__tarball_upstream_url_pattern = upstream_urls[0] if upstream_urls else None + # Name of the directory containing the checksums.ini file self.__tarball_package_name = os.path.realpath(checksums_ini).split(os.sep)[-2] diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index 54af50dd8f8..7343d816cc7 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -16,6 +16,8 @@ # **************************************************************************** import os +import sys +import subprocess import logging log = logging.getLogger() @@ -40,7 +42,7 @@ class FileNotMirroredError(Exception): class Tarball(object): - def __init__(self, tarball_name, package=None): + def __init__(self, tarball_name, package=None, tarball_info=None): """ A (third-party downloadable) tarball @@ -52,8 +54,13 @@ def __init__(self, tarball_name, package=None): - ``tarball_name`` - string. The full filename (``foo-1.3.tar.bz2``) of a tarball on the Sage mirror network. + - ``package`` - Package object, or None to auto-detect + - ``tarball_info`` - dict with tarball info (for multi-tarball packages) + containing sha256, sha1, upstream_url """ self.__filename = tarball_name + self.__tarball_info = tarball_info + if package is None: self.__package = None for pkg in Package.all(): @@ -65,7 +72,9 @@ def __init__(self, tarball_name, package=None): raise ValueError(error) else: self.__package = package - if package.tarball_filename != tarball_name: + # For multi-tarball packages (with tarball_info), skip the filename check + # since we're selecting a platform-specific wheel + if tarball_info is None and package.tarball_filename != tarball_name: error = 'tarball {0} is not referenced by the {1} package'.format(tarball_name, package.name) log.error(error) raise ValueError(error) @@ -126,22 +135,199 @@ def _compute_sha256(self): def checksum_verifies(self, force_sha256=False): """ Test whether the checksum of the downloaded file is correct. + + Uses tarball_info if available (for multi-tarball packages), + otherwise falls back to package-level checksums. """ - if self.package.sha256: + # Use tarball_info if available (for multi-tarball packages) + if self.__tarball_info: + sha256_expected = self.__tarball_info.get('sha256') + sha1_expected = self.__tarball_info.get('sha1') + else: + sha256_expected = self.package.sha256 + sha1_expected = self.package.sha1 + + if sha256_expected: sha256 = self._compute_sha256() - if sha256 != self.package.sha256: + if sha256 != sha256_expected: + log.error(f'SHA256 mismatch for {self.filename}') + log.error(f'Expected: {sha256_expected}') + log.error(f'Got: {sha256}') return False elif force_sha256: log.warning('sha256 not available for {0}'.format(self.package.name)) return False else: log.warning('sha256 not available for {0}, using sha1'.format(self.package.name)) - sha1 = self._compute_sha1() - return sha1 == self.package.sha1 + if sha1_expected: + sha1 = self._compute_sha1() + if sha1 != sha1_expected: + log.error(f'SHA1 mismatch for {self.filename}') + log.error(f'Expected: {sha1_expected}') + log.error(f'Got: {sha1}') + return False + return True + else: + log.warning('No checksum available for {0}'.format(self.package.name)) + return False + + return True def is_distributable(self): return 'do-not-distribute' not in self.filename + def is_platform_specific_wheel(self): + """ + Check if this is a platform-specific wheel. + + Platform-specific wheels have platform tags like: + - manylinux_2_17_x86_64 + - macosx_11_0_arm64 + - win_amd64 + + Platform-independent wheels end with -none-any.whl + """ + if not self.filename or not self.filename.endswith('.whl'): + return False + return not self.filename.endswith('-none-any.whl') + + def _download_wheel_with_pip(self, dest_dir): + """ + Download a platform-specific wheel using pip. + + This uses pip's built-in logic to automatically select the appropriate + wheel for the current platform and Python version. + + Returns True on success, False on failure (e.g., offline mode). + """ + try: + import subprocess + import sys + + log.info('Using pip to auto-detect and download wheel for {0}'.format(self.package.name)) + + # Extract package name and version from the package + package_name = self.package.name.replace('_', '-') # PyPI uses dashes + version = self.package.version + package_spec = f"{package_name}=={version}" + + # Let pip automatically detect platform and Python version + cmd = [ + sys.executable, '-m', 'pip', 'download', + package_spec, + '-d', dest_dir, + '--no-deps', + '--only-binary', ':all:' + ] + + log.info(f"Running pip command: {' '.join(cmd)}") + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=60 # 60 second timeout + ) + + if result.returncode == 0: + # Find the downloaded wheel file + wheel_files = [f for f in os.listdir(dest_dir) + if f.endswith('.whl') and package_name.replace('-', '_') in f.lower()] + + if wheel_files: + wheel_path = os.path.join(dest_dir, wheel_files[0]) + log.info(f'Successfully downloaded wheel using pip: {wheel_path}') + return True + else: + log.warning('pip command succeeded but no wheel file found') + return False + else: + log.info(f'pip download failed (possibly offline): {result.stderr}') + return False + + except subprocess.TimeoutExpired: + log.warning('pip download timed out (possibly offline or slow connection)') + return False + except Exception as e: + log.info(f'pip download failed: {e}') + return False + + def _find_cached_wheel_for_platform(self): + """ + Find a cached wheel file that matches the current platform. + + Looks in SAGE_DISTFILES for any wheel matching the package name, + version, and current platform/Python version. + + Returns the path to the cached wheel if found, None otherwise. + """ + import sys + import platform + + # Get platform info + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + py_tag = f"cp{python_version.replace('.', '')}" + + system = platform.system().lower() + machine = platform.machine().lower() + + # Build possible platform tags + platform_patterns = [] + if system == 'linux': + if machine == 'x86_64': + platform_patterns = ['manylinux', 'linux_x86_64'] + elif machine in ['aarch64', 'arm64']: + platform_patterns = ['manylinux', 'linux_aarch64'] + elif system == 'darwin': + if machine in ['arm64', 'aarch64']: + platform_patterns = ['macosx', 'arm64'] + else: + platform_patterns = ['macosx', 'x86_64'] + + # Look for matching wheel files in upstream directory + # Note: Wheel filenames use underscores, not dashes + pkg_name_wheel = self.package.name.replace('-', '_').lower() + pkg_name_pypi = self.package.name.replace('_', '-').lower() + pkg_version = self.package.version + + log.debug(f'Looking for cached wheel: pkg_name_wheel={pkg_name_wheel}, pkg_name_pypi={pkg_name_pypi}, version={pkg_version}, py_tag={py_tag}') + log.debug(f'Platform patterns: {platform_patterns}') + + try: + for filename in os.listdir(SAGE_DISTFILES): + if not filename.endswith('.whl'): + continue + + filename_lower = filename.lower() + + log.debug(f'Checking file: {filename}') + + # Check if matches package name and version + # Wheel filenames use underscores, not dashes + if not (filename_lower.startswith(pkg_name_wheel) or filename_lower.startswith(pkg_name_pypi)): + log.debug(f' Does not start with {pkg_name_wheel} or {pkg_name_pypi}') + continue + if pkg_version not in filename: + log.debug(f' Does not contain version {pkg_version}') + continue + + # Check if matches current platform + matches_platform = any(p in filename_lower for p in platform_patterns) + matches_python = py_tag in filename + + log.debug(f' matches_platform={matches_platform}, matches_python={matches_python}') + + if matches_platform and matches_python: + wheel_path = os.path.join(SAGE_DISTFILES, filename) + log.info(f'Found cached wheel for platform: {filename}') + return wheel_path + except OSError as e: + log.warning(f'Error listing directory {SAGE_DISTFILES}: {e}') + pass + + log.warning(f'No cached wheel found for {pkg_name_wheel}/{pkg_name_pypi}-{pkg_version} (py_tag={py_tag}, platform={platform_patterns})') + return None + def download(self, allow_upstream=False): """ Download the tarball to the upstream directory. @@ -149,10 +335,27 @@ def download(self, allow_upstream=False): If allow_upstream is False and the package cannot be found on the sage mirrors, fall back to downloading it from the upstream URL if the package has one. + + For platform-specific wheels, this method: + 1. Checks for a cached wheel matching the current platform + 2. If cached and checksum valid, uses it + 3. Otherwise, uses pip download to get the correct wheel + 4. Falls back to traditional download if pip fails """ if not self.filename: raise ValueError('non-normal package does define a tarball, so cannot download') + destination = self.upstream_fqn + + # Check if package has multiple tarballs (multi-platform wheels) + has_multiple_tarballs = len(self.package.tarballs_info) > 1 + + if has_multiple_tarballs: + log.info(f'Package {self.package.name} has {len(self.package.tarballs_info)} platform-specific tarballs') + return self._download_multiple_wheels(allow_upstream) + + # Single tarball case - existing logic + # Check if file already exists and is valid if os.path.isfile(destination): if self.checksum_verifies(): log.info('Using cached file {destination}'.format(destination=destination)) @@ -163,6 +366,36 @@ def download(self, allow_upstream=False): # update the checksum (Issue #23972). log.warning('Invalid checksum; ignoring cached file {destination}' .format(destination=destination)) + + # For platform-specific wheels, try pip download first + if self.is_platform_specific_wheel(): + log.info('Detected platform-specific wheel: {0}'.format(self.filename)) + dest_dir = os.path.dirname(destination) + + if self._download_wheel_with_pip(dest_dir): + # pip download succeeded, but it may have downloaded a different + # wheel filename (different platform). Find the downloaded wheel. + wheel_files = [f for f in os.listdir(dest_dir) + if f.endswith('.whl') and f.startswith(self.package.name.replace('_', '-'))] + + if wheel_files: + actual_wheel = os.path.join(dest_dir, wheel_files[0]) + if actual_wheel != destination: + # Rename or link to expected filename + log.info('Downloaded wheel: {0}'.format(wheel_files[0])) + log.info('Expected filename: {0}'.format(self.filename)) + # For now, just use the downloaded wheel as-is + # The build system should be flexible about the exact filename + # Note: Skip checksum verification for pip-downloaded wheels + # as pip verifies integrity internally + return + else: + log.warning('pip download succeeded but no wheel file found') + + # If pip download failed, fall through to traditional download methods + log.info('Falling back to traditional download methods') + + # Traditional download logic for tarballs and platform-independent wheels successful_download = False log.info('Attempting to download package {0} from mirrors'.format(self.filename)) for mirror in MirrorList(): @@ -190,6 +423,129 @@ def download(self, allow_upstream=False): if not self.checksum_verifies(): raise ChecksumError('checksum does not match') + def _download_multiple_wheels(self, allow_upstream=False): + """ + Handle download for packages with multiple platform-specific wheels. + + Strategy: + 1. Check if already cached with valid checksum + 2. If not cached, try pip download (auto-detects platform/Python version) + 3. If pip fails (offline/error), fall back to traditional download from mirrors/upstream + """ + # Find the appropriate tarball for this platform from checksums.ini + tarball_info = self.package.find_tarball_for_platform() + + if not tarball_info: + raise ValueError(f'No suitable tarball found for {self.package.name} on current platform') + + # Get the actual filename with version substituted + tarball_pattern = tarball_info['tarball'] + tarball_filename = self.package._substitute_variables(tarball_pattern) + + log.info(f'Selected tarball for platform: {tarball_filename}') + + # Step 1: Check cache first + cached_wheel = self._find_cached_wheel_for_platform() + if cached_wheel: + # Verify checksum of cached wheel + try: + cached_tarball = Tarball(os.path.basename(cached_wheel), + package=self.package, + tarball_info=tarball_info) + if cached_tarball.checksum_verifies(): + log.info(f'Using cached wheel with valid checksum: {os.path.basename(cached_wheel)}') + # Update self to point to the cached wheel + self.__filename = os.path.basename(cached_wheel) + return + else: + log.warning(f'Cached wheel has invalid checksum, will re-download') + except Exception as e: + log.warning(f'Error checking cached wheel: {e}') + + dest_dir = SAGE_DISTFILES + + # Step 2: Try pip download (auto-detection) for platform-specific wheels + if not tarball_filename.endswith('-none-any.whl'): + log.info('Trying pip to auto-detect and download correct wheel...') + if self._download_wheel_with_pip(dest_dir): + # Verify what pip downloaded + cached_wheel = self._find_cached_wheel_for_platform() + if cached_wheel: + try: + downloaded_tarball = Tarball(os.path.basename(cached_wheel), + package=self.package, + tarball_info=tarball_info) + if downloaded_tarball.checksum_verifies(): + log.info(f'Successfully downloaded and verified wheel via pip: {os.path.basename(cached_wheel)}') + # Update self to point to the actually downloaded wheel + self.__filename = os.path.basename(cached_wheel) + return + else: + log.warning('pip-downloaded wheel checksum mismatch, falling back to traditional download') + except Exception as e: + log.warning(f'Error verifying pip-downloaded wheel: {e}') + # pip has its own integrity checks, so we can trust it + log.info('Trusting pip integrity verification') + # Update self to point to the actually downloaded wheel + self.__filename = os.path.basename(cached_wheel) + return + else: + log.warning('pip succeeded but could not find wheel, falling back to traditional download') + else: + log.info('pip download failed (possibly offline), falling back to traditional download') + + # Step 3: Fall back to traditional download from mirrors/upstream + log.info(f'Downloading {tarball_filename} using traditional method (mirrors/upstream)...') + destination = os.path.join(dest_dir, tarball_filename) + upstream_url_pattern = tarball_info.get('upstream_url') + + if upstream_url_pattern: + upstream_url = self.package._substitute_variables(upstream_url_pattern) + else: + upstream_url = None + + successful_download = False + + # Try mirrors first + for mirror in MirrorList(): + url = mirror.replace('${SPKG}', self.package.name) + if not url.endswith('/'): + url += '/' + url += tarball_filename + log.debug(f'Trying mirror: {url}') + try: + Download(url, destination).run() + successful_download = True + log.info(f'Downloaded from mirror: {url}') + break + except IOError: + log.debug('File not on mirror') + + # Try upstream if mirrors failed + if not successful_download and upstream_url and allow_upstream: + log.info(f'Trying upstream: {upstream_url}') + try: + Download(upstream_url, destination).run() + successful_download = True + log.info(f'Downloaded from upstream: {upstream_url}') + except IOError: + log.debug('File not at upstream URL') + + if not successful_download: + raise FileNotMirroredError(f'Could not download {tarball_filename} from pip, mirrors, or upstream') + + # Verify checksum of traditionally-downloaded file + try: + downloaded_tarball = Tarball(tarball_filename, + package=self.package, + tarball_info=tarball_info) + if not downloaded_tarball.checksum_verifies(): + raise ChecksumError(f'Checksum verification failed for {tarball_filename}') + log.info(f'Successfully downloaded and verified: {tarball_filename}') + except Exception as e: + log.error(f'Error verifying downloaded tarball: {e}') + raise + def save_as(self, destination): """ Save the tarball as a new file From f6d35f6a021edcdf6dd39c0687be28606ec693eb Mon Sep 17 00:00:00 2001 From: cxzhong Date: Fri, 7 Nov 2025 18:36:52 +0800 Subject: [PATCH 02/22] Fix dependency specification for jsonschema by replacing 'pyrsistent' with 'rpds_py' --- build/pkgs/jsonschema/dependencies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/pkgs/jsonschema/dependencies b/build/pkgs/jsonschema/dependencies index 8c7c4532c8d..7d4480d635c 100644 --- a/build/pkgs/jsonschema/dependencies +++ b/build/pkgs/jsonschema/dependencies @@ -1,4 +1,4 @@ -jsonschema_specifications pyrsistent attrs fqdn isoduration jsonpointer uri_template webcolors | $(PYTHON_TOOLCHAIN) $(PYTHON) +jsonschema_specifications rpds_py attrs fqdn isoduration jsonpointer uri_template webcolors | $(PYTHON_TOOLCHAIN) $(PYTHON) ---------- All lines of this file are ignored except the first. From 1ac314f2da8081bd0e2f76381086d29a5f7197a8 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Fri, 7 Nov 2025 19:11:56 +0800 Subject: [PATCH 03/22] Refactor tarball.py: remove unused import and improve logging messages --- build/sage_bootstrap/tarball.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index 7343d816cc7..1b70f4b4113 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -261,7 +261,6 @@ def _find_cached_wheel_for_platform(self): Returns the path to the cached wheel if found, None otherwise. """ - import sys import platform # Get platform info @@ -458,7 +457,7 @@ def _download_multiple_wheels(self, allow_upstream=False): self.__filename = os.path.basename(cached_wheel) return else: - log.warning(f'Cached wheel has invalid checksum, will re-download') + log.warning('Cached wheel has invalid checksum, will re-download') except Exception as e: log.warning(f'Error checking cached wheel: {e}') @@ -542,6 +541,8 @@ def _download_multiple_wheels(self, allow_upstream=False): if not downloaded_tarball.checksum_verifies(): raise ChecksumError(f'Checksum verification failed for {tarball_filename}') log.info(f'Successfully downloaded and verified: {tarball_filename}') + # Update self to point to the actually downloaded file + self.__filename = tarball_filename except Exception as e: log.error(f'Error verifying downloaded tarball: {e}') raise From 838c4caabd2eae392a71e9722f0232d31c4df8a6 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Fri, 7 Nov 2025 19:14:28 +0800 Subject: [PATCH 04/22] Fix dependency specification for jsonschema by restoring 'pyrsistent' Remove unused import 'unicode_to_str' from formatter.py --- build/pkgs/jsonschema/dependencies | 2 +- src/sage/repl/display/formatter.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build/pkgs/jsonschema/dependencies b/build/pkgs/jsonschema/dependencies index 7d4480d635c..3392c6d7170 100644 --- a/build/pkgs/jsonschema/dependencies +++ b/build/pkgs/jsonschema/dependencies @@ -1,4 +1,4 @@ -jsonschema_specifications rpds_py attrs fqdn isoduration jsonpointer uri_template webcolors | $(PYTHON_TOOLCHAIN) $(PYTHON) +jsonschema_specifications pyrsistent rpds_py attrs fqdn isoduration jsonpointer uri_template webcolors | $(PYTHON_TOOLCHAIN) $(PYTHON) ---------- All lines of this file are ignored except the first. diff --git a/src/sage/repl/display/formatter.py b/src/sage/repl/display/formatter.py index 3b73674dd48..ceea50f36eb 100644 --- a/src/sage/repl/display/formatter.py +++ b/src/sage/repl/display/formatter.py @@ -62,7 +62,6 @@ from io import StringIO from IPython.core.formatters import DisplayFormatter, PlainTextFormatter -from IPython.utils.py3compat import unicode_to_str from IPython.core.display import DisplayObject from ipywidgets import Widget @@ -311,7 +310,7 @@ def __call__(self, obj): print('---- calling ipython formatter ----') stream = StringIO() printer = SagePrettyPrinter( - stream, self.max_width, unicode_to_str(self.newline)) + stream, self.max_width, self.newline) printer.pretty(obj) printer.flush() return stream.getvalue() From 72a0c1cf514c6be9093b68ba534f6cbb2de1a41f Mon Sep 17 00:00:00 2001 From: cxzhong Date: Fri, 7 Nov 2025 23:01:35 +0800 Subject: [PATCH 05/22] Enhance package.py and tarball.py: implement subprocess calls to fetch Python version and free-threading status from Sage's Python, and remove pip download logic for platform-specific wheels. --- build/sage_bootstrap/package.py | 49 +++++++++++-- build/sage_bootstrap/tarball.py | 123 +------------------------------- 2 files changed, 45 insertions(+), 127 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 9cf483cdbef..cdaf7e24707 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -282,6 +282,7 @@ def find_tarball_for_platform(self, python_version=None): """ import sys import platform + import subprocess if not self.__tarballs_info: return None @@ -292,18 +293,54 @@ def find_tarball_for_platform(self, python_version=None): # Auto-detect Python version if not provided if python_version is None: - python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + # Fetch from Sage's Python using ./sage -python + from sage_bootstrap.env import SAGE_ROOT + sage_script = os.path.join(SAGE_ROOT, 'sage') + if not os.path.exists(sage_script): + raise RuntimeError('Sage script not found at: {0}'.format(sage_script)) + + try: + result = subprocess.run( + [sage_script, '-python', '-c', 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'], + capture_output=True, + text=True, + timeout=5, + cwd=SAGE_ROOT + ) + if result.returncode != 0: + raise RuntimeError('Failed to get Python version from sage -python: {0}'.format(result.stderr)) + python_version = result.stdout.strip() + except subprocess.TimeoutExpired: + raise RuntimeError('Timeout while querying Sage Python via ./sage -python') + except Exception as e: + raise RuntimeError('Error querying Sage Python via ./sage -python: {0}'.format(str(e))) py_ver_tuple = tuple(int(x) for x in python_version.split('.')) # Check if running free-threaded Python (Python 3.13+) is_free_threaded = False try: - # Python 3.13+ has sys._is_gil_enabled() to check free-threading mode - if hasattr(sys, '_is_gil_enabled'): - is_free_threaded = not sys._is_gil_enabled() - except Exception: - pass + # Get free-threading status from Sage's Python via ./sage -python + from sage_bootstrap.env import SAGE_ROOT + sage_script = os.path.join(SAGE_ROOT, 'sage') + if not os.path.exists(sage_script): + raise RuntimeError('Sage script not found for free-threading check') + + try: + result = subprocess.run( + [sage_script, '-python', '-c', 'import sys; print(hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled())'], + capture_output=True, + text=True, + timeout=5, + cwd=SAGE_ROOT + ) + if result.returncode != 0: + raise RuntimeError('Failed to check free-threading status: {0}'.format(result.stderr)) + is_free_threaded = result.stdout.strip().lower() == 'true' + except subprocess.TimeoutExpired: + raise RuntimeError('Timeout while checking free-threading status via ./sage -python') + except Exception as e: + raise RuntimeError('Error checking free-threading status: {0}'.format(str(e))) # Get platform info system = platform.system().lower() diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index 1b70f4b4113..a88fb6a2bef 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -191,67 +191,6 @@ def is_platform_specific_wheel(self): return False return not self.filename.endswith('-none-any.whl') - def _download_wheel_with_pip(self, dest_dir): - """ - Download a platform-specific wheel using pip. - - This uses pip's built-in logic to automatically select the appropriate - wheel for the current platform and Python version. - - Returns True on success, False on failure (e.g., offline mode). - """ - try: - import subprocess - import sys - - log.info('Using pip to auto-detect and download wheel for {0}'.format(self.package.name)) - - # Extract package name and version from the package - package_name = self.package.name.replace('_', '-') # PyPI uses dashes - version = self.package.version - package_spec = f"{package_name}=={version}" - - # Let pip automatically detect platform and Python version - cmd = [ - sys.executable, '-m', 'pip', 'download', - package_spec, - '-d', dest_dir, - '--no-deps', - '--only-binary', ':all:' - ] - - log.info(f"Running pip command: {' '.join(cmd)}") - - result = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=60 # 60 second timeout - ) - - if result.returncode == 0: - # Find the downloaded wheel file - wheel_files = [f for f in os.listdir(dest_dir) - if f.endswith('.whl') and package_name.replace('-', '_') in f.lower()] - - if wheel_files: - wheel_path = os.path.join(dest_dir, wheel_files[0]) - log.info(f'Successfully downloaded wheel using pip: {wheel_path}') - return True - else: - log.warning('pip command succeeded but no wheel file found') - return False - else: - log.info(f'pip download failed (possibly offline): {result.stderr}') - return False - - except subprocess.TimeoutExpired: - log.warning('pip download timed out (possibly offline or slow connection)') - return False - except Exception as e: - log.info(f'pip download failed: {e}') - return False - def _find_cached_wheel_for_platform(self): """ Find a cached wheel file that matches the current platform. @@ -365,35 +304,7 @@ def download(self, allow_upstream=False): # update the checksum (Issue #23972). log.warning('Invalid checksum; ignoring cached file {destination}' .format(destination=destination)) - - # For platform-specific wheels, try pip download first - if self.is_platform_specific_wheel(): - log.info('Detected platform-specific wheel: {0}'.format(self.filename)) - dest_dir = os.path.dirname(destination) - - if self._download_wheel_with_pip(dest_dir): - # pip download succeeded, but it may have downloaded a different - # wheel filename (different platform). Find the downloaded wheel. - wheel_files = [f for f in os.listdir(dest_dir) - if f.endswith('.whl') and f.startswith(self.package.name.replace('_', '-'))] - if wheel_files: - actual_wheel = os.path.join(dest_dir, wheel_files[0]) - if actual_wheel != destination: - # Rename or link to expected filename - log.info('Downloaded wheel: {0}'.format(wheel_files[0])) - log.info('Expected filename: {0}'.format(self.filename)) - # For now, just use the downloaded wheel as-is - # The build system should be flexible about the exact filename - # Note: Skip checksum verification for pip-downloaded wheels - # as pip verifies integrity internally - return - else: - log.warning('pip download succeeded but no wheel file found') - - # If pip download failed, fall through to traditional download methods - log.info('Falling back to traditional download methods') - # Traditional download logic for tarballs and platform-independent wheels successful_download = False log.info('Attempting to download package {0} from mirrors'.format(self.filename)) @@ -462,38 +373,8 @@ def _download_multiple_wheels(self, allow_upstream=False): log.warning(f'Error checking cached wheel: {e}') dest_dir = SAGE_DISTFILES - - # Step 2: Try pip download (auto-detection) for platform-specific wheels - if not tarball_filename.endswith('-none-any.whl'): - log.info('Trying pip to auto-detect and download correct wheel...') - if self._download_wheel_with_pip(dest_dir): - # Verify what pip downloaded - cached_wheel = self._find_cached_wheel_for_platform() - if cached_wheel: - try: - downloaded_tarball = Tarball(os.path.basename(cached_wheel), - package=self.package, - tarball_info=tarball_info) - if downloaded_tarball.checksum_verifies(): - log.info(f'Successfully downloaded and verified wheel via pip: {os.path.basename(cached_wheel)}') - # Update self to point to the actually downloaded wheel - self.__filename = os.path.basename(cached_wheel) - return - else: - log.warning('pip-downloaded wheel checksum mismatch, falling back to traditional download') - except Exception as e: - log.warning(f'Error verifying pip-downloaded wheel: {e}') - # pip has its own integrity checks, so we can trust it - log.info('Trusting pip integrity verification') - # Update self to point to the actually downloaded wheel - self.__filename = os.path.basename(cached_wheel) - return - else: - log.warning('pip succeeded but could not find wheel, falling back to traditional download') - else: - log.info('pip download failed (possibly offline), falling back to traditional download') - - # Step 3: Fall back to traditional download from mirrors/upstream + + # Step 2: Try to download from mirrors/upstream log.info(f'Downloading {tarball_filename} using traditional method (mirrors/upstream)...') destination = os.path.join(dest_dir, tarball_filename) upstream_url_pattern = tarball_info.get('upstream_url') From da8cbf3aa2ba283b81de56b97f0dbb5e3314c79e Mon Sep 17 00:00:00 2001 From: cxzhong Date: Fri, 7 Nov 2025 23:49:15 +0800 Subject: [PATCH 06/22] Enhance tarball selection in package.py: utilize packaging.tags for improved compatibility checks and streamline the process of finding the best matching tarball for the current platform and Python version. --- build/sage_bootstrap/package.py | 223 +++++++++----------------------- 1 file changed, 64 insertions(+), 159 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index cdaf7e24707..ca7e4f0f3bf 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -263,13 +263,15 @@ def find_tarball_for_platform(self, python_version=None): Find the appropriate tarball for the current platform. For packages with multiple platform-specific wheels, this selects - the one matching the current platform and Python version. + the one matching the current platform and Python version using + the packaging.tags module to ensure compatibility. Properly handles wheel ABI tags: - - cp313 (CPython 3.13 specific) - - cp313t (CPython 3.13 free-threaded/nogil) - - abi3 (stable ABI, forward compatible) - - pp39 (PyPy 3.9) + - cp313-cp313 (CPython 3.13 specific) + - cp313-cp313t (CPython 3.13 free-threaded/nogil) + - cp313-abi3 (stable ABI, forward compatible) + - py3-none-any (universal pure Python wheel) + - pp39-pypy39_pp73 (PyPy wheels) INPUT: @@ -280,8 +282,6 @@ def find_tarball_for_platform(self, python_version=None): Dictionary with tarball info, or None if no suitable tarball found. The dictionary contains the same fields as tarballs_info entries. """ - import sys - import platform import subprocess if not self.__tarballs_info: @@ -291,175 +291,80 @@ def find_tarball_for_platform(self, python_version=None): if len(self.__tarballs_info) == 1: return self.__tarballs_info[0] - # Auto-detect Python version if not provided - if python_version is None: - # Fetch from Sage's Python using ./sage -python - from sage_bootstrap.env import SAGE_ROOT - sage_script = os.path.join(SAGE_ROOT, 'sage') - if not os.path.exists(sage_script): - raise RuntimeError('Sage script not found at: {0}'.format(sage_script)) - - try: - result = subprocess.run( - [sage_script, '-python', '-c', 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'], - capture_output=True, - text=True, - timeout=5, - cwd=SAGE_ROOT - ) - if result.returncode != 0: - raise RuntimeError('Failed to get Python version from sage -python: {0}'.format(result.stderr)) - python_version = result.stdout.strip() - except subprocess.TimeoutExpired: - raise RuntimeError('Timeout while querying Sage Python via ./sage -python') - except Exception as e: - raise RuntimeError('Error querying Sage Python via ./sage -python: {0}'.format(str(e))) + # Get compatible tags from Sage's Python using packaging.tags + from sage_bootstrap.env import SAGE_ROOT + sage_script = os.path.join(SAGE_ROOT, 'sage') + if not os.path.exists(sage_script): + raise RuntimeError('Sage script not found at: {0}'.format(sage_script)) - py_ver_tuple = tuple(int(x) for x in python_version.split('.')) - - # Check if running free-threaded Python (Python 3.13+) - is_free_threaded = False try: - # Get free-threading status from Sage's Python via ./sage -python - from sage_bootstrap.env import SAGE_ROOT - sage_script = os.path.join(SAGE_ROOT, 'sage') - if not os.path.exists(sage_script): - raise RuntimeError('Sage script not found for free-threading check') + # Get all compatible tags from Sage's Python + result = subprocess.run( + [sage_script, '-python', '-c', + 'import packaging.tags; import json; tags = [str(t) for t in packaging.tags.sys_tags()]; print(json.dumps(tags))'], + capture_output=True, + text=True, + timeout=10, + cwd=SAGE_ROOT + ) + if result.returncode != 0: + raise RuntimeError('Failed to get compatible tags from sage -python: {0}'.format(result.stderr)) - try: - result = subprocess.run( - [sage_script, '-python', '-c', 'import sys; print(hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled())'], - capture_output=True, - text=True, - timeout=5, - cwd=SAGE_ROOT - ) - if result.returncode != 0: - raise RuntimeError('Failed to check free-threading status: {0}'.format(result.stderr)) - is_free_threaded = result.stdout.strip().lower() == 'true' - except subprocess.TimeoutExpired: - raise RuntimeError('Timeout while checking free-threading status via ./sage -python') + import json + compatible_tags = json.loads(result.stdout.strip()) + + except subprocess.TimeoutExpired: + raise RuntimeError('Timeout while querying compatible tags via ./sage -python') except Exception as e: - raise RuntimeError('Error checking free-threading status: {0}'.format(str(e))) + raise RuntimeError('Error querying compatible tags via ./sage -python: {0}'.format(str(e))) - # Get platform info - system = platform.system().lower() - machine = platform.machine().lower() + # Convert tags list to a set for fast lookup with priority + # Lower index = higher priority + tag_priority = {tag: idx for idx, tag in enumerate(compatible_tags)} - # Build platform tags to match against - platform_tags = [] - exclude_tags = [] # Tags that should NOT be in the platform string - if system == 'linux': - if machine == 'x86_64': - platform_tags = ['manylinux', 'musllinux', 'linux_x86_64', 'x86_64'] - exclude_tags = ['aarch64', 'arm64', 'armv7l', 'ppc64', 's390x', 'i686', 'riscv64', 'macosx', 'win'] - elif machine in ['aarch64', 'arm64']: - platform_tags = ['manylinux', 'musllinux', 'linux_aarch64', 'aarch64', 'arm64'] - exclude_tags = ['x86_64', 'armv7l', 'ppc64', 's390x', 'i686', 'riscv64', 'macosx', 'win'] - elif system == 'darwin': - if machine in ['arm64', 'aarch64']: - platform_tags = ['macosx_', 'arm64'] - exclude_tags = ['x86_64', 'i386', 'manylinux', 'musllinux', 'win'] - else: - platform_tags = ['macosx_', 'x86_64'] - exclude_tags = ['arm64', 'i386', 'manylinux', 'musllinux', 'win'] - elif system == 'windows': - if machine == 'amd64' or machine == 'x86_64': - platform_tags = ['win_amd64'] - exclude_tags = ['win32', 'win_arm64', 'manylinux', 'musllinux', 'macosx'] - elif machine == 'arm64': - platform_tags = ['win_arm64'] - exclude_tags = ['win32', 'win_amd64', 'manylinux', 'musllinux', 'macosx'] - else: - platform_tags = ['win32'] - exclude_tags = ['win_amd64', 'win_arm64', 'manylinux', 'musllinux', 'macosx'] - - # Python version tag (e.g., 'cp311' for CPython 3.11) - py_ver_tag = f"cp{python_version.replace('.', '')}" - # Free-threaded ABI tag has 't' suffix (e.g., cp313t in cp313-cp313t) - if is_free_threaded: - abi_tag_free_threaded = py_ver_tag + 't' # e.g., cp313t - else: - abi_tag_free_threaded = None + # Import packaging utilities for parsing wheel filenames + try: + import packaging.utils + except ImportError: + raise RuntimeError('packaging module not available') - # Collect compatible wheels - compatible_wheels = [] + # Find the best matching tarball + best_match = None + best_priority = float('inf') for tarball_info in self.__tarballs_info: tarball = tarball_info['tarball'] - # Check if platform-independent wheel (-none-any.whl) - if tarball.endswith('-none-any.whl'): - compatible_wheels.append((tarball_info, 1000)) # Highest priority - continue - - # Check if not a wheel - use for non-wheel tarballs + # Handle non-wheel tarballs (source distributions) if not tarball.endswith('.whl'): - matches_platform = any(tag in tarball.lower() for tag in platform_tags) - if matches_platform or not platform_tags: - compatible_wheels.append((tarball_info, 500)) - continue - - # Parse wheel filename: {name}-{version}-{python}-{abi}-{platform}.whl - # Extract tags from the wheel filename - parts = tarball.rsplit('-', 3) - if len(parts) != 4: + # Source distributions have lowest priority + if best_priority > len(compatible_tags) + 1000: + best_match = tarball_info + best_priority = len(compatible_tags) + 1000 continue - _, python_tag, abi_tag, platform_tag = parts - platform_tag = platform_tag[:-4] # Remove .whl extension - - # Check if matches current platform (must have a matching tag AND not have any excluded tags) - matches_platform = any(tag in platform_tag.lower() for tag in platform_tags) - has_excluded = any(tag in platform_tag.lower() for tag in exclude_tags) - if not matches_platform or has_excluded: + # Parse wheel filename using packaging.utils + # This properly handles multi-platform wheels like: + # rpds_py-0.28.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + try: + _, _, _, wheel_tags = packaging.utils.parse_wheel_filename(tarball) + except Exception as e: + log.warning(f'Could not parse wheel filename {tarball}: {e}') continue - # Check Python/ABI compatibility - priority = 0 - - # Free-threaded wheel: python_tag=cp313, abi_tag=cp313t - if is_free_threaded and abi_tag_free_threaded and python_tag == py_ver_tag and abi_tag == abi_tag_free_threaded: - priority = 110 # Highest priority for free-threaded wheels on free-threaded Python - # Reject free-threaded ABI on standard Python - elif not is_free_threaded and abi_tag_free_threaded and abi_tag == abi_tag_free_threaded: - priority = 0 # Incompatible - free-threaded wheel on standard Python - # Exact Python version match (e.g., cp313-cp313) - elif py_ver_tag == python_tag and py_ver_tag == abi_tag: - # Standard Python wheel on free-threaded Python - lower priority - if is_free_threaded: - priority = 95 # Can work but prefer free-threaded wheels - else: - priority = 100 - # abi3 wheels - stable ABI, forward compatible - elif 'abi3' in abi_tag: - # Extract minimum Python version from abi3 wheel - # E.g., cp37-abi3 means Python 3.7+ - if python_tag.startswith('cp'): - min_version_str = python_tag[2:] # Remove 'cp' prefix - if len(min_version_str) >= 2: - min_major = int(min_version_str[0]) - min_minor = int(min_version_str[1:]) - min_version = (min_major, min_minor) - - # Check if current Python >= minimum required - if py_ver_tuple >= min_version: - # Lower priority than exact match, but still compatible - priority = 90 - (py_ver_tuple[1] - min_version[1]) # Prefer closer versions - # PyPy wheels (pp39, pp310, etc.) - elif python_tag.startswith('pp') and sys.implementation.name == 'pypy': - if python_tag == f"pp{python_version.replace('.', '')}": - priority = 100 - - if priority > 0: - compatible_wheels.append((tarball_info, priority)) + # Check each tag in the wheel (multi-platform wheels have multiple tags) + for wheel_tag in wheel_tags: + wheel_tag_str = str(wheel_tag) + if wheel_tag_str in tag_priority: + priority = tag_priority[wheel_tag_str] + if priority < best_priority: + best_match = tarball_info + best_priority = priority + log.debug(f'Found compatible wheel: {tarball} with tag {wheel_tag_str} (priority: {priority})') + break # Found a match, no need to check other tags for this wheel - # Sort by priority (highest first) and return the best match - if compatible_wheels: - compatible_wheels.sort(key=lambda x: x[1], reverse=True) - best_match = compatible_wheels[0][0] - threading_mode = "free-threaded" if is_free_threaded else "standard" - log.debug(f'Selected {best_match["tarball"]} for platform ({threading_mode} Python, priority: {compatible_wheels[0][1]})') + if best_match: + log.debug(f'Selected {best_match["tarball"]} with priority {best_priority}') return best_match # If no match found, return the first one (backward compatibility) From aba2315f0c93422b04a29bc2498cfa7cec779b5f Mon Sep 17 00:00:00 2001 From: cxzhong Date: Sat, 8 Nov 2025 00:11:56 +0800 Subject: [PATCH 07/22] Update dependency specification for rpds_py to include 'packaging' for improved functionality --- build/pkgs/rpds_py/dependencies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/pkgs/rpds_py/dependencies b/build/pkgs/rpds_py/dependencies index a1b589e38a3..a05443f5c4f 100644 --- a/build/pkgs/rpds_py/dependencies +++ b/build/pkgs/rpds_py/dependencies @@ -1 +1 @@ -pip +pip packaging From 1479c77e33ee6e690eaa06232856e9236a68cdc6 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Sat, 8 Nov 2025 00:42:59 +0800 Subject: [PATCH 08/22] Remove platform-specific wheel checks from Package and Tarball classes --- build/sage_bootstrap/package.py | 17 ----- build/sage_bootstrap/tarball.py | 122 ++++---------------------------- 2 files changed, 15 insertions(+), 124 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index ca7e4f0f3bf..173396ac4c8 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -486,23 +486,6 @@ def source(self): return 'script' return 'none' - def is_platform_specific_wheel(self): - """ - Check if this package uses a platform-specific wheel. - - Platform-specific wheels need special handling during download - as they contain platform tags in the filename. - - Returns True if this is a wheel package with platform-specific binaries, - False otherwise. - """ - if self.source != 'wheel': - return False - if not self.tarball_filename: - return False - # Platform-independent wheels end with -none-any.whl - return not self.tarball_filename.endswith('-none-any.whl') - @property def trees(self): """ diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index a88fb6a2bef..e94450ec910 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -176,96 +176,6 @@ def checksum_verifies(self, force_sha256=False): def is_distributable(self): return 'do-not-distribute' not in self.filename - def is_platform_specific_wheel(self): - """ - Check if this is a platform-specific wheel. - - Platform-specific wheels have platform tags like: - - manylinux_2_17_x86_64 - - macosx_11_0_arm64 - - win_amd64 - - Platform-independent wheels end with -none-any.whl - """ - if not self.filename or not self.filename.endswith('.whl'): - return False - return not self.filename.endswith('-none-any.whl') - - def _find_cached_wheel_for_platform(self): - """ - Find a cached wheel file that matches the current platform. - - Looks in SAGE_DISTFILES for any wheel matching the package name, - version, and current platform/Python version. - - Returns the path to the cached wheel if found, None otherwise. - """ - import platform - - # Get platform info - python_version = f"{sys.version_info.major}.{sys.version_info.minor}" - py_tag = f"cp{python_version.replace('.', '')}" - - system = platform.system().lower() - machine = platform.machine().lower() - - # Build possible platform tags - platform_patterns = [] - if system == 'linux': - if machine == 'x86_64': - platform_patterns = ['manylinux', 'linux_x86_64'] - elif machine in ['aarch64', 'arm64']: - platform_patterns = ['manylinux', 'linux_aarch64'] - elif system == 'darwin': - if machine in ['arm64', 'aarch64']: - platform_patterns = ['macosx', 'arm64'] - else: - platform_patterns = ['macosx', 'x86_64'] - - # Look for matching wheel files in upstream directory - # Note: Wheel filenames use underscores, not dashes - pkg_name_wheel = self.package.name.replace('-', '_').lower() - pkg_name_pypi = self.package.name.replace('_', '-').lower() - pkg_version = self.package.version - - log.debug(f'Looking for cached wheel: pkg_name_wheel={pkg_name_wheel}, pkg_name_pypi={pkg_name_pypi}, version={pkg_version}, py_tag={py_tag}') - log.debug(f'Platform patterns: {platform_patterns}') - - try: - for filename in os.listdir(SAGE_DISTFILES): - if not filename.endswith('.whl'): - continue - - filename_lower = filename.lower() - - log.debug(f'Checking file: {filename}') - - # Check if matches package name and version - # Wheel filenames use underscores, not dashes - if not (filename_lower.startswith(pkg_name_wheel) or filename_lower.startswith(pkg_name_pypi)): - log.debug(f' Does not start with {pkg_name_wheel} or {pkg_name_pypi}') - continue - if pkg_version not in filename: - log.debug(f' Does not contain version {pkg_version}') - continue - - # Check if matches current platform - matches_platform = any(p in filename_lower for p in platform_patterns) - matches_python = py_tag in filename - - log.debug(f' matches_platform={matches_platform}, matches_python={matches_python}') - - if matches_platform and matches_python: - wheel_path = os.path.join(SAGE_DISTFILES, filename) - log.info(f'Found cached wheel for platform: {filename}') - return wheel_path - except OSError as e: - log.warning(f'Error listing directory {SAGE_DISTFILES}: {e}') - pass - - log.warning(f'No cached wheel found for {pkg_name_wheel}/{pkg_name_pypi}-{pkg_version} (py_tag={py_tag}, platform={platform_patterns})') - return None - def download(self, allow_upstream=False): """ Download the tarball to the upstream directory. @@ -338,9 +248,9 @@ def _download_multiple_wheels(self, allow_upstream=False): Handle download for packages with multiple platform-specific wheels. Strategy: - 1. Check if already cached with valid checksum - 2. If not cached, try pip download (auto-detects platform/Python version) - 3. If pip fails (offline/error), fall back to traditional download from mirrors/upstream + 1. Use find_tarball_for_platform() to get the exact filename for this platform + 2. Check if file is already cached with valid checksum + 3. If not cached, download from mirrors/upstream """ # Find the appropriate tarball for this platform from checksums.ini tarball_info = self.package.find_tarball_for_platform() @@ -354,29 +264,27 @@ def _download_multiple_wheels(self, allow_upstream=False): log.info(f'Selected tarball for platform: {tarball_filename}') - # Step 1: Check cache first - cached_wheel = self._find_cached_wheel_for_platform() - if cached_wheel: - # Verify checksum of cached wheel + # Check if file already exists in cache + destination = os.path.join(SAGE_DISTFILES, tarball_filename) + + if os.path.isfile(destination): + # Verify checksum of cached file try: - cached_tarball = Tarball(os.path.basename(cached_wheel), + cached_tarball = Tarball(tarball_filename, package=self.package, tarball_info=tarball_info) if cached_tarball.checksum_verifies(): - log.info(f'Using cached wheel with valid checksum: {os.path.basename(cached_wheel)}') + log.info(f'Using cached file with valid checksum: {tarball_filename}') # Update self to point to the cached wheel - self.__filename = os.path.basename(cached_wheel) + self.__filename = tarball_filename return else: - log.warning('Cached wheel has invalid checksum, will re-download') + log.warning(f'Cached file {tarball_filename} has invalid checksum, will re-download') except Exception as e: - log.warning(f'Error checking cached wheel: {e}') + log.warning(f'Error checking cached file: {e}') - dest_dir = SAGE_DISTFILES - - # Step 2: Try to download from mirrors/upstream - log.info(f'Downloading {tarball_filename} using traditional method (mirrors/upstream)...') - destination = os.path.join(dest_dir, tarball_filename) + # Download from mirrors/upstream + log.info(f'Downloading {tarball_filename} from mirrors/upstream...') upstream_url_pattern = tarball_info.get('upstream_url') if upstream_url_pattern: From e9a145c1817c67ffbb5c87a843117c379c8150b6 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Sat, 8 Nov 2025 15:15:19 +0800 Subject: [PATCH 09/22] Refactor tarball parsing in package.py to utilize subprocess for packaging.utils, improving compatibility with multi-platform wheels. --- build/sage_bootstrap/package.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 173396ac4c8..821e9a417fc 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -322,12 +322,6 @@ def find_tarball_for_platform(self, python_version=None): # Lower index = higher priority tag_priority = {tag: idx for idx, tag in enumerate(compatible_tags)} - # Import packaging utilities for parsing wheel filenames - try: - import packaging.utils - except ImportError: - raise RuntimeError('packaging module not available') - # Find the best matching tarball best_match = None best_priority = float('inf') @@ -343,18 +337,33 @@ def find_tarball_for_platform(self, python_version=None): best_priority = len(compatible_tags) + 1000 continue - # Parse wheel filename using packaging.utils + # Parse wheel filename using packaging.utils via Sage's Python # This properly handles multi-platform wheels like: # rpds_py-0.28.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl try: - _, _, _, wheel_tags = packaging.utils.parse_wheel_filename(tarball) + python_code = 'import packaging.utils, json, sys; _, _, _, tags = packaging.utils.parse_wheel_filename(sys.argv[1]); print(json.dumps([str(t) for t in tags]))' + result = subprocess.run( + [sage_script, '-python', '-c', python_code, tarball], + capture_output=True, + text=True, + timeout=10, + cwd=SAGE_ROOT + ) + if result.returncode != 0: + log.warning(f'Could not parse wheel filename {tarball}') + continue + + wheel_tags_str = json.loads(result.stdout.strip()) + + except subprocess.TimeoutExpired: + log.warning(f'Timeout while parsing wheel filename {tarball}') + continue except Exception as e: log.warning(f'Could not parse wheel filename {tarball}: {e}') continue # Check each tag in the wheel (multi-platform wheels have multiple tags) - for wheel_tag in wheel_tags: - wheel_tag_str = str(wheel_tag) + for wheel_tag_str in wheel_tags_str: if wheel_tag_str in tag_priority: priority = tag_priority[wheel_tag_str] if priority < best_priority: From 5e3fd077840b5ef942e5c0360c86cdd972eb28b9 Mon Sep 17 00:00:00 2001 From: Chenxin Zhong Date: Tue, 11 Nov 2025 15:43:18 +0800 Subject: [PATCH 10/22] Use compatible write for old python --- build/sage_bootstrap/package.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 821e9a417fc..79113f61345 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -302,7 +302,9 @@ def find_tarball_for_platform(self, python_version=None): result = subprocess.run( [sage_script, '-python', '-c', 'import packaging.tags; import json; tags = [str(t) for t in packaging.tags.sys_tags()]; print(json.dumps(tags))'], - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, text=True, timeout=10, cwd=SAGE_ROOT @@ -344,7 +346,9 @@ def find_tarball_for_platform(self, python_version=None): python_code = 'import packaging.utils, json, sys; _, _, _, tags = packaging.utils.parse_wheel_filename(sys.argv[1]); print(json.dumps([str(t) for t in tags]))' result = subprocess.run( [sage_script, '-python', '-c', python_code, tarball], - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, text=True, timeout=10, cwd=SAGE_ROOT From dc2ed41fa559bcdcaae1bccb92f8c6e6053b6bf1 Mon Sep 17 00:00:00 2001 From: Chenxin Zhong Date: Tue, 11 Nov 2025 15:44:32 +0800 Subject: [PATCH 11/22] write the comment Updated the download fallback method in tarball.py. --- build/sage_bootstrap/tarball.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index e94450ec910..342b96245b9 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -187,8 +187,7 @@ def download(self, allow_upstream=False): For platform-specific wheels, this method: 1. Checks for a cached wheel matching the current platform 2. If cached and checksum valid, uses it - 3. Otherwise, uses pip download to get the correct wheel - 4. Falls back to traditional download if pip fails + 3. Otherwise, download from upstream """ if not self.filename: raise ValueError('non-normal package does define a tarball, so cannot download') From b51fe59305da2dcdaa6da8b9801fe7709726ec88 Mon Sep 17 00:00:00 2001 From: Chenxin Zhong Date: Tue, 11 Nov 2025 16:14:24 +0800 Subject: [PATCH 12/22] Remove 'text=True' from subprocess calls Removed 'text=True' argument from subprocess calls. --- build/sage_bootstrap/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 79113f61345..015552dd7f3 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -305,7 +305,6 @@ def find_tarball_for_platform(self, python_version=None): stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, - text=True, timeout=10, cwd=SAGE_ROOT ) @@ -349,7 +348,6 @@ def find_tarball_for_platform(self, python_version=None): stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, - text=True, timeout=10, cwd=SAGE_ROOT ) From f102156998676f5823fed91552df8ca39923e6c8 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Thu, 13 Nov 2025 13:52:05 +0800 Subject: [PATCH 13/22] Add support for downloading all wheels for distribution in Tarball class --- build/make/Makefile.in | 2 +- build/sage_bootstrap/tarball.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/build/make/Makefile.in b/build/make/Makefile.in index ff7ed18135e..6f15442ac5e 100644 --- a/build/make/Makefile.in +++ b/build/make/Makefile.in @@ -280,7 +280,7 @@ all-sage-docs: $(SAGE_DOCS_INSTALLED_PACKAGE_INSTS) $(SAGE_DOCS_UNINSTALLED_PAC # Download all packages which should be inside an sdist tarball (the -B # option to make forces all targets to be built unconditionally) download-for-sdist: - +env SAGE_INSTALL_FETCH_ONLY=yes $(MAKE_REC) -B SAGERUNTIME= \ + +env SAGE_INSTALL_FETCH_ONLY=yes SAGE_DOWNLOAD_ALL_WHEELS=yes $(MAKE_REC) -B SAGERUNTIME= \ $(SDIST_PACKAGES) # TOOLCHAIN consists of dependencies determined by configure. diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index 342b96245b9..cba840be4a0 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -247,16 +247,40 @@ def _download_multiple_wheels(self, allow_upstream=False): Handle download for packages with multiple platform-specific wheels. Strategy: - 1. Use find_tarball_for_platform() to get the exact filename for this platform - 2. Check if file is already cached with valid checksum - 3. If not cached, download from mirrors/upstream + 1. If SAGE_DOWNLOAD_ALL_WHEELS is set, download all wheels (for make dist) + 2. Otherwise, use find_tarball_for_platform() to get the exact filename for this platform + 3. Check if file is already cached with valid checksum + 4. If not cached, download from mirrors/upstream """ + download_all = os.environ.get('SAGE_DOWNLOAD_ALL_WHEELS', '').lower() in ['yes', '1', 'true'] + + if download_all: + # Download all wheels for distribution + log.info(f'Downloading all {len(self.package.tarballs_info)} wheels for {self.package.name}') + for tarball_info in self.package.tarballs_info: + self._download_single_wheel(tarball_info, allow_upstream) + # Keep self.__filename pointing to the first one for compatibility + tarball_pattern = self.package.tarballs_info[0]['tarball'] + self.__filename = self.package._substitute_variables(tarball_pattern) + return + # Find the appropriate tarball for this platform from checksums.ini tarball_info = self.package.find_tarball_for_platform() if not tarball_info: raise ValueError(f'No suitable tarball found for {self.package.name} on current platform') + # Download the single platform-specific wheel + self._download_single_wheel(tarball_info, allow_upstream) + + def _download_single_wheel(self, tarball_info, allow_upstream=False): + """ + Download a single wheel given its tarball info. + + INPUT: + - ``tarball_info`` - dict with tarball, sha256, sha1, upstream_url + - ``allow_upstream`` - whether to allow downloading from upstream + """ # Get the actual filename with version substituted tarball_pattern = tarball_info['tarball'] tarball_filename = self.package._substitute_variables(tarball_pattern) From 3e7f0ffef047124de4610c267ad9c4707b2fbb43 Mon Sep 17 00:00:00 2001 From: Chenxin Zhong Date: Thu, 13 Nov 2025 14:08:30 +0800 Subject: [PATCH 14/22] Update release.yml Removed steps for making distribution and downloading packages with '--disable-download-from-upstream-url'. --- .github/workflows/release.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29e0ad23363..dc9b2dc96cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,20 +38,7 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap _prereq) - - name: make dist (--disable-download-from-upstream-url) - id: make_dist - run: | - ./bootstrap -D && ./configure --disable-download-from-upstream-url && make dist - env: - MAKE: make -j8 - - name: make download (--disable-download-from-upstream-url) - id: make_download - run: | - make -k download DOWNLOAD_PACKAGES=":all: --no-file huge" - env: - MAKE: make -j8 - name: Reconfigure with --enable-download-from-upstream-url - if: (success() || failure()) && (steps.make_dist.outcome != 'success' || steps.make_download.outcome != 'success') run: | ./configure - name: make dist (--enable-download-from-upstream-url) From 0378ce7e633c2504c7ce6f91287fcb1625717fe4 Mon Sep 17 00:00:00 2001 From: Chenxin Zhong Date: Thu, 13 Nov 2025 14:12:07 +0800 Subject: [PATCH 15/22] Update release.yml --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dc9b2dc96cf..f2f8d12a433 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,7 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap _prereq) - name: Reconfigure with --enable-download-from-upstream-url run: | + make configure ./configure - name: make dist (--enable-download-from-upstream-url) if: (success() || failure()) && steps.make_dist.outcome != 'success' From af01d2cfe42e39e7df0b31503b5543763295241b Mon Sep 17 00:00:00 2001 From: Chenxin Zhong Date: Thu, 13 Nov 2025 16:06:55 +0800 Subject: [PATCH 16/22] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/workflows/release.yml | 3 ++- build/pkgs/rpds_py/SPKG.rst | 4 ++-- build/sage_bootstrap/package.py | 2 +- build/sage_bootstrap/tarball.py | 9 ++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f2f8d12a433..343cbb11c58 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,7 +43,8 @@ jobs: make configure ./configure - name: make dist (--enable-download-from-upstream-url) - if: (success() || failure()) && steps.make_dist.outcome != 'success' + id: make_dist + if: success() || failure() run: | make dist env: diff --git a/build/pkgs/rpds_py/SPKG.rst b/build/pkgs/rpds_py/SPKG.rst index 4dc3bbfdc16..aef5d3c2ead 100644 --- a/build/pkgs/rpds_py/SPKG.rst +++ b/build/pkgs/rpds_py/SPKG.rst @@ -9,7 +9,7 @@ Python bindings to the Rust rpds crate for persistent data structures. rpds-py provides efficient, immutable data structures including: * ``HashTrieMap`` - Persistent hash map -* ``HashTrieSet`` - Persistent hash set +* ``HashTrieSet`` - Persistent hash set * ``List`` - Persistent list with efficient operations These data structures are backed by Rust implementations for high performance @@ -53,7 +53,7 @@ versions and platforms: * Windows (win32, win_amd64, win_arm64) The Sage build system automatically selects and downloads the appropriate -wheel for your platform and Python version using pip's auto-detection. +wheel for your platform and Python version using the packaging library's compatibility tags. When building from source, a Rust toolchain is required as rpds-py contains Rust extensions for performance. diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 015552dd7f3..3aa91b830f1 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -258,7 +258,7 @@ def tarballs_info(self): """ return self.__tarballs_info - def find_tarball_for_platform(self, python_version=None): + def find_tarball_for_platform(self): """ Find the appropriate tarball for the current platform. diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index cba840be4a0..e247331aeba 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -16,8 +16,7 @@ # **************************************************************************** import os -import sys -import subprocess + import logging log = logging.getLogger() @@ -60,7 +59,7 @@ def __init__(self, tarball_name, package=None, tarball_info=None): """ self.__filename = tarball_name self.__tarball_info = tarball_info - + if package is None: self.__package = None for pkg in Package.all(): @@ -343,7 +342,7 @@ def _download_single_wheel(self, tarball_info, allow_upstream=False): log.debug('File not at upstream URL') if not successful_download: - raise FileNotMirroredError(f'Could not download {tarball_filename} from pip, mirrors, or upstream') + raise FileNotMirroredError(f'Could not download {tarball_filename} from mirrors or upstream') # Verify checksum of traditionally-downloaded file try: @@ -355,7 +354,7 @@ def _download_single_wheel(self, tarball_info, allow_upstream=False): log.info(f'Successfully downloaded and verified: {tarball_filename}') # Update self to point to the actually downloaded file self.__filename = tarball_filename - except Exception as e: + except (ChecksumError, IOError, ValueError) as e: log.error(f'Error verifying downloaded tarball: {e}') raise From d7ff54e45073003ab2bdf87ae38a4283e28d8fe0 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Thu, 13 Nov 2025 20:28:55 +0800 Subject: [PATCH 17/22] Refactor tarball selection logic to prioritize wheel tarballs and improve parsing efficiency --- build/sage_bootstrap/package.py | 67 +++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 015552dd7f3..ddb91f8a6a1 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -323,45 +323,51 @@ def find_tarball_for_platform(self, python_version=None): # Lower index = higher priority tag_priority = {tag: idx for idx, tag in enumerate(compatible_tags)} - # Find the best matching tarball - best_match = None - best_priority = float('inf') - + # Separate wheels from non-wheel tarballs + wheel_tarballs = [] + source_tarballs = [] for tarball_info in self.__tarballs_info: - tarball = tarball_info['tarball'] - - # Handle non-wheel tarballs (source distributions) - if not tarball.endswith('.whl'): - # Source distributions have lowest priority - if best_priority > len(compatible_tags) + 1000: - best_match = tarball_info - best_priority = len(compatible_tags) + 1000 - continue - - # Parse wheel filename using packaging.utils via Sage's Python - # This properly handles multi-platform wheels like: - # rpds_py-0.28.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + if tarball_info['tarball'].endswith('.whl'): + wheel_tarballs.append(tarball_info) + else: + source_tarballs.append(tarball_info) + + # Batch parse all wheel filenames in a single subprocess call + # This avoids subprocess overhead for packages with many wheels + wheel_tags_map = {} + if wheel_tarballs: try: - python_code = 'import packaging.utils, json, sys; _, _, _, tags = packaging.utils.parse_wheel_filename(sys.argv[1]); print(json.dumps([str(t) for t in tags]))' + wheel_filenames = [info['tarball'] for info in wheel_tarballs] + python_code = 'import packaging.utils as pu,json,sys;d={};[exec(f"try: d[f]=[str(t)for t in pu.parse_wheel_filename(f)[3]]\\nexcept: d[f]=None",{"f":f,"d":d,"pu":pu})for f in sys.argv[1:]];print(json.dumps(d))' result = subprocess.run( - [sage_script, '-python', '-c', python_code, tarball], + [sage_script, '-python', '-c', python_code] + wheel_filenames, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, - timeout=10, + timeout=30, cwd=SAGE_ROOT ) if result.returncode != 0: - log.warning(f'Could not parse wheel filename {tarball}') - continue - - wheel_tags_str = json.loads(result.stdout.strip()) - + log.warning(f'Failed to parse wheel filenames: {result.stderr}') + else: + wheel_tags_map = json.loads(result.stdout.strip()) + except subprocess.TimeoutExpired: - log.warning(f'Timeout while parsing wheel filename {tarball}') - continue + log.warning('Timeout while parsing wheel filenames') except Exception as e: - log.warning(f'Could not parse wheel filename {tarball}: {e}') + log.warning(f'Error parsing wheel filenames: {e}') + + # Find the best matching tarball + best_match = None + best_priority = float('inf') + + # Check wheel tarballs first (they have higher priority than source) + for tarball_info in wheel_tarballs: + tarball = tarball_info['tarball'] + wheel_tags_str = wheel_tags_map.get(tarball) + + if wheel_tags_str is None: + log.debug(f'Could not parse wheel filename {tarball}') continue # Check each tag in the wheel (multi-platform wheels have multiple tags) @@ -374,6 +380,11 @@ def find_tarball_for_platform(self, python_version=None): log.debug(f'Found compatible wheel: {tarball} with tag {wheel_tag_str} (priority: {priority})') break # Found a match, no need to check other tags for this wheel + # If no wheel matched, consider source distributions + if best_match is None and source_tarballs: + best_match = source_tarballs[0] + best_priority = len(compatible_tags) + 1000 + if best_match: log.debug(f'Selected {best_match["tarball"]} with priority {best_priority}') return best_match From 5a6baedd68c1a2753efca28e04d9ceba8c8f1d80 Mon Sep 17 00:00:00 2001 From: Chenxin Zhong Date: Thu, 13 Nov 2025 16:06:55 +0800 Subject: [PATCH 18/22] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/workflows/release.yml | 3 ++- build/pkgs/rpds_py/SPKG.rst | 4 ++-- build/sage_bootstrap/package.py | 2 +- build/sage_bootstrap/tarball.py | 9 ++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f2f8d12a433..343cbb11c58 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,7 +43,8 @@ jobs: make configure ./configure - name: make dist (--enable-download-from-upstream-url) - if: (success() || failure()) && steps.make_dist.outcome != 'success' + id: make_dist + if: success() || failure() run: | make dist env: diff --git a/build/pkgs/rpds_py/SPKG.rst b/build/pkgs/rpds_py/SPKG.rst index 4dc3bbfdc16..aef5d3c2ead 100644 --- a/build/pkgs/rpds_py/SPKG.rst +++ b/build/pkgs/rpds_py/SPKG.rst @@ -9,7 +9,7 @@ Python bindings to the Rust rpds crate for persistent data structures. rpds-py provides efficient, immutable data structures including: * ``HashTrieMap`` - Persistent hash map -* ``HashTrieSet`` - Persistent hash set +* ``HashTrieSet`` - Persistent hash set * ``List`` - Persistent list with efficient operations These data structures are backed by Rust implementations for high performance @@ -53,7 +53,7 @@ versions and platforms: * Windows (win32, win_amd64, win_arm64) The Sage build system automatically selects and downloads the appropriate -wheel for your platform and Python version using pip's auto-detection. +wheel for your platform and Python version using the packaging library's compatibility tags. When building from source, a Rust toolchain is required as rpds-py contains Rust extensions for performance. diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index ddb91f8a6a1..fb8f51c2633 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -258,7 +258,7 @@ def tarballs_info(self): """ return self.__tarballs_info - def find_tarball_for_platform(self, python_version=None): + def find_tarball_for_platform(self): """ Find the appropriate tarball for the current platform. diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index cba840be4a0..e247331aeba 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -16,8 +16,7 @@ # **************************************************************************** import os -import sys -import subprocess + import logging log = logging.getLogger() @@ -60,7 +59,7 @@ def __init__(self, tarball_name, package=None, tarball_info=None): """ self.__filename = tarball_name self.__tarball_info = tarball_info - + if package is None: self.__package = None for pkg in Package.all(): @@ -343,7 +342,7 @@ def _download_single_wheel(self, tarball_info, allow_upstream=False): log.debug('File not at upstream URL') if not successful_download: - raise FileNotMirroredError(f'Could not download {tarball_filename} from pip, mirrors, or upstream') + raise FileNotMirroredError(f'Could not download {tarball_filename} from mirrors or upstream') # Verify checksum of traditionally-downloaded file try: @@ -355,7 +354,7 @@ def _download_single_wheel(self, tarball_info, allow_upstream=False): log.info(f'Successfully downloaded and verified: {tarball_filename}') # Update self to point to the actually downloaded file self.__filename = tarball_filename - except Exception as e: + except (ChecksumError, IOError, ValueError) as e: log.error(f'Error verifying downloaded tarball: {e}') raise From 448f5eeefb33bd831563e1bf549f87ee741254d2 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Thu, 13 Nov 2025 23:13:34 +0800 Subject: [PATCH 19/22] Remove redundant json import and add it at the top of the find_tarball_for_platform method --- build/sage_bootstrap/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index fb8f51c2633..57052e2b21f 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -283,6 +283,7 @@ def find_tarball_for_platform(self): The dictionary contains the same fields as tarballs_info entries. """ import subprocess + import json if not self.__tarballs_info: return None @@ -311,7 +312,6 @@ def find_tarball_for_platform(self): if result.returncode != 0: raise RuntimeError('Failed to get compatible tags from sage -python: {0}'.format(result.stderr)) - import json compatible_tags = json.loads(result.stdout.strip()) except subprocess.TimeoutExpired: From 7d67167cff9a8ac07df0033ee053679be4033ffb Mon Sep 17 00:00:00 2001 From: cxzhong Date: Thu, 13 Nov 2025 23:15:26 +0800 Subject: [PATCH 20/22] Reorder import statements in find_tarball_for_platform method for clarity --- build/sage_bootstrap/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 57052e2b21f..5c16eba623e 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -282,8 +282,10 @@ def find_tarball_for_platform(self): Dictionary with tarball info, or None if no suitable tarball found. The dictionary contains the same fields as tarballs_info entries. """ - import subprocess import json + import subprocess + + from sage_bootstrap.env import SAGE_ROOT if not self.__tarballs_info: return None @@ -293,7 +295,6 @@ def find_tarball_for_platform(self): return self.__tarballs_info[0] # Get compatible tags from Sage's Python using packaging.tags - from sage_bootstrap.env import SAGE_ROOT sage_script = os.path.join(SAGE_ROOT, 'sage') if not os.path.exists(sage_script): raise RuntimeError('Sage script not found at: {0}'.format(sage_script)) From a032ac8ef4261fc5c55dc151b777127936bcc3c4 Mon Sep 17 00:00:00 2001 From: cxzhong Date: Sun, 16 Nov 2025 20:13:23 +0800 Subject: [PATCH 21/22] pass package rpds_py in the disable-notebook --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 12c3e99e6d2..e91615f2dc6 100644 --- a/configure.ac +++ b/configure.ac @@ -506,7 +506,7 @@ AC_ARG_ENABLE([cvxopt], AC_ARG_ENABLE([notebook], AS_HELP_STRING([--disable-notebook], [disable build of the Jupyter notebook and related packages]), [ - for pkg in notebook nbconvert beautifulsoup4 sagenb_export nbformat nbclient terminado send2trash prometheus_client mistune pandocfilters bleach defusedxml jsonschema jupyter_jsmol argon2_cffi argon2_cffi_bindings webencodings tinycss2 ipympl soupsieve fastjsonschema anyio arrow async_lru fqdn isoduration json5 jsonpointer jsonschema_specifications jupyter_events jupyter_lsp jupyter_server jupyter_server_terminals jupyterlab jupyterlab_server jupyterlab_pygments jupyterlab_mathjax2 jupyter_sphinx notebook_shim overrides python_json_logger pyyaml referencing rfc3339_validator rfc3986_validator sniffio types_python_dateutil uri_template webcolors websocket_client httpx httpcore h11; do + for pkg in notebook nbconvert beautifulsoup4 sagenb_export nbformat nbclient terminado send2trash prometheus_client mistune pandocfilters bleach defusedxml jsonschema jupyter_jsmol argon2_cffi argon2_cffi_bindings webencodings tinycss2 ipympl soupsieve fastjsonschema anyio arrow async_lru fqdn isoduration json5 jsonpointer jsonschema_specifications jupyter_events jupyter_lsp jupyter_server jupyter_server_terminals jupyterlab jupyterlab_server jupyterlab_pygments jupyterlab_mathjax2 jupyter_sphinx notebook_shim overrides python_json_logger pyyaml referencing rfc3339_validator rfc3986_validator sniffio types_python_dateutil uri_template webcolors websocket_client httpx httpcore h11 rpds_py; do AS_VAR_SET([SAGE_ENABLE_$pkg], [$enableval]) done ]) From aa30c171ac51ed8243753d59a010bc2106584d9c Mon Sep 17 00:00:00 2001 From: cxzhong Date: Tue, 18 Nov 2025 21:46:46 +0800 Subject: [PATCH 22/22] Refactor the download_single_wheel as the previous download logic --- build/sage_bootstrap/tarball.py | 88 ++++++++++++++------------------- 1 file changed, 37 insertions(+), 51 deletions(-) diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index e247331aeba..0cfd2a0e9d2 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -289,74 +289,60 @@ def _download_single_wheel(self, tarball_info, allow_upstream=False): # Check if file already exists in cache destination = os.path.join(SAGE_DISTFILES, tarball_filename) - if os.path.isfile(destination): - # Verify checksum of cached file - try: - cached_tarball = Tarball(tarball_filename, - package=self.package, - tarball_info=tarball_info) - if cached_tarball.checksum_verifies(): - log.info(f'Using cached file with valid checksum: {tarball_filename}') - # Update self to point to the cached wheel - self.__filename = tarball_filename - return - else: - log.warning(f'Cached file {tarball_filename} has invalid checksum, will re-download') - except Exception as e: - log.warning(f'Error checking cached file: {e}') + # Create a temporary Tarball object for this specific wheel + wheel_tarball = Tarball(tarball_filename, + package=self.package, + tarball_info=tarball_info) - # Download from mirrors/upstream - log.info(f'Downloading {tarball_filename} from mirrors/upstream...') - upstream_url_pattern = tarball_info.get('upstream_url') - - if upstream_url_pattern: - upstream_url = self.package._substitute_variables(upstream_url_pattern) - else: - upstream_url = None + # Check if file already exists and is valid + if os.path.isfile(destination): + if wheel_tarball.checksum_verifies(): + log.info('Using cached file {destination}'.format(destination=destination)) + # Update self to point to the cached wheel + self.__filename = tarball_filename + self.__tarball_info = tarball_info + return + else: + # Garbage in the upstream directory? Ignore it. + # Don't delete it because maybe somebody just forgot to + # update the checksum (Issue #23972). + log.warning('Invalid checksum; ignoring cached file {destination}' + .format(destination=destination)) + # Download logic for platform-specific wheels successful_download = False - - # Try mirrors first + log.info('Attempting to download package {0} from mirrors'.format(tarball_filename)) for mirror in MirrorList(): url = mirror.replace('${SPKG}', self.package.name) if not url.endswith('/'): url += '/' url += tarball_filename - log.debug(f'Trying mirror: {url}') + log.info(url) try: Download(url, destination).run() successful_download = True - log.info(f'Downloaded from mirror: {url}') break except IOError: log.debug('File not on mirror') - # Try upstream if mirrors failed - if not successful_download and upstream_url and allow_upstream: - log.info(f'Trying upstream: {upstream_url}') - try: - Download(upstream_url, destination).run() - successful_download = True - log.info(f'Downloaded from upstream: {upstream_url}') - except IOError: - log.debug('File not at upstream URL') - if not successful_download: - raise FileNotMirroredError(f'Could not download {tarball_filename} from mirrors or upstream') + upstream_url_pattern = tarball_info.get('upstream_url') + url = self.package._substitute_variables(upstream_url_pattern) if upstream_url_pattern else None + if allow_upstream and url: + log.info('Attempting to download from {}'.format(url)) + try: + Download(url, destination).run() + except IOError: + raise FileNotMirroredError('tarball does not exist on mirror network and neither at the upstream URL') + else: + raise FileNotMirroredError('tarball does not exist on mirror network') - # Verify checksum of traditionally-downloaded file - try: - downloaded_tarball = Tarball(tarball_filename, - package=self.package, - tarball_info=tarball_info) - if not downloaded_tarball.checksum_verifies(): - raise ChecksumError(f'Checksum verification failed for {tarball_filename}') - log.info(f'Successfully downloaded and verified: {tarball_filename}') - # Update self to point to the actually downloaded file - self.__filename = tarball_filename - except (ChecksumError, IOError, ValueError) as e: - log.error(f'Error verifying downloaded tarball: {e}') - raise + if not wheel_tarball.checksum_verifies(): + raise ChecksumError('checksum does not match') + + # Update self to point to the successfully downloaded wheel + self.__filename = tarball_filename + self.__tarball_info = tarball_info def save_as(self, destination): """