From d073ee36fd99ccec672f8d1862cbc3e4c80491c1 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Mon, 5 May 2025 20:39:11 +0200 Subject: [PATCH 1/6] Python packaging and wheel build --- .github/workflows/build-windows.yaml | 7 + .../20250505-7bf6e1df-python-packaging.patch | 123 ++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 patches/20250505-7bf6e1df-python-packaging.patch diff --git a/.github/workflows/build-windows.yaml b/.github/workflows/build-windows.yaml index 7216837e23..a0cad8f6d6 100644 --- a/.github/workflows/build-windows.yaml +++ b/.github/workflows/build-windows.yaml @@ -134,6 +134,13 @@ jobs: - name: Build git-annex run: stack install --no-haddock --local-bin-path . + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build the Python wheel + run: | + uv build --wheel + - name: Build the installer run: | stack ghc --no-haddock --package nsis Build/NullSoftInstaller.hs diff --git a/patches/20250505-7bf6e1df-python-packaging.patch b/patches/20250505-7bf6e1df-python-packaging.patch new file mode 100644 index 0000000000..7479b5d88b --- /dev/null +++ b/patches/20250505-7bf6e1df-python-packaging.patch @@ -0,0 +1,123 @@ +diff --git a/pyproject.toml b/pyproject.toml +new file mode 100644 +index 0000000000..bc754a40a6 +--- /dev/null ++++ b/pyproject.toml +@@ -0,0 +1,74 @@ ++[project] ++authors = [ ++ { name = "Joey Hess", email = "id@joeyh.name" }, ++] ++classifiers = [ ++ "Development Status :: 6 - Mature", ++ "Environment :: Console", ++ "Intended Audience :: Developers", ++ "Intended Audience :: End Users/Desktop", ++ "Intended Audience :: Information Technology", ++ "Intended Audience :: Science/Research", ++ "License :: DFSG approved", ++ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", ++ "Natural Language :: English", ++ "Operating System :: MacOS", ++ "Operating System :: Microsoft :: Windows", ++ "Operating System :: POSIX :: Linux", ++ "Programming Language :: Haskell", ++ "Topic :: Software Development :: Version Control", ++ "Topic :: Software Development :: Version Control :: Git", ++ "Topic :: System :: Archiving :: Backup", ++ "Topic :: System :: Archiving :: Mirroring", ++ "Topic :: System :: Archiving :: Packaging", ++ "Topic :: Utilities", ++] ++description = "manage files with git, without checking their contents into git" ++dynamic = ["version"] ++keywords = [ ++ "git", ++ "data logistics", ++ "version control", ++] ++license = "AGPL-3.0-or-later" ++maintainers = [ ++ { name = "Michael Hanke", email = "mih@ngln.eu" }, ++] ++name = "git-annex" ++readme = "python/README.md" ++requires-python = ">=3.9" ++ ++[project.urls] ++Homepage = "https://git-annex.branchable.com/" ++Documentation = "https://git-annex.branchable.com/git-annex" ++Issues = "https://git-annex.branchable.com/bugs" ++Source = "http://source.git-annex.branchable.com/?p=source.git" ++Changelog = "http://source.git-annex.branchable.com/?p=source.git;a=blob;f=CHANGELOG;hb=HEAD" ++ ++[build-system] ++requires = [ ++ "hatchling", ++ "hatch-vcs", ++] ++build-backend = "hatchling.build" ++ ++[tool.hatch.version] ++source = "vcs" ++ ++[tool.hatch.build.targets.wheel.shared-data] ++"git-annex" = "bin/git-annex" ++ ++[tool.hatch.build.targets.wheel] ++# we need to ship at least one file ++only-include = ["python/py.typed"] ++ ++[tool.hatch.build.targets.wheel.sources] ++# give the 'only-include' files a base directory in the wheel ++"python" = "git-annex" ++ ++[tool.hatch.build.targets.wheel.hooks.custom] ++# custom build hook to set some metadata ++path = "python/build_hook_plugin.py" ++# if set, actually perform a build. Otherwise assume that ++# the built binary is in bin/ ++#build = "stack" +diff --git a/python/README.md b/python/README.md +new file mode 100644 +index 0000000000..e69de29bb2 +diff --git a/python/build_hook_plugin.py b/python/build_hook_plugin.py +new file mode 100644 +index 0000000000..fb5107184d +--- /dev/null ++++ b/python/build_hook_plugin.py +@@ -0,0 +1,31 @@ ++from pathlib import Path ++from subprocess import run ++from sysconfig import get_platform ++from typing import Any ++ ++from hatchling.builders.hooks.plugin.interface import BuildHookInterface ++ ++ ++class SpecialBuildHook(BuildHookInterface): ++ ++ def initialize( ++ self, ++ version: str, # noqa: ARG002 ++ build_data: dict[str, Any], ++ ) -> None: ++ # we have platform-specific builds ++ build_data['pure_python'] = False ++ # set a tag that says: any python3 for the build platform ++ build_data['tag'] = \ ++ f'py3-none-{get_platform().replace("-", "_").replace(".", "_")}' ++ ++ build_cmds = [] ++ if self.config.get('build') == 'stack': ++ build_cmds = [ ++ ['stack', 'setup'], ++ ['stack', 'build', '--no-haddock'], ++ ['stack', 'install', '--no-haddock', '--local-bin-path', '.'], ++ ] ++ ++ for cmd in build_cmds: ++ run(cmd, cwd=self.root, check=True) +diff --git a/python/py.typed b/python/py.typed +new file mode 100644 +index 0000000000..e69de29bb2 From af11ce9ce52ce97848a5a0f60186ab9da2985a64 Mon Sep 17 00:00:00 2001 From: GitHub Almighty Date: Tue, 6 May 2025 16:28:01 -0400 Subject: [PATCH 2/6] Apply "unix2dos" to patch files on Windows when applying Apparently git is touchy to line endings on Windows while applying patches, so we will just modify them in place on windows to guarantee correct ending --- .github/workflows/tools/apply-patches | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tools/apply-patches b/.github/workflows/tools/apply-patches index 55dfe76cbe..0f5399356f 100755 --- a/.github/workflows/tools/apply-patches +++ b/.github/workflows/tools/apply-patches @@ -17,6 +17,9 @@ git config --global user.name "GitHub Almighty" find "$patches_dir" -type f '!' -name '.*' -print | sort | while read -r patchfile do echo "[INFO] Applying patch $patchfile" + if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then # Check if on Windows (adjust as needed) + unix2dos "$patchfile" # Convert to CRLF on Windows + fi # Test whether the patch has already been applied by seeing if it can be # applied in reverse if git apply -R --check "$patchfile" From bc96ccbc57a4c97262cb144180f5b7ff3209b886 Mon Sep 17 00:00:00 2001 From: GitHub Almighty Date: Tue, 6 May 2025 16:41:02 -0400 Subject: [PATCH 3/6] Mark apply-patches as the one to trigger CI too --- .github/workflows/template/build-{{ostype}}.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/template/build-{{ostype}}.yaml.j2 b/.github/workflows/template/build-{{ostype}}.yaml.j2 index 63fb3d186e..0c66808505 100644 --- a/.github/workflows/template/build-{{ostype}}.yaml.j2 +++ b/.github/workflows/template/build-{{ostype}}.yaml.j2 @@ -5,6 +5,7 @@ on: pull_request: paths: - '.github/workflows/build-{{ostype}}.yaml' + - '.github/workflows/tools/apply-patches' - 'patches/*.patch' schedule: - cron: '30 {{cron_hour}} * * *' From 8cdc18b20c3f4250c01d0b12067e3b8d252fe94d Mon Sep 17 00:00:00 2001 From: GitHub Almighty Date: Tue, 6 May 2025 16:44:37 -0400 Subject: [PATCH 4/6] Add uv building of the wheel for all OSes --- .github/workflows/template/build-{{ostype}}.yaml.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/template/build-{{ostype}}.yaml.j2 b/.github/workflows/template/build-{{ostype}}.yaml.j2 index 0c66808505..bd9accef29 100644 --- a/.github/workflows/template/build-{{ostype}}.yaml.j2 +++ b/.github/workflows/template/build-{{ostype}}.yaml.j2 @@ -269,6 +269,14 @@ jobs: git-annex-installer_"${{ steps.build-version.outputs.version }}".exe {% endif %} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build the Python wheel + run: | + uv build --wheel + - name: Upload packages uses: actions/upload-artifact@v4 with: From 3c0e7c53bcc5011a9ff926580f6e46f3e6805f42 Mon Sep 17 00:00:00 2001 From: GitHub Almighty Date: Tue, 6 May 2025 16:44:39 -0400 Subject: [PATCH 5/6] [DATALAD RUNCMD] regenerate templates after modifying workflow === Do not change lines below === { "chain": [ "47fa1b854c9e94abc0fe7aadb8df8a8cd6525808", "ff26f850198044041285400f56f451b5a76d95d3", "424897650eadcbf7c13636064073bde0df7a6eda" ], "cmd": "make -C .github/workflows/template", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- .github/workflows/build-macos.yaml | 9 +++++++++ .github/workflows/build-ubuntu.yaml | 9 +++++++++ .github/workflows/build-windows.yaml | 16 +++++++++------- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-macos.yaml b/.github/workflows/build-macos.yaml index d74d488ee0..48c7fee659 100644 --- a/.github/workflows/build-macos.yaml +++ b/.github/workflows/build-macos.yaml @@ -5,6 +5,7 @@ on: pull_request: paths: - '.github/workflows/build-macos.yaml' + - '.github/workflows/tools/apply-patches' - 'patches/*.patch' schedule: - cron: '30 01 * * *' @@ -121,6 +122,14 @@ jobs: mv tmp/git-annex.dmg \ git-annex_"${{ steps.build-version.outputs.version }}".dmg + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build the Python wheel + run: | + uv build --wheel + - name: Upload packages uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/build-ubuntu.yaml b/.github/workflows/build-ubuntu.yaml index 9e3aa6964a..2477e5774d 100644 --- a/.github/workflows/build-ubuntu.yaml +++ b/.github/workflows/build-ubuntu.yaml @@ -5,6 +5,7 @@ on: pull_request: paths: - '.github/workflows/build-ubuntu.yaml' + - '.github/workflows/tools/apply-patches' - 'patches/*.patch' schedule: - cron: '30 02 * * *' @@ -141,6 +142,14 @@ jobs: run: grep -E '^All [[:digit:]]{3} tests passed' "${bbuild_log}" if: "!contains(env.DEB_BUILD_OPTIONS, 'nocheck')" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build the Python wheel + run: | + uv build --wheel + - name: Upload packages uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/build-windows.yaml b/.github/workflows/build-windows.yaml index a0cad8f6d6..a411cb594a 100644 --- a/.github/workflows/build-windows.yaml +++ b/.github/workflows/build-windows.yaml @@ -5,6 +5,7 @@ on: pull_request: paths: - '.github/workflows/build-windows.yaml' + - '.github/workflows/tools/apply-patches' - 'patches/*.patch' schedule: - cron: '30 03 * * *' @@ -134,13 +135,6 @@ jobs: - name: Build git-annex run: stack install --no-haddock --local-bin-path . - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Build the Python wheel - run: | - uv build --wheel - - name: Build the installer run: | stack ghc --no-haddock --package nsis Build/NullSoftInstaller.hs @@ -151,6 +145,14 @@ jobs: mv git-annex-installer.exe \ git-annex-installer_"${{ steps.build-version.outputs.version }}".exe + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build the Python wheel + run: | + uv build --wheel + - name: Upload packages uses: actions/upload-artifact@v4 with: From d9646d7e23395cbd75a3b9de4e37303463431b12 Mon Sep 17 00:00:00 2001 From: GitHub Almighty Date: Tue, 6 May 2025 17:21:35 -0400 Subject: [PATCH 6/6] Visualize value of the OSTYPE since all of a sudden applying on Windows stopped working --- .github/workflows/tools/apply-patches | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tools/apply-patches b/.github/workflows/tools/apply-patches index 0f5399356f..9faf849473 100755 --- a/.github/workflows/tools/apply-patches +++ b/.github/workflows/tools/apply-patches @@ -16,7 +16,7 @@ git config --global user.name "GitHub Almighty" find "$patches_dir" -type f '!' -name '.*' -print | sort | while read -r patchfile do - echo "[INFO] Applying patch $patchfile" + echo "[INFO] Applying patch '$patchfile' on '$OSTYPE'" if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then # Check if on Windows (adjust as needed) unix2dos "$patchfile" # Convert to CRLF on Windows fi