From 047b88db8aec92d244c3de474085cca77a958675 Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Wed, 10 Jan 2024 02:17:58 -0500 Subject: [PATCH 1/3] Invoke pip using `python3 -m pip` rather than `pip3` This is closer to the recommended way to invoke pip according to the [latest pip documentation](https://pip.pypa.io/en/stable/getting-started/). The reason why this invocation is more robust is that it only requires the Python interpreter itself to be on the PATH, and it is robust to the bin directory corresponding to site-packages being omitted from the PATH (usually by accident). --- .github/workflows/build.yml | 2 +- changelog.md | 2 +- readme.md | 4 ++-- src/actions/autoinstall/python/check-py-tools.js | 6 +++--- src/actions/autoinstall/python/get-dep-tree.js | 4 ++-- src/actions/install-update.js | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 091dc19..9b87d6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: - name: Install run: | npm install - pip3 install -r requirements.txt + python3 -m pip install -r requirements.txt - name: Test run: npm test diff --git a/changelog.md b/changelog.md index 91b9908..563be92 100644 --- a/changelog.md +++ b/changelog.md @@ -54,7 +54,7 @@ - Docs: https://arc.codes/docs/en/guides/developer-experience/dependency-management#python - This supports global options passed in a root `requirements.txt` file (example: `--extra-index-url https://test.pypi.org/simple/`), but does not yet support dependencies versioned at root, or shared or views dependencies - All project dependencies must be installed on the system prior to deployment - - Python Lambda treeshaking also requires the `pipdeptree` package to be available from shell; ensure you've run `pip3 install pipdeptree` prior to use + - Python Lambda treeshaking also requires the `pipdeptree` package to be available from shell; ensure you've run `python3 -m pip install pipdeptree` prior to use --- diff --git a/readme.md b/readme.md index 07d5e0f..15b5f3c 100644 --- a/readme.md +++ b/readme.md @@ -62,7 +62,7 @@ Installs function dependencies, then invokes [`hydrate.shared()`][shared]. To ensure local development behavior is as close to `staging` and `production` as possible, `hydrate.install()` (and other hydrate functions) uses: - **Node.js**: `npm ci` if `package-lock.json` is present and `npm i` if not; or `yarn` -- **Python**: `pip3 install` +- **Python**: `python3 -m pip install` - **Ruby**: `bundle install` Note: by default `update` also installs dependencies in shared folders like `src/shared` and `src/views`. @@ -75,7 +75,7 @@ Updates function dependencies, then invokes [`hydrate.shared()`][shared]. `update` is functionally almost identical to [`install`][install], except it will update dependencies to newer versions _if they exist_. This is done via: - **Node.js**: `npm update` or `yarn upgrade` -- **Python**: `pip3 install -U --upgrade-strategy eager` +- **Python**: `python3 -m pip install -U --upgrade-strategy eager` - **Ruby**: `bundle update` Note: by default `update` also updates dependencies in shared folders like `src/shared` and `src/views`. diff --git a/src/actions/autoinstall/python/check-py-tools.js b/src/actions/autoinstall/python/check-py-tools.js index beada9d..1fcee34 100644 --- a/src/actions/autoinstall/python/check-py-tools.js +++ b/src/actions/autoinstall/python/check-py-tools.js @@ -2,8 +2,8 @@ let { spawnSync } = require('child_process') module.exports = function checkPyTools () { - let cmd = 'pip3' - let args = [ 'list' ] + let cmd = 'python3' + let args = [ '-m', 'pip', 'list' ] let raw = spawnSync(cmd, args, { cwd: __dirname, shell: true, @@ -19,7 +19,7 @@ module.exports = function checkPyTools () { if (l.split(' ')[0] === 'pipdeptree') pipdeptree = true }) if (!pipdeptree) { - throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'pip3 install pipdeptree'`) + throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'python3 -m pip install pipdeptree'`) } return true } diff --git a/src/actions/autoinstall/python/get-dep-tree.js b/src/actions/autoinstall/python/get-dep-tree.js index 9bfea0e..0cb9972 100644 --- a/src/actions/autoinstall/python/get-dep-tree.js +++ b/src/actions/autoinstall/python/get-dep-tree.js @@ -5,7 +5,7 @@ module.exports = function getDepTree (topLevelDeps) { let shell = true if (!pipdeptree) { - let pipdeptreeCheck = spawnSync('pip3', [ 'list' ], { shell }) + let pipdeptreeCheck = spawnSync('python3', [ '-m', 'pip', 'list' ], { shell }) if (pipdeptreeCheck.status) { console.error(pipdeptreeCheck.output.toString()) throw Error(`pip3 error`) @@ -16,7 +16,7 @@ module.exports = function getDepTree (topLevelDeps) { if (l.split(' ')[0] === 'pipdeptree') pipdeptree = true }) if (!pipdeptree) { - throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'pip3 install pipdeptree'`) + throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'python3 -m pip install pipdeptree'`) } } diff --git a/src/actions/install-update.js b/src/actions/install-update.js index e57e5ab..8c84f7d 100644 --- a/src/actions/install-update.js +++ b/src/actions/install-update.js @@ -129,7 +129,7 @@ module.exports = function hydrator (params, callback) { flags += '-U --upgrade-strategy eager' } } - let cmd = `pip3 install -r requirements.txt -t ./vendor ${flags}`.trim() + let cmd = `python3 -m pip install -r requirements.txt -t ./vendor ${flags}`.trim() exec(cmd, options, callback) } From 99c475228fa811365943214eafa29344d6010e47 Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Wed, 10 Jan 2024 02:22:28 -0500 Subject: [PATCH 2/3] Use `pip --version`, not `pip list`, to test for working pip `pip list` can be very time consuming if there are many packages installed. `pip --version` is fast regardless. --- src/actions/autoinstall/python/check-py-tools.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/actions/autoinstall/python/check-py-tools.js b/src/actions/autoinstall/python/check-py-tools.js index 1fcee34..c2399d4 100644 --- a/src/actions/autoinstall/python/check-py-tools.js +++ b/src/actions/autoinstall/python/check-py-tools.js @@ -3,7 +3,7 @@ let { spawnSync } = require('child_process') module.exports = function checkPyTools () { let cmd = 'python3' - let args = [ '-m', 'pip', 'list' ] + let args = [ '-m', 'pip', '--version' ] let raw = spawnSync(cmd, args, { cwd: __dirname, shell: true, From 77c34c3c57cb19ccb1878cd3a16498a5796bde35 Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Wed, 10 Jan 2024 02:53:45 -0500 Subject: [PATCH 3/3] Permit installing Python packages from source if host and target match Some Python packages _must_ be installed from source: either PyPI only holds source distributions of them, or lacks wheels for the target platform. If the host platform matches the target Lambda platform, then permit installation of Python packages from source. --- src/actions/install-update.js | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/actions/install-update.js b/src/actions/install-update.js index 8c84f7d..135d995 100644 --- a/src/actions/install-update.js +++ b/src/actions/install-update.js @@ -110,17 +110,28 @@ module.exports = function hydrator (params, callback) { else if (isPy) { let flags = '' if (lambda) { - // Technique per AWS, found that `--python-version` was essential, but `--implementation cp` may not be - // https://repost.aws/knowledge-center/lambda-python-package-compatible - // This may still not work because of glibc version differences, see: - // https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html#glibc-gcc-and-binutils - let arch = lambda.config.architecture === 'arm64' ? 'manylinux2014_aarch64' : 'manylinux2014_x86_64' - let ver = lambda.config.runtime.split('python')[1] - flags = '--only-binary=:all: ' + - `--platform=${arch} ` + - `--python-version ${ver} ` - // Reset flags if installing from Sandbox - if (local) flags = '' + if (!local) { + let arch = lambda.config.architecture === 'arm64' ? 'aarch64' : 'x86_64' + let ver = lambda.config.runtime.split('python')[1] + + let pythonPlatformCheck = child.spawnSync('python3', [ '-c', "'import platform; import sys; print(platform.system(), platform.machine(), *platform.python_version_tuple())'", ], { shell: true }) + if (pythonPlatformCheck.status) { + console.error(pythonPlatformCheck.output.toString()) + throw Error(`python3 error`) + } + let [ platformSystem, platformMachine, platformPythonVersionMajor, platformPythonVersionMinor, ] = pythonPlatformCheck.stdout.toString().trim().split(' ') + let isNative = platformSystem === 'Linux' && platformMachine === arch && `${platformPythonVersionMajor}.${platformPythonVersionMinor}` === ver + + if (!isNative) { + // Technique per AWS, found that `--python-version` was essential, but `--implementation cp` may not be + // https://repost.aws/knowledge-center/lambda-python-package-compatible + // This may still not work because of glibc version differences, see: + // https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html#glibc-gcc-and-binutils + flags = '--only-binary=:all: ' + + `--platform=manylinux2014_${arch} ` + + `--python-version ${ver} ` + } + } // Update Python deps if (!installing) {