diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 091dc19..9b87d6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: - name: Install run: | npm install - pip3 install -r requirements.txt + python3 -m pip install -r requirements.txt - name: Test run: npm test diff --git a/changelog.md b/changelog.md index 91b9908..563be92 100644 --- a/changelog.md +++ b/changelog.md @@ -54,7 +54,7 @@ - Docs: https://arc.codes/docs/en/guides/developer-experience/dependency-management#python - This supports global options passed in a root `requirements.txt` file (example: `--extra-index-url https://test.pypi.org/simple/`), but does not yet support dependencies versioned at root, or shared or views dependencies - All project dependencies must be installed on the system prior to deployment - - Python Lambda treeshaking also requires the `pipdeptree` package to be available from shell; ensure you've run `pip3 install pipdeptree` prior to use + - Python Lambda treeshaking also requires the `pipdeptree` package to be available from shell; ensure you've run `python3 -m pip install pipdeptree` prior to use --- diff --git a/readme.md b/readme.md index 07d5e0f..15b5f3c 100644 --- a/readme.md +++ b/readme.md @@ -62,7 +62,7 @@ Installs function dependencies, then invokes [`hydrate.shared()`][shared]. To ensure local development behavior is as close to `staging` and `production` as possible, `hydrate.install()` (and other hydrate functions) uses: - **Node.js**: `npm ci` if `package-lock.json` is present and `npm i` if not; or `yarn` -- **Python**: `pip3 install` +- **Python**: `python3 -m pip install` - **Ruby**: `bundle install` Note: by default `update` also installs dependencies in shared folders like `src/shared` and `src/views`. @@ -75,7 +75,7 @@ Updates function dependencies, then invokes [`hydrate.shared()`][shared]. `update` is functionally almost identical to [`install`][install], except it will update dependencies to newer versions _if they exist_. This is done via: - **Node.js**: `npm update` or `yarn upgrade` -- **Python**: `pip3 install -U --upgrade-strategy eager` +- **Python**: `python3 -m pip install -U --upgrade-strategy eager` - **Ruby**: `bundle update` Note: by default `update` also updates dependencies in shared folders like `src/shared` and `src/views`. diff --git a/src/actions/autoinstall/python/check-py-tools.js b/src/actions/autoinstall/python/check-py-tools.js index beada9d..c2399d4 100644 --- a/src/actions/autoinstall/python/check-py-tools.js +++ b/src/actions/autoinstall/python/check-py-tools.js @@ -2,8 +2,8 @@ let { spawnSync } = require('child_process') module.exports = function checkPyTools () { - let cmd = 'pip3' - let args = [ 'list' ] + let cmd = 'python3' + let args = [ '-m', 'pip', '--version' ] let raw = spawnSync(cmd, args, { cwd: __dirname, shell: true, @@ -19,7 +19,7 @@ module.exports = function checkPyTools () { if (l.split(' ')[0] === 'pipdeptree') pipdeptree = true }) if (!pipdeptree) { - throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'pip3 install pipdeptree'`) + throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'python3 -m pip install pipdeptree'`) } return true } diff --git a/src/actions/autoinstall/python/get-dep-tree.js b/src/actions/autoinstall/python/get-dep-tree.js index 9bfea0e..0cb9972 100644 --- a/src/actions/autoinstall/python/get-dep-tree.js +++ b/src/actions/autoinstall/python/get-dep-tree.js @@ -5,7 +5,7 @@ module.exports = function getDepTree (topLevelDeps) { let shell = true if (!pipdeptree) { - let pipdeptreeCheck = spawnSync('pip3', [ 'list' ], { shell }) + let pipdeptreeCheck = spawnSync('python3', [ '-m', 'pip', 'list' ], { shell }) if (pipdeptreeCheck.status) { console.error(pipdeptreeCheck.output.toString()) throw Error(`pip3 error`) @@ -16,7 +16,7 @@ module.exports = function getDepTree (topLevelDeps) { if (l.split(' ')[0] === 'pipdeptree') pipdeptree = true }) if (!pipdeptree) { - throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'pip3 install pipdeptree'`) + throw Error(`pipdeptree required for treeshaking Python Lambdas, please run 'python3 -m pip install pipdeptree'`) } } diff --git a/src/actions/install-update.js b/src/actions/install-update.js index e57e5ab..135d995 100644 --- a/src/actions/install-update.js +++ b/src/actions/install-update.js @@ -110,17 +110,28 @@ module.exports = function hydrator (params, callback) { else if (isPy) { let flags = '' if (lambda) { - // Technique per AWS, found that `--python-version` was essential, but `--implementation cp` may not be - // https://repost.aws/knowledge-center/lambda-python-package-compatible - // This may still not work because of glibc version differences, see: - // https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html#glibc-gcc-and-binutils - let arch = lambda.config.architecture === 'arm64' ? 'manylinux2014_aarch64' : 'manylinux2014_x86_64' - let ver = lambda.config.runtime.split('python')[1] - flags = '--only-binary=:all: ' + - `--platform=${arch} ` + - `--python-version ${ver} ` - // Reset flags if installing from Sandbox - if (local) flags = '' + if (!local) { + let arch = lambda.config.architecture === 'arm64' ? 'aarch64' : 'x86_64' + let ver = lambda.config.runtime.split('python')[1] + + let pythonPlatformCheck = child.spawnSync('python3', [ '-c', "'import platform; import sys; print(platform.system(), platform.machine(), *platform.python_version_tuple())'", ], { shell: true }) + if (pythonPlatformCheck.status) { + console.error(pythonPlatformCheck.output.toString()) + throw Error(`python3 error`) + } + let [ platformSystem, platformMachine, platformPythonVersionMajor, platformPythonVersionMinor, ] = pythonPlatformCheck.stdout.toString().trim().split(' ') + let isNative = platformSystem === 'Linux' && platformMachine === arch && `${platformPythonVersionMajor}.${platformPythonVersionMinor}` === ver + + if (!isNative) { + // Technique per AWS, found that `--python-version` was essential, but `--implementation cp` may not be + // https://repost.aws/knowledge-center/lambda-python-package-compatible + // This may still not work because of glibc version differences, see: + // https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html#glibc-gcc-and-binutils + flags = '--only-binary=:all: ' + + `--platform=manylinux2014_${arch} ` + + `--python-version ${ver} ` + } + } // Update Python deps if (!installing) { @@ -129,7 +140,7 @@ module.exports = function hydrator (params, callback) { flags += '-U --upgrade-strategy eager' } } - let cmd = `pip3 install -r requirements.txt -t ./vendor ${flags}`.trim() + let cmd = `python3 -m pip install -r requirements.txt -t ./vendor ${flags}`.trim() exec(cmd, options, callback) }