From 1b4b25eee0169046ac842f1f0e877e4b6a94cdb9 Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Tue, 7 Nov 2017 10:50:19 -0400 Subject: [PATCH 01/10] Add support for custom pip arguments --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index 12fa48e..5b6dbe2 100644 --- a/index.js +++ b/index.js @@ -32,6 +32,7 @@ class PkgPyFuncs { this.useDocker = config.useDocker || false this.dockerImage = config.dockerImage || `lambci/lambda:build-${this.serverless.service.provider.runtime}` this.containerName = config.containerName || 'serverless-package-python-functions' + this.pipArgs = config.pipCmdExtraArgs || [] } clean(){ @@ -76,6 +77,9 @@ class PkgPyFuncs { let cmd = 'pip' let args = ['install','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] + if (this.pipArgs){ + args = _.concat(args, this.pipArgs) + } if ( this.useDocker === true ){ cmd = 'docker' args = ['exec',this.containerName, 'pip', ...args] From d5714433921f11e58226d1c92ca3f4fc472a61da Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Tue, 7 Nov 2017 12:15:52 -0400 Subject: [PATCH 02/10] Add exclude option Add ability to exclude glob patterns and exclude the following by default: * "**/.serverless", * "**/node_modules", * "**/requirements.txt", * "**/package.json", * "**/package-lock.json" --- index.js | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 5b6dbe2..050d149 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ const BbPromise = require('bluebird'); const _ = require('lodash'); const Fse = require('fs-extra'); +const minimatch = require('minimatch') const Path = require('path'); const ChildProcess = require('child_process'); const zipper = require('zip-local'); @@ -28,11 +29,20 @@ class PkgPyFuncs { config.buildDir ? this.buildDir = config.buildDir : this.error("No buildDir configuration specified") this.globalRequirements = config.globalRequirements || [] this.globalIncludes = config.globalIncludes || [] + this.globalExcludes = config.globalExcludes || [] config.cleanup === undefined ? this.cleanup = true : this.cleanup = config.cleanup this.useDocker = config.useDocker || false this.dockerImage = config.dockerImage || `lambci/lambda:build-${this.serverless.service.provider.runtime}` this.containerName = config.containerName || 'serverless-package-python-functions' this.pipArgs = config.pipCmdExtraArgs || [] + + this.defaultExcludes = [ + "**/.serverless", + "**/node_modules", + "**/requirements.txt", + "**/package.json", + "**/package-lock.json" + ] } clean(){ @@ -154,7 +164,21 @@ class PkgPyFuncs { if (this.globalIncludes){ includes = _.concat(includes, this.globalIncludes) } - _.forEach(includes, (item) => { Fse.copySync(item, buildPath) } ) + let excludes = target.excludes || [] + excludes = _.concat(excludes, this.defaultExcludes) + if (this.globalExcludes){ + excludes = _.concat(excludes, this.globalExcludes) + } + + let filter = (src, dest) => { + for(var i = 0; i < excludes.length; i++) { + if (minimatch(src, excludes[i])){ + return false + } + } + return true + } + _.forEach(includes, (item) => { Fse.copySync(item, buildPath, filter) } ) // Install requirements let requirements = [requirementsPath] From db4a75c9a9a71feb628966ffc83aaf2192a8343d Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Tue, 7 Nov 2017 12:17:28 -0400 Subject: [PATCH 03/10] add minimatch dependency --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 244a699..60b57d2 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "bluebird": "^3.5.0", "fs-extra": "^3.0.0", "lodash": "^4.17.4", + "minimatch": "^3.0.4", "upath": "^1.0.0", "zip-local": "^0.3.4" }, From d6ebde3f8fdcd0f7805001047f666ee7b0da988e Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Tue, 3 Apr 2018 14:03:54 -0300 Subject: [PATCH 04/10] Removing requirements from exclusion. By excluding the function's requirements file it wasn't actually installed anymore --- index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/index.js b/index.js index 050d149..da5e298 100644 --- a/index.js +++ b/index.js @@ -39,7 +39,6 @@ class PkgPyFuncs { this.defaultExcludes = [ "**/.serverless", "**/node_modules", - "**/requirements.txt", "**/package.json", "**/package-lock.json" ] From c13ff813831b2c03454e2ee9a51a48551b0c163c Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Tue, 3 Apr 2018 14:09:01 -0300 Subject: [PATCH 05/10] Flipping order of requirements installation Ensuring a function's requirements file overrides the global one. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index da5e298..b3cab6a 100644 --- a/index.js +++ b/index.js @@ -182,7 +182,7 @@ class PkgPyFuncs { // Install requirements let requirements = [requirementsPath] if (this.globalRequirements){ - requirements = _.concat(requirements, this.globalRequirements) + requirements = _.concat(this.globalRequirements, requirements) } _.forEach(requirements, (req) => { this.installRequirements(buildPath,req)}) zipper.sync.zip(buildPath).compress().save(`${buildPath}.zip`) From 04e22948a254b74df4b569fab101a1e4b5e838b3 Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Wed, 18 Apr 2018 12:22:41 -0300 Subject: [PATCH 06/10] Disable pip version check. Previously, every time a new version came out, this would simply fail until you updated pip. This obviously shouldn't happen. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index b3cab6a..89a0db0 100644 --- a/index.js +++ b/index.js @@ -85,7 +85,7 @@ class PkgPyFuncs { } let cmd = 'pip' - let args = ['install','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] + let args = ['install','--disable-pip-version-check','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] if (this.pipArgs){ args = _.concat(args, this.pipArgs) } From 12fc49d6b0780e53e68c1f1125126368ab8ea24a Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Mon, 23 Apr 2018 17:38:52 -0300 Subject: [PATCH 07/10] add --no-warn-conflicts Pip 10 will log error messages about broken dependencies. This is interpreted by serverless as an actual error and we don't want that. So disable the check. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 89a0db0..0635dca 100644 --- a/index.js +++ b/index.js @@ -85,7 +85,7 @@ class PkgPyFuncs { } let cmd = 'pip' - let args = ['install','--disable-pip-version-check','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] + let args = ['install','--disable-pip-version-check','--no-warn-conflicts','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] if (this.pipArgs){ args = _.concat(args, this.pipArgs) } From ce0194a323c3a31c04d42a423bd925cbd142b323 Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Mon, 23 Apr 2018 19:22:41 -0300 Subject: [PATCH 08/10] Only add '--no-warn-conflicts' for pip version >= 10 --- index.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 0635dca..a987e93 100644 --- a/index.js +++ b/index.js @@ -85,7 +85,14 @@ class PkgPyFuncs { } let cmd = 'pip' - let args = ['install','--disable-pip-version-check','--no-warn-conflicts','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] + const out = this.runProcess(cmd, ['-V']) + const pipVersion = out.split(" ")[1] + + let args = ['install','--disable-pip-version-check','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] + if (pipVersion.split('.')[0] >= 10) { + args.push('--no-warn-conflicts') + } + let args = ['install','--disable-pip-version-check','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] if (this.pipArgs){ args = _.concat(args, this.pipArgs) } From c21b4ea33cbcd0a6f07ac4eaf5f74974811f2404 Mon Sep 17 00:00:00 2001 From: MichielVanderlee Date: Mon, 23 Apr 2018 19:33:22 -0300 Subject: [PATCH 09/10] fix copy paste mistake --- index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/index.js b/index.js index a987e93..5d899dd 100644 --- a/index.js +++ b/index.js @@ -92,7 +92,6 @@ class PkgPyFuncs { if (pipVersion.split('.')[0] >= 10) { args.push('--no-warn-conflicts') } - let args = ['install','--disable-pip-version-check','--upgrade','-t', upath.normalize(buildPath), '-r', upath.normalize(requirementsPath)] if (this.pipArgs){ args = _.concat(args, this.pipArgs) } From 322500e66e63a586c59383132bf744c962dad726 Mon Sep 17 00:00:00 2001 From: Michiel Vanderlee Date: Thu, 14 Jun 2018 12:34:09 -0300 Subject: [PATCH 10/10] Fix individual packaging --- index.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 5d899dd..42feaa1 100644 --- a/index.js +++ b/index.js @@ -35,7 +35,7 @@ class PkgPyFuncs { this.dockerImage = config.dockerImage || `lambci/lambda:build-${this.serverless.service.provider.runtime}` this.containerName = config.containerName || 'serverless-package-python-functions' this.pipArgs = config.pipCmdExtraArgs || [] - + this.defaultExcludes = [ "**/.serverless", "**/node_modules", @@ -61,7 +61,7 @@ class PkgPyFuncs { } selectAll() { - const functions = this.serverless.service.functions + const functions = _.pickBy(this.serverless.service.functions, (value, key) => !this.options.function || key === this.options.function ) const info = _.map(functions, (target) => { return { name: target.name, @@ -206,7 +206,12 @@ class PkgPyFuncs { .then(this.setupDocker) .then(this.selectAll) .map(this.makePackage), - + 'deploy:function:packageFunction': () => BbPromise.bind(this) + .then(this.fetchConfig) + .then( () => { Fse.ensureDirAsync(this.buildDir) }) + .then(this.setupDocker) + .then(this.selectAll) + .map(this.makePackage), 'after:deploy:deploy': () => BbPromise.bind(this) .then(this.clean) };