From bb186d0ca52ec055607579eb924c59673e0c8bba Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Mon, 29 Mar 2021 13:40:12 -0500 Subject: [PATCH 01/12] Improve Travis build log information --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 79555d24a..c0709af87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -88,6 +88,7 @@ before_deploy: - echo $TRAVIS_EVENT_TYPE - echo "GIT_TAG is " - echo $GIT_TAG + - echo "GIT_TAG is $GIT_TAG" deploy: - provider: releases From 8b572f6f2a23ad0d27adba4da331b07bd8b323a2 Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Mon, 29 Mar 2021 13:50:33 -0500 Subject: [PATCH 02/12] Improve Travis build log information --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c0709af87..34fe1df94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,6 +72,10 @@ before_deploy: export GIT_TAG=$TRAVIS_TAG; export TAG=true; fi + - echo "TRAVIS_BRANCH is $TRAVIS_BRANCH" + - echo "TRAVIS_EVENT_TYPE is $TRAVIS_EVENT_TYPE" + - echo "GIT_TAG=$GIT_TAG" + - echo "TAG=$TAG" - ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - "./tools/travis/build_tag_releases.sh $build_file_name $GIT_TAG" - export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)" @@ -86,9 +90,11 @@ before_deploy: fi - echo "TRAVIS_EVENT_TYPE is " - echo $TRAVIS_EVENT_TYPE + - env TRAVIS_EVENT_TYPE - echo "GIT_TAG is " - echo $GIT_TAG - - echo "GIT_TAG is $GIT_TAG" + - env GIT_TAG + - echo "GIT_TAG is ${GIT_TAG}" deploy: - provider: releases From a5ba1ae181a0254d699d2aa3498411871c56ff1a Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Mon, 29 Mar 2021 13:57:59 -0500 Subject: [PATCH 03/12] Improve Travis build log information --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 34fe1df94..2baa64f10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,6 +76,11 @@ before_deploy: - echo "TRAVIS_EVENT_TYPE is $TRAVIS_EVENT_TYPE" - echo "GIT_TAG=$GIT_TAG" - echo "TAG=$TAG" + - echo "Using Gradle Wrapper to produce release binaries..." + - echo "GIT_TAG=$GIT_TAG" + - env GIT_TAG + - echo "TAG=$TAG" + - env TAG - ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - "./tools/travis/build_tag_releases.sh $build_file_name $GIT_TAG" - export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)" @@ -88,6 +93,7 @@ before_deploy: GIT_COMMITTER_DATE="$(git show --format=%aD | head -1)" git tag $GIT_TAG -a -m "Generated tag from Travis CI build $TRAVIS_BUILD_NUMBER"; git push -f -q https://$API_KEY@github.com/apache/openwhisk-wskdeploy $GIT_TAG; fi + - echo "Completing before_deploy stage" - echo "TRAVIS_EVENT_TYPE is " - echo $TRAVIS_EVENT_TYPE - env TRAVIS_EVENT_TYPE From 707dc0f30338751c1cbdad01b5a763ff45d6b5cf Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Wed, 31 Mar 2021 10:42:00 -0500 Subject: [PATCH 04/12] Improve Travis builds --- .gitignore | 3 +- .travis.yml | 70 ++++++++++++++++++++++-------- build.gradle | 2 +- go.mod | 3 -- go.sum | 8 ++-- tools/travis/build_tag_releases.sh | 9 +++- 6 files changed, 67 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index b2b868354..9f7a014ff 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ _testmain.go *.prof -#OS X +# OS X **/.DS_Store #binaries @@ -43,6 +43,7 @@ Packages/ .gradle .gogradle build/ +releases/ #emacs *~ diff --git a/.travis.yml b/.travis.yml index 2baa64f10..b7281a8c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,12 +36,36 @@ notifications: # travis2slack webhook to enable DMs on openwhisk-team.slack.com to PR authors with TravisCI results secure: "NaDCtHwbFUEy0h0u/QrAt395o0/9FstJ5/Tv4uBSLCBdaUNU+0q6/RXVH76dKbujoxrLded2gyeJSAbI4kl4SM7FyilSHzKTikF0VsgM3G5veh65tVY5ztsvl40hDmK2E/PmWI3HkZNeWJO32WjraqPmhSWL0BZOWhA/4gJSCIsLnNRQXmpZoDqC7V8IG1BWYGwd5qcNkEjItLjYCUy1s2K8bj3QmhshYRVLTNbXxh0yJVKjM04I+bWhMXqvL8GS55qeicJ3fUwm5g1PDVCtzxLwAHCPo2jusjtNSnV+BaZgMBSahv3MD7ApxfF58e1buejggH3qZAhNGccC4bYJCahqVv/KKoA10kO6exH5iGwlHPWQTjMuF++PHmAk+FeQ1jh+JoUSBnIHExPnaD4CQIArHvUuQom+WJBnOz+L1H755VyPGzeDQ+ZUDlhOiQ6CDP/sqaRH3Wmo4IzhsqTsLaQs0dS1YeEWOS8gcawwNTVf/WtqiedYu24rQN9RiUXelRoEaXJy/pldb2KpyQRo2hbKoStDDB5fEij1xyGsj4kSXZ8uk5G10oDGnZYd+okduzJazN8wr0flrLvH32DRgDZDcG/D90I19JmqqfqbcG/SEmhxTikHZeVQd5DfWcqssLvWNrtQa/EJgAQkmZ2gaSsZ73AvPk/KyMV30+m8xTU=" +# The complete sequence of phases of a job is the lifecycle: +# --------------------------------------------------------- +# before_install +# install +# before_script +# script +# before_cache (OPTIONAL) (if and only if caching is effective) +# after_success (or after_failure) +# before_deploy (OPTIONAL) (if and only if deployment is active) +# deploy (OPTIONAL) +# after_deploy (OPTIONAL) (if and only if deployment is active) +# after_script + before_install: - "./tools/travis/cloneutils.sh" + +# Install tools we use for linting and unit/integration testing +# TODO: build.gradle (gradlew) already installs 'golint' and runs linting +# in the `goLint` task; see if we can remove here. +# TODO: The "gogradle" plugin in build.gradle (gradlew) already provides a +# task called "goTest" which runs all `unit` tests. See if we can use that +# instead and only run `integration` tests here. install: - export DEPLOY_BUILD_READY=false - go get -u golang.org/x/lint/golint - go get -u github.com/stretchr/testify + +# Identify Golang files that are not properly formatted +# TODO: See if we can create custom tasks to perform this in build.gradle +# as a custom `goFmt` task before_script: - GO_FILES=$(find . -iname '*.go' -type f -not -path "./wski18n/i18n_resources.go") - export BAD_GO=$(gofmt -s -l $(echo $GO_FILES)) @@ -49,20 +73,26 @@ before_script: #- test -z "$BAD_GO" #- test -z "$(gofmt -s -l $(echo $GO_FILES))" +# Clone and build the OpenWhisk platform (i.e., apache/openwhisk) +# and then use Ansible to deploy it and prepare it to run our +# integration tests. script: - echo $TRAVIS - echo $TRAVIS_PULL_REQUEST - echo $TRAVIS_SECURE_ENV_VARS - printenv - "./tools/travis/script.sh" + after_success: - DEPLOY_BUILD_READY=true - if [ "$TRAVIS_EVENT_TYPE" == "cron" ] ; then export DEPLOY_BUILD_READY=false; fi -after_script: - - make clean +# If this is a "tagged" GitHub build, then use Gradle Wrapper +# to build binaries for all supported OS platforms and architectures. +# The result would be a "/build" directory that has a subdirectory named +# for each supported GOOS/ before_deploy: - export build_file_name=wskdeploy - go get github.com/inconshreveable/mousetrap @@ -72,17 +102,16 @@ before_deploy: export GIT_TAG=$TRAVIS_TAG; export TAG=true; fi - - echo "TRAVIS_BRANCH is $TRAVIS_BRANCH" - - echo "TRAVIS_EVENT_TYPE is $TRAVIS_EVENT_TYPE" - - echo "GIT_TAG=$GIT_TAG" - - echo "TAG=$TAG" + - echo "TRAVIS_BRANCH = [$TRAVIS_BRANCH]" + - echo "TRAVIS_EVENT_TYPE = [$TRAVIS_EVENT_TYPE]" + - echo "GIT_TAG = [$GIT_TAG]" + - echo "TAG = [$TAG]" + - env TRAVIS_BRANCH + - env TRAVIS_EVENT_TYPE + - env $GIT_TAG + - env $TAG - echo "Using Gradle Wrapper to produce release binaries..." - - echo "GIT_TAG=$GIT_TAG" - - env GIT_TAG - - echo "TAG=$TAG" - - env TAG - ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - "./tools/travis/build_tag_releases.sh $build_file_name $GIT_TAG" - export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)" - echo "Deploying $RELEASE_PKG_FILE to GitHub releases." - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_EVENT_TYPE" == "push" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then @@ -94,14 +123,10 @@ before_deploy: git push -f -q https://$API_KEY@github.com/apache/openwhisk-wskdeploy $GIT_TAG; fi - echo "Completing before_deploy stage" - - echo "TRAVIS_EVENT_TYPE is " - - echo $TRAVIS_EVENT_TYPE - - env TRAVIS_EVENT_TYPE - - echo "GIT_TAG is " - - echo $GIT_TAG - - env GIT_TAG - - echo "GIT_TAG is ${GIT_TAG}" +# Utilize two providers to create a github release and publish a docker image. +# For information on the "releases" provider, +# see https://docs.travis-ci.com/user/deployment/releases/ deploy: - provider: releases api_key: @@ -125,6 +150,15 @@ deploy: repo: apache/openwhisk-wskdeploy tags: true condition: "$DEPLOY_BUILD_READY = true" + +# Clean up wskdeploy binary (single) +# TODO: See if this is really needed in Travis environments as the +# Makefile is not used elsewhere here (especially if we do not use th +# build_tag_release.sh script) as it is for Windows developers primarily +# and we are building using Linux. +after_script: + - make clean + env: global: - BLUEMIX_APIHOST=openwhisk.ng.bluemix.net diff --git a/build.gradle b/build.gradle index 4c9ce40d5..92834222e 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ rat { storage is in "GO" format: OS: linux, darwin, windows - Arch: 386, amd64, s390x, ppc64le, arm + Arch: 386, amd64, s390x, ppc64le, arm, arm64 TODO - It may be appropriate to refactor into a general purpose Platform class for all builds, then to extend with specifics needed for diff --git a/go.mod b/go.mod index 998c63dac..f84526cd7 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,12 @@ require ( github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 github.com/fatih/color v1.10.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 - github.com/google/go-querystring v1.0.0 // indirect github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 github.com/magiconair/properties v1.8.4 // indirect github.com/mattn/go-colorable v0.1.8 github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/nicksnyder/go-i18n v1.10.1 github.com/nxadm/tail v1.4.8 // indirect - github.com/onsi/ginkgo v1.15.0 // indirect github.com/pelletier/go-toml v1.8.1 // indirect github.com/spf13/afero v1.5.1 // indirect github.com/spf13/cast v1.3.1 // indirect @@ -26,5 +24,4 @@ require ( golang.org/x/text v0.3.5 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 514be3d0e..da6772436 100644 --- a/go.sum +++ b/go.sum @@ -16,10 +16,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/apache/openwhisk-client-go v0.0.0-20200927152356-49c58e00354d h1:eRcVp/TIKFqI7adU1egUr4IYGhOA1PYu4B5C1IvwSgI= -github.com/apache/openwhisk-client-go v0.0.0-20200927152356-49c58e00354d/go.mod h1:jLLKYP7+1+LFlIJW1n9U1gqeveLM1HIwa4ZHNOFxjPw= -github.com/apache/openwhisk-client-go v0.0.0-20210308161059-5cd1006dc35f h1:SVXVE9+snbgp5vtxfOQ7QrANTZQ/bcP9IFev6HXrtWc= -github.com/apache/openwhisk-client-go v0.0.0-20210308161059-5cd1006dc35f/go.mod h1:SAQU4bHGJ0sg6c1vQ8ojmQKXgGaneVnexWX4+2/KMr8= github.com/apache/openwhisk-client-go v0.0.0-20210311185314-87edc2364717 h1:7MsAB3W6JH0d9TZ4UJ55rwV8AR9jXXZf97Uzk+CJqqs= github.com/apache/openwhisk-client-go v0.0.0-20210311185314-87edc2364717/go.mod h1:SAQU4bHGJ0sg6c1vQ8ojmQKXgGaneVnexWX4+2/KMr8= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -43,6 +39,7 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -180,6 +177,7 @@ github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISq github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -191,6 +189,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -302,6 +301,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dD golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/tools/travis/build_tag_releases.sh b/tools/travis/build_tag_releases.sh index 074dd27f9..b6a74160d 100755 --- a/tools/travis/build_tag_releases.sh +++ b/tools/travis/build_tag_releases.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # - +set -x declare -a os_list=("linux" "darwin" "windows") declare -a arc_list=("amd64" "386") build_file_name=${1:-"wskdeploy"} @@ -23,6 +23,13 @@ build_version=${2:-"$TRAVIS_TAG"} gitCommit=$(git rev-parse HEAD) buildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +echo "build_file_name: $build_file_name" +echo "buildDate: $buildDate" +echo "build_version: $build_version" +echo "gitCommit: $gitCommit" + +printenv $TRAVIS_TAG + for os in "${os_list[@]}" do for arc in "${arc_list[@]}" From 9b3ba853ff3d457651edc50b0a8153b16feba3df Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Wed, 31 Mar 2021 10:48:43 -0500 Subject: [PATCH 05/12] Improve Travis builds --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9f7a014ff..d7b11b754 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,7 @@ Packages/ .gradle .gogradle build/ -releases/ +release/ #emacs *~ From f4166731a7231a769e93b2862d0d9ce805a0a026 Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Thu, 1 Apr 2021 11:48:44 -0500 Subject: [PATCH 06/12] improve task information for logs --- .travis.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b7281a8c6..22ac4deb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,6 +51,7 @@ notifications: before_install: - "./tools/travis/cloneutils.sh" + - sudo apt-get install tree # Install tools we use for linting and unit/integration testing # TODO: build.gradle (gradlew) already installs 'golint' and runs linting @@ -76,11 +77,24 @@ before_script: # Clone and build the OpenWhisk platform (i.e., apache/openwhisk) # and then use Ansible to deploy it and prepare it to run our # integration tests. +# TODO: Do not use Makefile for running integration tests; +# change to using gradle wrapper script: - echo $TRAVIS - echo $TRAVIS_PULL_REQUEST - echo $TRAVIS_SECURE_ENV_VARS - - printenv + - echo "TRAVIS_BRANCH = [$TRAVIS_BRANCH]" + - echo "TRAVIS_EVENT_TYPE = [$TRAVIS_EVENT_TYPE]" + - echo "GIT_TAG = [$GIT_TAG]" + - echo "TAG = [$TAG]" + - env TRAVIS_BRANCH + - env TRAVIS_EVENT_TYPE + - env $GIT_TAG + - env $TAG + - echo "Installing and building OpenWhisk platform using Ansible..." + - echo "Running Golang integration tests" + - printenv | sort + - tree tests/src/integration/ - "./tools/travis/script.sh" after_success: @@ -112,6 +126,8 @@ before_deploy: - env $TAG - echo "Using Gradle Wrapper to produce release binaries..." - ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - tree /build + - tree /release - export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)" - echo "Deploying $RELEASE_PKG_FILE to GitHub releases." - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_EVENT_TYPE" == "push" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then From 9986799886cd73ee851cc18dcd65e545f29c82b4 Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Thu, 1 Apr 2021 14:45:19 -0500 Subject: [PATCH 07/12] improve task information for logs --- .travis.yml | 15 ++++++--------- main.go | 9 +++++++-- tools/travis/showenv.sh | 28 ++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 11 deletions(-) create mode 100755 tools/travis/showenv.sh diff --git a/.travis.yml b/.travis.yml index 22ac4deb3..31b9e0077 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,19 +82,16 @@ before_script: script: - echo $TRAVIS - echo $TRAVIS_PULL_REQUEST - - echo $TRAVIS_SECURE_ENV_VARS - - echo "TRAVIS_BRANCH = [$TRAVIS_BRANCH]" - - echo "TRAVIS_EVENT_TYPE = [$TRAVIS_EVENT_TYPE]" - - echo "GIT_TAG = [$GIT_TAG]" - - echo "TAG = [$TAG]" - - env TRAVIS_BRANCH - - env TRAVIS_EVENT_TYPE - - env $GIT_TAG - - env $TAG + - echo "$TRAVIS_SECURE_ENV_VARS + - echo "TRAVIS_BRANCH = ["$TRAVIS_BRANCH"]" + - echo "TRAVIS_EVENT_TYPE = ["$TRAVIS_EVENT_TYPE"]" + - echo "GIT_TAG = ["$GIT_TAG"]" + - echo "TAG = [$"TAG"]" - echo "Installing and building OpenWhisk platform using Ansible..." - echo "Running Golang integration tests" - printenv | sort - tree tests/src/integration/ + - "./tools/travis/showenv.sh" - "./tools/travis/script.sh" after_success: diff --git a/main.go b/main.go index 5a4fe1cd1..784d885df 100644 --- a/main.go +++ b/main.go @@ -26,10 +26,15 @@ func main() { cmd.Execute() } +// Struct used to hold tagged (release) build information +// Which is displayed by the `version` command. +// Added automatically for CI/CD Travis builds in GitHub var ( - //Version ...The Version of the tool - Version = "unset" + // Apache OpenWhisk Whisk Deploy release/build version + Version = "unset" + // Git source code commit # that initiated the build GitCommit = "unset" + // Date stamp indicating when build was initiated BuildDate = "unset" ) diff --git a/tools/travis/showenv.sh b/tools/travis/showenv.sh new file mode 100755 index 000000000..305ede3b7 --- /dev/null +++ b/tools/travis/showenv.sh @@ -0,0 +1,28 @@ + #!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + +echo "================================================" +echo "TRAVIS = ["$TRAVIS"]" +echo "TRAVIS_PULL_REQUEST = ["$TRAVIS_PULL_REQUEST"]" +echo "TRAVIS_SECURE_ENV_VARS = ["$TRAVIS_SECURE_ENV_VARS"]" +echo "TRAVIS_BRANCH = ["$TRAVIS_BRANCH"]" +echo "TRAVIS_EVENT_TYPE = ["$TRAVIS_EVENT_TYPE"]" +echo "GIT_TAG = ["$GIT_TAG"]" +echo "TAG = ["$TAG"]" +echo "================================================" From 8291ca761f134b3e9c65989d8d6ddbc6f8273022 Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Thu, 1 Apr 2021 17:19:43 -0500 Subject: [PATCH 08/12] improve task information for logs --- .travis.yml | 33 +++++++++++++-------------------- tools/travis/script.sh | 4 ++++ tools/travis/showenv.sh | 2 ++ 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 31b9e0077..a4a81a384 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,18 +79,15 @@ before_script: # integration tests. # TODO: Do not use Makefile for running integration tests; # change to using gradle wrapper +# - echo $TRAVIS +# - echo $TRAVIS_PULL_REQUEST +# - echo "$TRAVIS_SECURE_ENV_VARS +# - echo "TRAVIS_BRANCH = ["$TRAVIS_BRANCH"]" +# - echo "TRAVIS_EVENT_TYPE = ["$TRAVIS_EVENT_TYPE"]" +# - echo "GIT_TAG = ["$GIT_TAG"]" +# - echo "TAG = [$"TAG"]" script: - - echo $TRAVIS - - echo $TRAVIS_PULL_REQUEST - - echo "$TRAVIS_SECURE_ENV_VARS - - echo "TRAVIS_BRANCH = ["$TRAVIS_BRANCH"]" - - echo "TRAVIS_EVENT_TYPE = ["$TRAVIS_EVENT_TYPE"]" - - echo "GIT_TAG = ["$GIT_TAG"]" - - echo "TAG = [$"TAG"]" - echo "Installing and building OpenWhisk platform using Ansible..." - - echo "Running Golang integration tests" - - printenv | sort - - tree tests/src/integration/ - "./tools/travis/showenv.sh" - "./tools/travis/script.sh" @@ -113,20 +110,14 @@ before_deploy: export GIT_TAG=$TRAVIS_TAG; export TAG=true; fi - - echo "TRAVIS_BRANCH = [$TRAVIS_BRANCH]" - - echo "TRAVIS_EVENT_TYPE = [$TRAVIS_EVENT_TYPE]" - - echo "GIT_TAG = [$GIT_TAG]" - - echo "TAG = [$TAG]" - - env TRAVIS_BRANCH - - env TRAVIS_EVENT_TYPE - - env $GIT_TAG - - env $TAG - - echo "Using Gradle Wrapper to produce release binaries..." + - echo "Calling Gradle Wrapper task to build and 'releaseBinaries'..." + - "./tools/travis/showenv.sh" - ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - echo "Build binaries and release archives created:" - tree /build - tree /release - export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)" - - echo "Deploying $RELEASE_PKG_FILE to GitHub releases." + - echo "Deploying "$RELEASE_PKG_FILE" to GitHub releases..." - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_EVENT_TYPE" == "push" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then git config --global user.email "builds@travis-ci.com"; git config --global user.name "Travis CI"; @@ -171,6 +162,8 @@ deploy: # and we are building using Linux. after_script: - make clean + - tree /build + - tree /release env: global: diff --git a/tools/travis/script.sh b/tools/travis/script.sh index 12d534b6b..80de24797 100755 --- a/tools/travis/script.sh +++ b/tools/travis/script.sh @@ -59,5 +59,9 @@ $ANSIBLE_CMD routemgmt.yml export OPENWHISK_HOME="$(dirname "$TRAVIS_BUILD_DIR")/openwhisk" +# TODO: Implement integration tests in GradleWrapper as new task +# then use that version instead of invoking from the makefile +echo "Running Golang integration tests" +tree tests/src/integration/ cd $TRAVIS_BUILD_DIR make integration_test diff --git a/tools/travis/showenv.sh b/tools/travis/showenv.sh index 305ede3b7..67f10d957 100755 --- a/tools/travis/showenv.sh +++ b/tools/travis/showenv.sh @@ -26,3 +26,5 @@ echo "TRAVIS_EVENT_TYPE = ["$TRAVIS_EVENT_TYPE"]" echo "GIT_TAG = ["$GIT_TAG"]" echo "TAG = ["$TAG"]" echo "================================================" + +printenv | sort From 6a592672a7606aa992ad31a1f8e4ff4b83543e9a Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Mon, 5 Apr 2021 10:51:32 -0500 Subject: [PATCH 09/12] Add spew for export commands that get SIGINT faults --- .travis.yml | 2 +- cmd/export.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a4a81a384..b7105f5da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,7 +100,7 @@ after_success: # If this is a "tagged" GitHub build, then use Gradle Wrapper # to build binaries for all supported OS platforms and architectures. # The result would be a "/build" directory that has a subdirectory named -# for each supported GOOS/ +# for each supported GOOS/GOARCH before_deploy: - export build_file_name=wskdeploy - go get github.com/inconshreveable/mousetrap diff --git a/cmd/export.go b/cmd/export.go index 402007582..73289c1a2 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -31,6 +31,7 @@ import ( "github.com/apache/openwhisk-wskdeploy/utils" "github.com/apache/openwhisk-wskdeploy/wski18n" "github.com/apache/openwhisk-wskdeploy/wskprint" + "github.com/davecgh/go-spew/spew" "github.com/spf13/cobra" ) @@ -128,18 +129,24 @@ func exportProject(projectName string, targetManifest string) error { if err != nil { return err } + spew.Dump(packages) var bindings = make(map[string]whisk.Binding) + spew.Dump(bindings) // iterate over each package to find managed annotations // check if "managed" annotation is attached to a package // add to export when managed project name matches with the // specified project name for _, pkg := range packages { + spew.Dump(pkg) + if a := pkg.Annotations.GetValue(utils.MANAGED); a != nil { // decode the JSON blob and retrieve __OW_PROJECT_NAME pa := a.(map[string]interface{}) + spew.Dump(pa) + // we have found a package which is part of the current project if pa[utils.OW_PROJECT_NAME] == projectName { @@ -199,6 +206,8 @@ func exportProject(projectName string, targetManifest string) error { // iterate over the list of triggers to determine whether any of them part of specified managed project for _, trg := range triggers { + spew.Dump(trg) + // trigger has attached managed annotation if a := trg.Annotations.GetValue(utils.MANAGED); a != nil { // decode the JSON blob and retrieve __OW_PROJECT_NAME @@ -263,6 +272,8 @@ func exportProject(projectName string, targetManifest string) error { // iterate over the list of rules to determine whether any of them is part of the manage dproject for _, rule := range rules { + spew.Dump(rule) + // get rule from OW wskRule, _, _ := client.Rules.Get(rule.Name) // rule has attached managed annotation @@ -308,6 +319,7 @@ func exportProject(projectName string, targetManifest string) error { // iterate over the list of APIs to determine whether any of them part of the managed project retApiArray := (*whisk.RetApiArray)(retApiList) for _, api := range retApiArray.Apis { + spew.Dump(api) apiName := api.ApiValue.Swagger.Info.Title apiBasePath := strings.TrimPrefix(api.ApiValue.Swagger.BasePath, "/") From 24619460cc0cf5ce1a20f3c3332c9bbafac82304 Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Mon, 5 Apr 2021 14:29:47 -0500 Subject: [PATCH 10/12] Add spew for export commands that get SIGINT faults --- cmd/export.go | 28 +++++++++++++++-------- go.mod | 1 + tests/src/integration/common/wskdeploy.go | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/export.go b/cmd/export.go index 73289c1a2..d3afe9e29 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -129,24 +129,23 @@ func exportProject(projectName string, targetManifest string) error { if err != nil { return err } - spew.Dump(packages) + + if wskprint.DetectVerbose() { + spew.Dump(packages) + } var bindings = make(map[string]whisk.Binding) - spew.Dump(bindings) // iterate over each package to find managed annotations // check if "managed" annotation is attached to a package // add to export when managed project name matches with the // specified project name for _, pkg := range packages { - spew.Dump(pkg) if a := pkg.Annotations.GetValue(utils.MANAGED); a != nil { // decode the JSON blob and retrieve __OW_PROJECT_NAME pa := a.(map[string]interface{}) - spew.Dump(pa) - // we have found a package which is part of the current project if pa[utils.OW_PROJECT_NAME] == projectName { @@ -204,10 +203,12 @@ func exportProject(projectName string, targetManifest string) error { return err } + if wskprint.DetectVerbose() { + spew.Dump(triggers) + } + // iterate over the list of triggers to determine whether any of them part of specified managed project for _, trg := range triggers { - spew.Dump(trg) - // trigger has attached managed annotation if a := trg.Annotations.GetValue(utils.MANAGED); a != nil { // decode the JSON blob and retrieve __OW_PROJECT_NAME @@ -270,9 +271,12 @@ func exportProject(projectName string, targetManifest string) error { return err } + if wskprint.DetectVerbose() { + spew.Dump(rules) + } + // iterate over the list of rules to determine whether any of them is part of the manage dproject for _, rule := range rules { - spew.Dump(rule) // get rule from OW wskRule, _, _ := client.Rules.Get(rule.Name) @@ -319,7 +323,6 @@ func exportProject(projectName string, targetManifest string) error { // iterate over the list of APIs to determine whether any of them part of the managed project retApiArray := (*whisk.RetApiArray)(retApiList) for _, api := range retApiArray.Apis { - spew.Dump(api) apiName := api.ApiValue.Swagger.Info.Title apiBasePath := strings.TrimPrefix(api.ApiValue.Swagger.BasePath, "/") @@ -398,7 +401,12 @@ func exportProject(projectName string, targetManifest string) error { // find exported manifest parent directory manifestDir := filepath.Dir(utils.Flags.ManifestPath) - os.MkdirAll(manifestDir, os.ModePerm) + err1 := os.MkdirAll(manifestDir, os.ModePerm) + + if err1 != nil { + wskprint.PrintOpenWhiskError(err1.Error()) + return err1 + } // export manifest to file parsers.Write(maniyaml, targetManifest) diff --git a/go.mod b/go.mod index f84526cd7..26048ad20 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/apache/openwhisk-client-go v0.0.0-20210311185314-87edc2364717 github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/fatih/color v1.10.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 diff --git a/tests/src/integration/common/wskdeploy.go b/tests/src/integration/common/wskdeploy.go index 07773203d..fb8dac55d 100644 --- a/tests/src/integration/common/wskdeploy.go +++ b/tests/src/integration/common/wskdeploy.go @@ -195,7 +195,7 @@ func (Wskdeploy *Wskdeploy) HeadlessManagedDeployment(manifestPath string, deplo } func (wskdeploy *Wskdeploy) ExportProject(projectName string, targetManifestPath string) (string, error) { - return wskdeploy.RunCommand("export", "-m", targetManifestPath, "--projectname", projectName) + return wskdeploy.RunCommand("export", "-m", targetManifestPath, "--projectname", projectName, "-v") } func (wskdeploy *Wskdeploy) ExportProjectWithCredentials(projectName string, targetManifestPath string, wskprops *whisk.Wskprops) (string, error) { From 1e4bf3be958cfb3f79eb63f8c878f0b7a9d898b2 Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Mon, 5 Apr 2021 17:05:55 -0500 Subject: [PATCH 11/12] Add trace around intermittent failings on export project --- cmd/export.go | 139 +++++++++++----------- tests/src/integration/common/wskdeploy.go | 8 +- wskprint/console.go | 7 +- 3 files changed, 83 insertions(+), 71 deletions(-) diff --git a/cmd/export.go b/cmd/export.go index d3afe9e29..c5eecd930 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -29,6 +29,7 @@ import ( "github.com/apache/openwhisk-wskdeploy/parsers" "github.com/apache/openwhisk-wskdeploy/runtimes" "github.com/apache/openwhisk-wskdeploy/utils" + "github.com/apache/openwhisk-wskdeploy/wskderrors" "github.com/apache/openwhisk-wskdeploy/wski18n" "github.com/apache/openwhisk-wskdeploy/wskprint" "github.com/davecgh/go-spew/spew" @@ -130,7 +131,7 @@ func exportProject(projectName string, targetManifest string) error { return err } - if wskprint.DetectVerbose() { + if utils.Flags.Trace { spew.Dump(packages) } @@ -203,65 +204,68 @@ func exportProject(projectName string, targetManifest string) error { return err } - if wskprint.DetectVerbose() { - spew.Dump(triggers) - } - // iterate over the list of triggers to determine whether any of them part of specified managed project for _, trg := range triggers { - // trigger has attached managed annotation - if a := trg.Annotations.GetValue(utils.MANAGED); a != nil { - // decode the JSON blob and retrieve __OW_PROJECT_NAME - ta := a.(map[string]interface{}) - if ta[utils.OW_PROJECT_NAME] == projectName { - - for pkgName := range maniyaml.Packages { - if maniyaml.Packages[pkgName].Namespace == trg.Namespace { - if maniyaml.Packages[pkgName].Triggers == nil { - pkg := maniyaml.Packages[pkgName] - pkg.Triggers = make(map[string]parsers.Trigger) - maniyaml.Packages[pkgName] = pkg - } - // export trigger to manifest - - if feedname, isFeed := utils.IsFeedAction(&trg); isFeed { - // check if feed name starts with namespace and workaround it - // the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action - // we need to transform the path from e.g. - // /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions//whisk.system/alarms/interval?blocking=true - // in to - // /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true - if strings.HasPrefix(feedname, "/") { - // /whisk.system/alarms/interval -> ../../whisk.system/actions/alarms/interval - prts := strings.SplitN(feedname, "/", 3) - feedname = "../../" + prts[1] + "/actions/" + prts[2] + // trigger has attached managed annotation + if trg.Annotations != nil { + + if a := trg.Annotations.GetValue(utils.MANAGED); a != nil { + // decode the JSON blob and retrieve __OW_PROJECT_NAME + ta := a.(map[string]interface{}) + if ta[utils.OW_PROJECT_NAME] == projectName { + + for pkgName := range maniyaml.Packages { + if maniyaml.Packages[pkgName].Namespace == trg.Namespace { + if maniyaml.Packages[pkgName].Triggers == nil { + pkg := maniyaml.Packages[pkgName] + pkg.Triggers = make(map[string]parsers.Trigger) + maniyaml.Packages[pkgName] = pkg } - // export feed input parameters - params := make(map[string]interface{}) - params["authKey"] = client.Config.AuthToken - params["lifecycleEvent"] = "READ" - params["triggerName"] = "/" + client.Namespace + "/" + trg.Name - res, _, err := client.Actions.Invoke(feedname, params, true, true) - if err != nil { - return err - } - feedConfig := res["config"] + // export trigger to manifest + + if feedname, isFeed := utils.IsFeedAction(&trg); isFeed { + // check if feed name starts with namespace and workaround it + // the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action + // we need to transform the path from e.g. + // /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions//whisk.system/alarms/interval?blocking=true + // in to + // /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true + if strings.HasPrefix(feedname, "/") { + // /whisk.system/alarms/interval -> ../../whisk.system/actions/alarms/interval + prts := strings.SplitN(feedname, "/", 3) + feedname = "../../" + prts[1] + "/actions/" + prts[2] + } - if feedConfig != nil { - for key, val := range feedConfig.(map[string]interface{}) { - if key != "startDate" { - trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val}) + // export feed input parameters + params := make(map[string]interface{}) + params["authKey"] = client.Config.AuthToken + params["lifecycleEvent"] = "READ" + params["triggerName"] = "/" + client.Namespace + "/" + trg.Name + res, _, err := client.Actions.Invoke(feedname, params, true, true) + if err != nil { + return err + } + feedConfig := res["config"] + + if feedConfig != nil { + for key, val := range feedConfig.(map[string]interface{}) { + if key != "startDate" { + trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val}) + } } } } - } - maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg) + maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg) + } } } } + } else { + spew.Dump(trg) + return wskderrors.NewCommandError("Export", "Trigger missing annotations.") } } @@ -271,36 +275,37 @@ func exportProject(projectName string, targetManifest string) error { return err } - if wskprint.DetectVerbose() { - spew.Dump(rules) - } - // iterate over the list of rules to determine whether any of them is part of the manage dproject for _, rule := range rules { // get rule from OW wskRule, _, _ := client.Rules.Get(rule.Name) - // rule has attached managed annotation - if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil { - // decode the JSON blob and retrieve __OW_PROJECT_NAME - ta := a.(map[string]interface{}) - if ta[utils.OW_PROJECT_NAME] == projectName { - - for pkgName := range maniyaml.Packages { - if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace { - if maniyaml.Packages[pkgName].Rules == nil { - pkg := maniyaml.Packages[pkgName] - pkg.Rules = make(map[string]parsers.Rule) - maniyaml.Packages[pkgName] = pkg - } - // export rule to manifest - maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule) + if wskRule.Annotations != nil { + // rule has attached managed annotation + if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil { + // decode the JSON blob and retrieve __OW_PROJECT_NAME + ta := a.(map[string]interface{}) + if ta[utils.OW_PROJECT_NAME] == projectName { + + for pkgName := range maniyaml.Packages { + if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace { + if maniyaml.Packages[pkgName].Rules == nil { + pkg := maniyaml.Packages[pkgName] + pkg.Rules = make(map[string]parsers.Rule) + maniyaml.Packages[pkgName] = pkg + } + + // export rule to manifest + maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule) + } } } } + } else { + spew.Dump(wskRule) + return wskderrors.NewCommandError("Export", "Rule missing annotations.") } - } // API Gateway is an optional component. Export APIs only when ApigwAccessToken is configured diff --git a/tests/src/integration/common/wskdeploy.go b/tests/src/integration/common/wskdeploy.go index fb8dac55d..1b9e49386 100644 --- a/tests/src/integration/common/wskdeploy.go +++ b/tests/src/integration/common/wskdeploy.go @@ -194,8 +194,14 @@ func (Wskdeploy *Wskdeploy) HeadlessManagedDeployment(manifestPath string, deplo return Wskdeploy.RunCommand("sync", "-m", manifestPath, "-d", deploymentPath, "--projectname", name) } +// Note: this test often fails due to server-related issues trying to access +// newly created objects (e.g., primarily Rules) too quickly which often lose Annotations. +// causing SIGSEGV errors. +// It also sometimes fails accessing local storage (for tmp directories and files +// that are created in Travis (constrained, shared environments). +// You can add the "-t" or "--trace" to the RunCommand to get extensive trace on all HTTP calls func (wskdeploy *Wskdeploy) ExportProject(projectName string, targetManifestPath string) (string, error) { - return wskdeploy.RunCommand("export", "-m", targetManifestPath, "--projectname", projectName, "-v") + return wskdeploy.RunCommand("export", "-m", targetManifestPath, "--projectname", projectName, "-t") } func (wskdeploy *Wskdeploy) ExportProjectWithCredentials(projectName string, targetManifestPath string, wskprops *whisk.Wskprops) (string, error) { diff --git a/wskprint/console.go b/wskprint/console.go index 91ee5932a..8367eb563 100644 --- a/wskprint/console.go +++ b/wskprint/console.go @@ -19,13 +19,14 @@ package wskprint import ( "fmt" - "github.com/apache/openwhisk-wskdeploy/wski18n" - "github.com/fatih/color" - "github.com/mattn/go-colorable" "os" "path/filepath" "runtime" "strings" + + "github.com/apache/openwhisk-wskdeploy/wski18n" + "github.com/fatih/color" + "github.com/mattn/go-colorable" ) const ( From 200e3f523b6077eb9f79f45b5ffebc6e594fdbe1 Mon Sep 17 00:00:00 2001 From: Matt Rutkowski Date: Tue, 6 Apr 2021 12:07:50 -0500 Subject: [PATCH 12/12] Cleanup changes and add comments and TODOs --- .travis.yml | 12 +++--------- cmd/export.go | 17 +++++++++-------- tests/src/integration/common/wskdeploy.go | 9 ++++++++- tests/src/integration/export/export_test.go | 6 ------ tools/travis/build_tag_releases.sh | 13 ++++++------- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7105f5da..93568a3bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ notifications: # travis2slack webhook to enable DMs on openwhisk-team.slack.com to PR authors with TravisCI results secure: "NaDCtHwbFUEy0h0u/QrAt395o0/9FstJ5/Tv4uBSLCBdaUNU+0q6/RXVH76dKbujoxrLded2gyeJSAbI4kl4SM7FyilSHzKTikF0VsgM3G5veh65tVY5ztsvl40hDmK2E/PmWI3HkZNeWJO32WjraqPmhSWL0BZOWhA/4gJSCIsLnNRQXmpZoDqC7V8IG1BWYGwd5qcNkEjItLjYCUy1s2K8bj3QmhshYRVLTNbXxh0yJVKjM04I+bWhMXqvL8GS55qeicJ3fUwm5g1PDVCtzxLwAHCPo2jusjtNSnV+BaZgMBSahv3MD7ApxfF58e1buejggH3qZAhNGccC4bYJCahqVv/KKoA10kO6exH5iGwlHPWQTjMuF++PHmAk+FeQ1jh+JoUSBnIHExPnaD4CQIArHvUuQom+WJBnOz+L1H755VyPGzeDQ+ZUDlhOiQ6CDP/sqaRH3Wmo4IzhsqTsLaQs0dS1YeEWOS8gcawwNTVf/WtqiedYu24rQN9RiUXelRoEaXJy/pldb2KpyQRo2hbKoStDDB5fEij1xyGsj4kSXZ8uk5G10oDGnZYd+okduzJazN8wr0flrLvH32DRgDZDcG/D90I19JmqqfqbcG/SEmhxTikHZeVQd5DfWcqssLvWNrtQa/EJgAQkmZ2gaSsZ73AvPk/KyMV30+m8xTU=" -# The complete sequence of phases of a job is the lifecycle: +# The complete sequence of phases of the lifecycle: # --------------------------------------------------------- # before_install # install @@ -77,15 +77,8 @@ before_script: # Clone and build the OpenWhisk platform (i.e., apache/openwhisk) # and then use Ansible to deploy it and prepare it to run our # integration tests. -# TODO: Do not use Makefile for running integration tests; +# TODO: Do not use Makefile for running integration tests (in script.sh); # change to using gradle wrapper -# - echo $TRAVIS -# - echo $TRAVIS_PULL_REQUEST -# - echo "$TRAVIS_SECURE_ENV_VARS -# - echo "TRAVIS_BRANCH = ["$TRAVIS_BRANCH"]" -# - echo "TRAVIS_EVENT_TYPE = ["$TRAVIS_EVENT_TYPE"]" -# - echo "GIT_TAG = ["$GIT_TAG"]" -# - echo "TAG = [$"TAG"]" script: - echo "Installing and building OpenWhisk platform using Ansible..." - "./tools/travis/showenv.sh" @@ -156,6 +149,7 @@ deploy: condition: "$DEPLOY_BUILD_READY = true" # Clean up wskdeploy binary (single) +# Fundamentally, remove both /build and /release directory trees and contents # TODO: See if this is really needed in Travis environments as the # Makefile is not used elsewhere here (especially if we do not use th # build_tag_release.sh script) as it is for Windows developers primarily diff --git a/cmd/export.go b/cmd/export.go index c5eecd930..aa8ee7daa 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -119,9 +119,6 @@ func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML, func exportProject(projectName string, targetManifest string) error { - whisk.SetVerbose(utils.Flags.Verbose) - whisk.SetDebug(utils.Flags.Trace) - maniyaml := &parsers.YAML{} maniyaml.Project.Name = projectName @@ -131,6 +128,7 @@ func exportProject(projectName string, targetManifest string) error { return err } + // Emit additional trace data (primarily in Travis) if utils.Flags.Trace { spew.Dump(packages) } @@ -209,7 +207,6 @@ func exportProject(projectName string, targetManifest string) error { // trigger has attached managed annotation if trg.Annotations != nil { - if a := trg.Annotations.GetValue(utils.MANAGED); a != nil { // decode the JSON blob and retrieve __OW_PROJECT_NAME ta := a.(map[string]interface{}) @@ -264,6 +261,7 @@ func exportProject(projectName string, targetManifest string) error { } } } else { + // Emit additional trace data on common Travis failures spew.Dump(trg) return wskderrors.NewCommandError("Export", "Trigger missing annotations.") } @@ -303,6 +301,7 @@ func exportProject(projectName string, targetManifest string) error { } } } else { + // Emit additional trace data on common Travis failures spew.Dump(wskRule) return wskderrors.NewCommandError("Export", "Rule missing annotations.") } @@ -406,11 +405,13 @@ func exportProject(projectName string, targetManifest string) error { // find exported manifest parent directory manifestDir := filepath.Dir(utils.Flags.ManifestPath) - err1 := os.MkdirAll(manifestDir, os.ModePerm) + errMkDir := os.MkdirAll(manifestDir, os.ModePerm) - if err1 != nil { - wskprint.PrintOpenWhiskError(err1.Error()) - return err1 + // Exit if unable to create export dir. structure + // TODO: This sometimes fails in Travis, perhaps retry? + if errMkDir != nil { + wskprint.PrintOpenWhiskError(errMkDir.Error()) + return errMkDir } // export manifest to file diff --git a/tests/src/integration/common/wskdeploy.go b/tests/src/integration/common/wskdeploy.go index 1b9e49386..8c3908a34 100644 --- a/tests/src/integration/common/wskdeploy.go +++ b/tests/src/integration/common/wskdeploy.go @@ -26,6 +26,7 @@ import ( "path" "path/filepath" "strings" + "time" "github.com/apache/openwhisk-client-go/whisk" "github.com/apache/openwhisk-wskdeploy/dependencies" @@ -201,10 +202,16 @@ func (Wskdeploy *Wskdeploy) HeadlessManagedDeployment(manifestPath string, deplo // that are created in Travis (constrained, shared environments). // You can add the "-t" or "--trace" to the RunCommand to get extensive trace on all HTTP calls func (wskdeploy *Wskdeploy) ExportProject(projectName string, targetManifestPath string) (string, error) { - return wskdeploy.RunCommand("export", "-m", targetManifestPath, "--projectname", projectName, "-t") + // TODO: There is a server-side timing issue with Annotations being + // retrieved too soon after creation; need to explore in OW main + time.Sleep(3 * time.Second) // should it sleep for few seconds before export?! + return wskdeploy.RunCommand("export", "-m", targetManifestPath, "--projectname", projectName) } func (wskdeploy *Wskdeploy) ExportProjectWithCredentials(projectName string, targetManifestPath string, wskprops *whisk.Wskprops) (string, error) { + // TODO: There is a server-side timing issue with Annotations being + // retrieved too soon after creation; need to explore in OW main + time.Sleep(3 * time.Second) // should it sleep for few seconds before export?! return wskdeploy.RunCommand("export", "-m", targetManifestPath, "--projectname", projectName, "--auth", wskprops.AuthKey, "--namespace", wskprops.Namespace, "--apihost", wskprops.APIHost, "--apiversion", wskprops.Apiversion) } diff --git a/tests/src/integration/export/export_test.go b/tests/src/integration/export/export_test.go index bdcbd430c..2ec86c651 100644 --- a/tests/src/integration/export/export_test.go +++ b/tests/src/integration/export/export_test.go @@ -25,7 +25,6 @@ import ( "os" "strconv" "testing" - "time" "github.com/apache/openwhisk-wskdeploy/tests/src/integration/common" "github.com/stretchr/testify/assert" @@ -54,8 +53,6 @@ func TestExport(t *testing.T) { _, err = wskdeploy.ManagedDeploymentOnlyManifest(manifestExtPath) assert.Equal(t, nil, err, "Failed to deploy the ext manifest file.") - time.Sleep(2 * time.Second) // should it sleep for few seconds before export?! - _, err = wskdeploy.ExportProject(projectName, targetManifestPath) assert.Equal(t, nil, err, "Failed to export project.") @@ -127,7 +124,6 @@ func TestExport2Pack(t *testing.T) { _, err := wskdeploy.ManagedDeploymentOnlyManifest(manifest2PackPath) assert.Equal(t, nil, err, "Failed to deploy the 2pack manifest file.") - time.Sleep(2 * time.Second) // should it sleep for few seconds before export?! _, err = wskdeploy.ExportProject(projectName, target2PackManifestPath) assert.Equal(t, nil, err, "Failed to export project.") @@ -153,7 +149,6 @@ func TestExportApi(t *testing.T) { _, err := wskdeploy.ManagedDeploymentManifestAndProject(manifestApiExpPath, projectName) assert.Equal(t, nil, err, "Failed to deploy the ApiExp manifest file.") - time.Sleep(2 * time.Second) // should it sleep for few seconds before export?! _, err = wskdeploy.ExportProject(projectName, targetApiExpManifestPath) assert.Equal(t, nil, err, "Failed to export project.") @@ -189,7 +184,6 @@ func SkipTestExportTriggerFeed(t *testing.T) { _, err = wskdeploy.ManagedDeploymentManifestAndProjectWithCredentials(manifestFeedExpPath, projectName, wskprops) assert.Equal(t, nil, err, "Failed to deploy the FeedExp manifest file.") - time.Sleep(2 * time.Second) // should it sleep for few seconds before export?! _, err = wskdeploy.ExportProjectWithCredentials(projectName, targetFeedExpManifestPath, wskprops) assert.Equal(t, nil, err, "Failed to export project with trigger feed.") diff --git a/tools/travis/build_tag_releases.sh b/tools/travis/build_tag_releases.sh index b6a74160d..87726924f 100755 --- a/tools/travis/build_tag_releases.sh +++ b/tools/travis/build_tag_releases.sh @@ -20,15 +20,14 @@ declare -a os_list=("linux" "darwin" "windows") declare -a arc_list=("amd64" "386") build_file_name=${1:-"wskdeploy"} build_version=${2:-"$TRAVIS_TAG"} -gitCommit=$(git rev-parse HEAD) buildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +gitCommit=$(git rev-parse HEAD) -echo "build_file_name: $build_file_name" -echo "buildDate: $buildDate" -echo "build_version: $build_version" -echo "gitCommit: $gitCommit" - -printenv $TRAVIS_TAG +echo "build_file_name: ["$build_file_name"]" +echo "build_version: ["$build_version"]" +echo "buildDate: ["$buildDate"]" +echo "gitCommit: ["$gitCommit"]" +echo "TRAVIS_TAG: ["$TRAVIS_TAG"]" for os in "${os_list[@]}" do