Travis build stages

This fixes our Travis auto-deployment mechnanisms, and switches
over to making use of Travis build stages along the way.

There are now three stages:

1. verify: this *just* runs the verification suite, which currently
   consists of the flake8 tests.  The verification is run under
   both Python 2 and Python 3, using the fedora:rawhide image

2. test: this runs all of non-verification tests, under the full
   test matrix we were using before

3. deploy: this runs our deployements, as before

The deploy phase now precompiles the sdist, and asks for that to be
uploaded instead of regenerating the code before upload.  This is a bit
of a hack.

Finally, the actual Travis scripts themselves were refactored into
a series of smaller files and utility functions, which makes the code
a bit more readable in some cases, and allows individual functions
to be called separately (useful in testing stuff out, and in creating
new build stages).

Author: DirectXMan12

.travis.sh

@@ -14,49 +14,63 @@ env:
   - DISTRO=fedora:rawhide PYTHON="3"
   - DISTRO=fedora:rawhide PYTHON="2"
 
+# we do everything in docker
+install: skip
+before_install: skip
+
+stages:
+- verify
+- test
+- name: deploy
+  if: tag is PRESENT
+
 script:
   - sudo sed -i '1i 127.0.0.1 test.box' /etc/hosts
   - sudo hostname test.box
-  - >
-    docker run
-    -v `pwd`:/tmp/build
-    -w /tmp/build
-    -e KRB5_VER=$KRB5_VER
-    -e PYTHON=$PYTHON
-    $DISTRO
-    /bin/bash -ex .travis.sh
-
-before_deploy:
-  - >
-    docker run
-    -v `pwd`:/tmp/build
-    -w /tmp/build
-    -e TRAVIS_TAG=$TRAVIS_TAG
-    -e PKG_NAME_VER=$PKG_NAME_VER
-    -e KRB5_VER=$KRB5_VER
-    -e PYTHON=$PYTHON
-    $DISTRO
-    /bin/bash -ex .travis.before-deploy.sh
-deploy:
-  - provider: releases
-    api_key:
-      secure: e8kjZEY28gWySD5FsnG9CMgeKgFS12gB/xm1Ck1jEgIhYT01VwxvBMHIU7SV/92+aeBao5clN4/qN4P035hD8TShHUMzV7rsmXmqcaCwi3J5sh2hyy4e1PfYAshuX7bxvkfqJSvSonLz0AKHW4pj17yMrTXpWiEioGg0fSLtTHo=
-    file_glob: true
-    file:
-      - tag_build/*
-    on:
-      repo: pythongssapi/python-gssapi
-      tags: true
-      condition: "$DISTRO = 'fedora:rawhide' && $PYTHON = '3'"
-    skip_cleanup: true
-  - provider: pypi
-    user:
-      secure: gMwGf6Zt2BuPHN6Qy4feaJaZQBIYZ1lZIdFcpugyhnWtxytzwApx2LTXqHgdaVK7Ik6Xk97RmWS5SyciKmC2V1nyaHCX/f31QZAD3TrLqhScPtvE6j9j/srJhDBjp5yAT8NENMykeSvoIhiidw1tPzWYRW9c93roaOUv6Kqd3zc=
-    password:
-      secure: D0lX+DRVac2GN3bZuKxd45DluBrcvwMfV+cciannEJ39kjbHTQEdpi0Uv92rDLNicjqyjy9rAEP7UAQQ1GDOvZcN5OG1+uWU1tSV84uKPgT92YbbQQdHRs/Rh3KSHKeaiJ+3TTDSeBuXYgWRs2gCeVAb18LBYjtIHLZxCFB8ln0=
-    on:
-      repo: pythongssapi/python-gssapi
-      tags: true
-      condition: "$DISTRO = 'fedora:rawhide' && $PYTHON = '3'"
-    skip_cleanup: true
-    docs_dir: travis_docs_build/html
+  - source ./.travis/lib-util.sh
+  - util::docker-run $DISTRO ./.travis/build.sh
+
+jobs:
+  include:
+  - stage: verify
+    env: DISTRO=fedora:rawhide PYTHON="2"
+    script:
+    - source ./.travis/lib-util.sh
+    - util::docker-run $DISTRO ./.travis/verify.sh
+
+  - stage: verify
+    env: DISTRO=fedora:rawhide PYTHON="3"
+    script:
+    - source ./.travis/lib-util.sh
+    - util::docker-run $DISTRO ./.travis/verify.sh
+
+  - stage: deploy
+    script: skip
+    env: DISTRO=fedora:rawhide PYTHON="3"
+    before_deploy:
+    - source ./.travis/lib-util.sh
+    - util::docker-run $DISTRO ./.travis/before-deploy.sh
+    - ls -alR `pwd`
+
+    deploy:
+    - provider: releases
+      api_key:
+        secure: e8kjZEY28gWySD5FsnG9CMgeKgFS12gB/xm1Ck1jEgIhYT01VwxvBMHIU7SV/92+aeBao5clN4/qN4P035hD8TShHUMzV7rsmXmqcaCwi3J5sh2hyy4e1PfYAshuX7bxvkfqJSvSonLz0AKHW4pj17yMrTXpWiEioGg0fSLtTHo=
+      file_glob: true
+      file:
+        - tag_build/*
+      skip_cleanup: true
+      on:
+        all_branches: true
+
+    - provider: pypi
+      user:
+        secure: "jUAMucBq+9xH8x9u0I0LOwrs3Zb++KN7FwIIwz2CyAt/+TyyrJzeGJaV+dTiJ1OqcUIFqQG6jopzpnAe4biL1O68PEwz9BphKetFLpLHiFNm/n67LYno6NFonWmxndIy99pOP6NZu29nzSNeYq/KgEHo/5OkqEGOxk//lh7X/OY="
+      password:
+        secure: "ZqywwnR+G5VeM2sStwfLeutOvqbULHtnStjrdYc8WcC/FBVwmH/W48fTlvxrnswmfKx7Eljv0nN4VcBpoFf1tvz4O2oK/tCRpf0N8SvpT0jBx8bLGUxJ1/3Po6rFgBRWgSb/mzKHPKI6fLlQNcNg8lrd9e1j/zgbVRSwNeMUOR8="
+      skip_cleanup: true
+      docs_dir: travis_docs_build/html
+      on:
+        all_branches: true
+      # NB(directxman12): this is a hack.  Check ./.travis/before-deploy.sh for an explanation.
+      distributions: "check"

.travis.yml

@@ -1,11 +1,14 @@
 #!/bin/bash -ex
 
+source ./.travis/lib-setup.sh
+
 # build again since I can't figure out how to get travis to recognize the old
 # build in the new container.  The other alternative (besides actually solving
 # the issue) is to run the docs build and tarball generation every time.
-./.travis.sh
 
-source ./.venv/bin/activate
+./.travis/build.sh
+
+setup::activate
 
 yum -y install tar coreutils git
 
@@ -20,10 +23,28 @@ pip install -e .
 # place in a non-standard location so that they don't get cleaned up
 python setup.py build_sphinx --build-dir travis_docs_build
 
+# NB(directxman12): this is a *terrible* hack, but basically,
+# dpl (the Travis deployer) uses `twine` instead of `setup.py sdist upload`.
+# like this:
+# - python setup.py $PYPI_DISTRIBUTIONS
+# - twine upload -r pypi dist/*
+# - [some other stuff]
+#
+# so if we set $PYPI_DISTRIBUTIONS to something harmless, like `check`,
+# and then build the dist ourselves (and save it from the cleanup),
+# dpl will upload that
+
+# build the sdist
+python setup.py sdist
+mv dist dist_saved
+
 # for the tarball upload
 # clean up
 git clean -Xdf
 
+# restore the saved "dist" directory
+mv dist_saved dist
+
 # make the dir
 rm -rf ./tag_build || true
 mkdir ./tag_build
@@ -38,5 +59,5 @@ fi
 
 PKG_NAME_VER="python-gssapi-${PYTHON_GSSAPI_VERSION}"
 
-tar -czvf ./tag_build/${PKG_NAME_VER}.tar.gz --exclude='tag_build' --exclude='.git' --exclude='travis_docs_build' --exclude='.git' --transform "s,^\.,${PKG_NAME_VER}," .
+tar -czvf ./tag_build/${PKG_NAME_VER}.tar.gz --exclude='dist' --exclude='tag_build' --exclude='.git' --exclude='travis_docs_build' --exclude='.git' --transform "s,^\.,${PKG_NAME_VER}," .
 sha512sum --binary ./tag_build/${PKG_NAME_VER}.tar.gz > ./tag_build/${PKG_NAME_VER}.sha512sum

.travis.before-deploy.sh renamed to .travis/before-deploy.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -ex
+
+# set up dependencies, etc
+source ./.travis/lib-setup.sh
+setup::install
+
+# always build in-place so that Sphinx can find the modules
+python setup.py build_ext --inplace
+BUILD_RES=$?
+
+if [ x"$KRB5_VER" = "xheimdal" ]; then
+    # heimdal can't run the tests yet, so just exit
+    exit $BUILD_RES
+fi
+
+if [ $BUILD_RES -ne 0 ]; then
+    # if the build failed, don't run the tests
+    exit $BUILD_RES
+fi
+
+python setup.py nosetests --verbosity=3
+TEST_RES=$?
+
+exit $TEST_RES

.travis/build.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+setup::python-suffix() {
+    if [ x"$PYTHON" = "x3" ]; then
+        echo "3"
+    else
+        echo ""
+    fi
+}
+
+# We test Debian's cython.  el7's cython is too old, and Rawhide's virtualenv
+# doesn't work right (usrmerge bugs) so we can only test Debian's cython.
+
+setup::debian::install() {
+    local IS3=$(setup::python-suffix)
+
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update
+
+    if [ x"$KRB5_VER" = "xheimdal" ]; then
+        apt-get -y install heimdal-dev
+    else
+        apt-get -y install krb5-{user,kdc,admin-server,multidev} libkrb5-dev
+    fi
+
+    apt-get -y install gcc virtualenv python$IS3-{virtualenv,dev} cython$IS3
+
+    virtualenv --system-site-packages -p $(which python${PYTHON}) .venv
+    source ./.venv/bin/activate
+}
+
+setup::rh::yuminst() {
+    # yum has no update-only verb
+    yum -y --nogpgcheck install $@
+}
+
+setup::centos::install() {
+    local IS3=$(setup::python-suffix)
+
+    # Cython on el7 is too old - downstream patches
+    setup::rh::yuminst python$IS3-{virtualenv,devel}
+    virtualenv -p $(which python$IS3) .venv
+    source ./.venv/bin/activate
+    pip install --upgrade pip # el7 pip doesn't quite work right
+    pip install --install-option='--no-cython-compile' cython
+}
+
+setup::fedora::install() {
+    # path to binary here in case Rawhide changes it
+    setup::rh::yuminst redhat-rpm-config \
+        /usr/bin/virtualenv python${PYTHON}-{virtualenv,devel}
+    virtualenv -p $(which python${PYTHON}) .venv
+    source ./.venv/bin/activate
+    pip install --install-option='--no-cython-compile' cython
+}
+
+setup::rh::install() {
+    setup::rh::yuminst krb5-{devel,libs,server,workstation} which gcc findutils
+
+    if [ -f /etc/fedora-release ]; then
+        setup::fedora::install
+    else
+        setup::centos::install
+    fi
+}
+
+setup::install() {
+    if [ -f /etc/debian_version ]; then
+        setup::debian::install
+    elif [ -f /etc/redhat-release ]; then
+        setup::rh::install
+    else
+        echo "Distro not found!"
+        false
+    fi
+
+    pip install -r test-requirements.txt
+}
+
+setup::activate() {
+    # remove (and restore) set -x to avoid log-spam the source
+    # script, which we don't care about
+    wastrace=${-//[^x]/}
+    set +x
+    source .venv/bin/activate
+    if [[ -n "$wastrace" ]]; then set -x; fi
+}

.travis/lib-setup.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+util::docker-run() {
+    local distro=$1
+    shift
+
+    docker run \
+    -v `pwd`:/tmp/build \
+    -w /tmp/build \
+    -e TRAVIS_TAG=$TRAVIS_TAG \
+    -e PKG_NAME_VER=$PKG_NAME_VER \
+    -e KRB5_VER=$KRB5_VER \
+    -e PYTHON=$PYTHON \
+    $distro \
+    /bin/bash -ex $@
+}