diff --git a/.github/workflows/gold.yml b/.github/workflows/gold.yml index de96180..eb46156 100644 --- a/.github/workflows/gold.yml +++ b/.github/workflows/gold.yml @@ -3,11 +3,7 @@ name: Gold-Compare and Functional Tests # Runs on every push or pull request. on: push: - branches: - - master pull_request: - branches: - - master # StackOverflow 58033366 env: @@ -20,12 +16,6 @@ env: # Jobs arranged in order of which is expected to finish first etc. jobs: -# id: -# runs-on: ubuntu-latest -# steps: -# - name: Action, branch, and repo information. -# run: printf "$SHOW_ACTION\n$SHOW_BRANCH\n$SHOW_BNAME\n$SHOW_SHA\n" - quick_local_test_5s: runs-on: ubuntu-latest steps: @@ -59,3 +49,12 @@ jobs: uses: actions/checkout@v2 - name: Run the test run: test/garnet.sh $BRANCH_NAME 4x2 apps/pointwise + + # Technically not a gold test maybe I dunno + pip-install_30s: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Run the test + run: pip/genesis2/test/docker-test.sh diff --git a/pip/genesis2/README-pip.md b/pip/genesis2/README-pip.md index a27afea..3c9c73c 100644 --- a/pip/genesis2/README-pip.md +++ b/pip/genesis2/README-pip.md @@ -1,30 +1,48 @@ # HOW TO update the pypi package for pip install -1. Update `setup.py` with new version number e.g. "0.0.7" => "0.0.8" +### BUILD DISTRIBUTION FILE e.g. "dist/genesis2-0.0.10.tar.gz" +``` + # First, edit 'setup.py' to update version number e.g. + old_version=0.0.9; new_version=0.0.10 + sed "s/$old_version/$new_version/" setup.py > tmp + diff tmp setup.py + mv tmp setup.py + + # Create tar file e.g. "dist/genesis2-0.0.10.tar.gz" + # -- We use "setup.py sdist" (*tar.gz) instead of "setup bdist_wheel" (*.whl) (why?) + # This creates dirs 'dist' (which we want) and 'genesis2.egg-info' (which we don't need) + python3 setup.py sdist |& tee sdist.log | less +``` + +### TEST the new distribution file +``` + deactivate || echo okay # (optional) deactivate existing venv + python3 -m venv /tmp/venv # Build new venv + source /tmp/venv/bin/activate + pip install dist/genesis2-0.0.11.tar.gz # Install genesis2 + deactivate # Clean up + + # AND/OR + test/docker-test.sh --help + test/docker-test.sh dist/genesis2-0.0.11.tar.gz +``` -2. Use the `make-pip` script to install the new package. `make-pip` is -completely non-destructive, all it does is print out the instructions -for each step. +### UPLOAD to pypi +``` + twine upload dist/genesis2-0.0.11.tar.gz |& tee twine-0.0.11.log ``` - % make-pip.sh --help + +### TEST uploaded pypi package ``` -Optionally, can do it all manually: + test/docker-test.sh --help + test/docker-test.sh --pypi ``` - # BUILD AND INSTALL - version=0.0.9 - ls dist - mv dist/* archives - python3 setup.py sdist |& tee sdist-$version.log - ls dist - twine upload dist/* |& tee twine-$version.log - # CLEANUP +### OPTIONAL CLEANUP (only need 'dist' dir for upload maybe) +``` + # Unwanted files go to old/ subdirectory d=old/pipfiles-`date +%y%m%d.%H%M`; echo mkdir -p $d; mkdir -p $d for p in Genesis2/ build/ genesis2.egg-info/ dist/; do test -e $p && echo mv $p $d; done # (cut'n'paste commands resulting from above) -``` -Try it out: - source /nobackup/steveri/garnet_venv/bin/activate - pip uninstall genesis2 - pip install genesis2 |& tee tmp.log +``` diff --git a/pip/genesis2/dist/genesis2-0.0.11.tar.gz b/pip/genesis2/dist/genesis2-0.0.11.tar.gz new file mode 100644 index 0000000..5daf700 Binary files /dev/null and b/pip/genesis2/dist/genesis2-0.0.11.tar.gz differ diff --git a/pip/genesis2/make-pip.sh b/pip/genesis2/make-pip.sh deleted file mode 100755 index e240f57..0000000 --- a/pip/genesis2/make-pip.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# After building pip, will have these new dirs/files: -# % ls dist/* -# dist/genesis2-0.0.6-cp38-cp38-linux_x86_64.whl -# -# % ls genesis2.egg-info/ -# dependency_links.txt PKG-INFO SOURCES.txt top_level.txt - -# This *could* be a Makefile, you know...just sayin'... - -HELP=" - DESCRIPTION - Non-destructively emits commands necessary to complete the given action - - OPTIONS - $0 --help - $0 --build # How-to: Build a pip distribution including 'dist/*' and 'genesis2.egg-info/*' - $0 --upload # How-to: Use twine to upload the distribution - $0 --clean # How-to: Cleanup from make-pip -" - -if [ "$1" == "" ]; then echo "$HELP"; exit; fi -if [ "$1" == "--help" ]; then echo "$HELP"; exit; fi - -if [ "$1" == "--build" ]; then - cat <<' EOF' - # Build a pip distribution; creates dirs 'dist' and 'genesis2.egg-info' - # First, should edit 'setup.py' to update version number. Then: - - # Produces a *.whl, but maybe I want *.tar.gz - # python3 setup.py bdist_wheel |& tee bdist_wheel.log | less - - # Creates e.g. "dist/genesis2-0.0.7.tar.gz" - python3 setup.py sdist |& tee sdist.log | less - - - # Optional cleanup (only need 'dist' dir for upload maybe) - d=deleteme/pipfiles-`date +%y%m%d.%H%M`; echo mkdir -p $d - mkdir -p $d; mv Genesis2/ build/ genesis2.egg-info/ $d; ls -l $d - EOF - exit -fi - -if [ "$1" == "--upload" ]; then - echo 'twine upload dist/* |& tee twine.log' - exit -fi - -if [ "$1" == "--clean" ]; then - echo 'd=old/pipfiles-`date +%y%m%d.%H%M`; echo mkdir -p $d' - echo 'for p in Genesis2/ build/ genesis2.egg-info/ dist/; do test -e $p && echo mv $p $d; done' - exit -fi - -echo "$HELP" -exit - -######################################################################## -# NOTES - -######################################################################## -# Not sure if setup.cfg is used for anything...? -# It has info for bdist_wheel but we are maybe using sdist..? - - -######################################################################## -# The twine command requires a pypi API; e.g. something like this (below) -# in a file ~/.pypirc -# -# [distutils] -# index-servers = -# pypi -# genesis2 -# -# [pypi] -# username = __token__ -# password = # either a user-scoped token or a project-scoped token you want to set as the default -# [genesis2] -# repository = https://upload.pypi.org/legacy/ -# username = __token__ -# password = # a project token - diff --git a/pip/genesis2/pyproject.toml b/pip/genesis2/pyproject.toml new file mode 100644 index 0000000..3acd765 --- /dev/null +++ b/pip/genesis2/pyproject.toml @@ -0,0 +1,5 @@ +[build-system] +requires = [ + "setuptools>=40.8.0", + "wheel", +] diff --git a/pip/genesis2/setup.py b/pip/genesis2/setup.py index afe7ebd..8a634d7 100644 --- a/pip/genesis2/setup.py +++ b/pip/genesis2/setup.py @@ -52,7 +52,7 @@ def run(self): setup( name='genesis2', - version='0.0.9', + version='0.0.11', packages=[ "genesis2" ], diff --git a/pip/genesis2/test/Dockerfile.genesis b/pip/genesis2/test/Dockerfile.genesis new file mode 100644 index 0000000..99ba76f --- /dev/null +++ b/pip/genesis2/test/Dockerfile.genesis @@ -0,0 +1,31 @@ +FROM docker.io/ubuntu:20.04 + +# Note "pip install genesis2" uses pypi to fetch wheel (default) +# "pip install as wheel +ARG PIPFILE=genesis2 + +RUN \ + printf "\n\nUPDATE\n" && \ + apt-get -q update && \ + printf "\n\nINSTALL\n" && \ + apt-get install --no-install-recommends -y git python3 python3-pip python3-venv && \ + printf "\nALTS\n" && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 100 && \ + update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 100 && \ + mkdir -p /piptest && \ + printf "DONE setup\n\n\n" + +# Switch shell to bash +SHELL ["/bin/bash", "--login", "-c"] + +# Copy any wheels you can find, in case e.g. user specifies "PIPFILE=genesis2-0.0.9.tar.gz" +COPY ./genesis2*.gz /piptest/ + +# Awk filter detects if pip install output contains the word "ERROR" +WORKDIR /piptest +RUN \ + printf "\n\nPIP INSTALL $PIPFILE\n"; \ + python -m venv . && source bin/activate; \ + pip list -v; \ + python3 -m pip install $PIPFILE |& awk '{print}/ERROR/{err=1}END{if(err)exit 13}' || exit 13; \ + printf "DONE pip install\n\n\n" diff --git a/pip/genesis2/test/docker-test.sh b/pip/genesis2/test/docker-test.sh new file mode 100755 index 0000000..9591e68 --- /dev/null +++ b/pip/genesis2/test/docker-test.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +HELP=' +DESCRIPTION + Launches a docker image build that will try to do + "pip install genesis2" in a clean python virtual environment. + Test succeeds if no errors occur during this process. + + Can test pypi distribution file or can optionally test a file locally. + In the latter case, local file must match template "*/genesis2*.gz". + +USAGE + docker-test.sh --pypi # Launch docker-build that does "pip install genesis2" + docker-test.sh # Launch docker-build that does "pip install " + +EXAMPLE + docker-test.sh dist/genesis2-0.0.9.tar.gz # Should FAIL + docker-test.sh dist/genesis2-0.0.11.tar.gz # Should PASS + docker-test.sh --pypi +' + +# Function to verify and return wheel filename +function wheel { + if ! test -f "$1"; then + echo "ERROR: Cannot find requested wheel '$1'" > /dev/stderr + echo ERROR + else + echo "$1" + fi +} + +# Unpack args +pipfile="genesis2" # default value +case "$1" in + -h|--help) echo "$HELP"; exit ;; + --pypi) pipfile="genesis2" ;; + .*) pipfile=$(wheel "$1" | tail -1) ;; +esac +[ "$pipfile" == "ERROR" ] && exit 13 + +# Make a temporary workspace maybe +# Must be lower case to use later as docker image tag (which disallows uppercase!?) +workspace=$(mktemp -u /tmp/docker-test-XXXXX | tr [:upper:] [:lower:]) +echo "Building temporary workspace '$workspace'" +mkdir "$workspace" + +# Copy relevant files to the workspace and cd +cp $(dirname $0)/Dockerfile.genesis "$workspace" +test -f "$pipfile" && cp "$pipfile" "$workspace" + +# Dockerfile will try to do "COPY ./genesis2*.gz", this ensures that don't fail +touch "$workspace"/genesis2-dummy.tar.gz # Hack so that dockerfile COPY does not fail +ls -l "$workspace" + +# Prep for docker image cleanup +function cleanup { + printf "TEST $1\n\n" # "TEST PASSED" or "TEST FAILED" + echo BEFORE: "$(docker images)" + docker system prune -f || echo okay + test -e iidfile && docker rmi $(cat iidfile) || echo okay + cd /tmp && /bin/rm -rf "$workspace" || echo okay + echo AFTER: "$(docker images)" + printf "\nTEST $1\n\n" # "TEST PASSED" or "TEST FAILED" +} + +# Try the build +set -x +cd $workspace +if docker build \ + --no-cache \ + --iidfile iidfile \ + --file Dockerfile.genesis \ + --build-arg PIPFILE="$(basename $pipfile)" \ + -t $(basename $workspace) \ + . +then + set +x; cleanup PASSED +else + set +x; cleanup FAILED; exit 13 +fi