readme: add testing section (#323)

Author: sdimitro

.github/scripts/clear-dump.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -eu
+
+DATA_DIR="tests/integration/data"
+
+echo "checking folder structure ..."
+if [ ! -d $DATA_DIR ]; then
+	exit 1
+fi
+
+echo "removing current crash dump if any ..."
+rm -f $DATA_DIR/dump.*
+
+echo "removing any extracted vmlinux ..."
+rm -f $DATA_DIR/vmlinux*
+
+echo "removing any extracted modules ..."
+rm -rf $DATA_DIR/mods
+rm -rf $DATA_DIR/usr
+
+echo "removing any savedump scripts ..."
+rm -rf $DATA_DIR/run-*.sh
+
+echo "Done"

.github/scripts/download-dump-from-s3.sh

@@ -22,14 +22,24 @@
 
 DATA_DIR="tests/integration/data"
 
+if [ $# -eq 0 ]; then
+	echo "error: no crash dump archive argument supplied"
+	exit 1
+fi
+
 echo "checking folder structure ..."
 if [ ! -d $DATA_DIR ]; then
+	echo "error: please cd to the root of the git repo"
 	exit 1
 fi
 
-echo "initiating download of $1 from S3 ..."
-wget https://sdb-testing-bucket.s3.us-west-2.amazonaws.com/$1
-[ $? -eq 0 ] || exit 1
+if [ -f "$1" ]; then
+	echo "Found $1 locally, skip download ..."
+else
+	echo "downloading of $1 from S3 ..."
+	wget https://sdb-testing-bucket.s3.us-west-2.amazonaws.com/$1
+	[ $? -eq 0 ] || exit 1
+fi
 
 if [[ $1 == *.lzma ]]; then
 	# Profile A

.github/workflows/main.yml

@@ -56,7 +56,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.8, 3.9]
-        dump: [dump-201912060006.tar.lzma, dump.202303131823.tar.gz]
+        dump: [dump.201912060006.tar.lzma, dump.202303131823.tar.gz]
     env:
       AWS_DEFAULT_REGION: 'us-west-2'
     steps:

.gitignore

@@ -110,3 +110,7 @@ tests/integration/data/mods/
 
 # vim
 *.swp
+
+# zipped folders - usually crash dumps
+*.lzma
+*.tar.gz

README.md

@@ -50,6 +50,79 @@ sdb> addr modules | lxlist "struct module" list | member name ! sort | head -n 3
 (char [56])"async_pq"
 ```
 
-### Resources
+### Developer Testing
 
-User and developer resources for sdb can be found in the [project's wiki](https://github.com/delphix/sdb/wiki).
+#### Linting
+
+```
+$ python3 -m pip install pylint pytest
+$ python3 -m pylint -d duplicate-code -d invalid-name sdb
+$ python3 -m pylint -d duplicate-code -d invalid-name tests
+```
+
+#### Type Checking
+
+```
+$ python3 -m pip install mypy==0.730
+$ python3 -m mypy --strict --show-error-codes -p sdb
+$ python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests
+```
+
+#### Style Checks
+
+```
+$ python3 -m pip install yapf
+$ python3 -m yapf --diff --style google --recursive sdb
+$ python3 -m yapf --diff --style google --recursive tests
+```
+
+If `yapf` has suggestions you can apply them automatically by substituting
+`--diff` with `-i` like this:
+```
+$ python3 -m yapf -i --style google --recursive sdb
+$ python3 -m yapf -i --style google --recursive tests
+```
+
+#### Regression Testing
+
+Regression testing is currently done by downloading a refererence crash/core
+dump and running a predetermined set of commads on it, ensuring that they
+return the same reference output before and after your changes. To see the list
+of reference crash dumps refer to the testing matrix of the `pytest` Github
+action in `.github/workflows/main.yml`. Here is an example of running the
+regression test commands against `dump.201912060006.tar.lzma` with code
+coverage and verbose output:
+
+```
+$ python3 -m pip install python-config pytest pytest-cov
+$ .github/scripts/download-dump-from-s3.sh dump.201912060006.tar.lzma
+$ python3 -m pytest -v --cov sdb --cov-report xml tests
+```
+
+If you want `pytest` to stop on the first failure it encounters add
+`-x/--exitfirst` in the command above.
+
+If you've added new test commands or found mistakes in the current reference
+output and you want (re)generate reference output for a crash dump - let's say
+`dump.201912060006` from above:
+```
+$ PYTHONPATH=$(pwd) python3 tests/integration/gen_regression_output.py dump.201912060006
+```
+
+or more generically:
+```
+$ PYTHONPATH=$(pwd) python3 tests/integration/gen_regression_output.py <dump name>
+```
+
+For the time being, the test suite is not smart enought to handle the testing
+of multiple crash dumps at the same time. Until that happens, developers that
+want to test multiple crash dumps need to delete their current crash dump
+before downloading the next to run `pytest`. Here is a sequence of commands to
+run `pytest` against two crash dumps:
+```
+$ .github/scripts/download-dump-from-s3.sh dump.201912060006.tar.lzma
+$ python3 -m pytest -v --cov sdb --cov-report xml tests
+$ .github/scripts/clear-dump.sh
+$ .github/scripts/download-dump-from-s3.sh dump.202303131823.tar.gz
+$ python3 -m pytest -v --cov sdb --cov-report xml tests
+```

setup.py

@@ -27,5 +27,5 @@
     author_email='serapheim@delphix.com',
     description='The Slick/Simple Debugger',
     license='Apache-2.0',
-    url='https://github.com/sdimitro/sdb',
+    url='https://github.com/delphix/sdb',
 )