diff --git a/.github/workflows/alert-discord.yml b/.github/workflows/old/alert-discord.yml similarity index 100% rename from .github/workflows/alert-discord.yml rename to .github/workflows/old/alert-discord.yml diff --git a/.github/workflows/build-next.yml b/.github/workflows/old/build-next.yml similarity index 100% rename from .github/workflows/build-next.yml rename to .github/workflows/old/build-next.yml diff --git a/.github/workflows/gh-pr-assignees.yml b/.github/workflows/old/gh-pr-assignees.yml similarity index 100% rename from .github/workflows/gh-pr-assignees.yml rename to .github/workflows/old/gh-pr-assignees.yml diff --git a/.github/workflows/gh-projects-qa.yml b/.github/workflows/old/gh-projects-qa.yml similarity index 100% rename from .github/workflows/gh-projects-qa.yml rename to .github/workflows/old/gh-projects-qa.yml diff --git a/.github/workflows/gh-submodules.yml b/.github/workflows/old/gh-submodules.yml similarity index 100% rename from .github/workflows/gh-submodules.yml rename to .github/workflows/old/gh-submodules.yml diff --git a/.github/workflows/gh-sync.yml b/.github/workflows/old/gh-sync.yml similarity index 100% rename from .github/workflows/gh-sync.yml rename to .github/workflows/old/gh-sync.yml diff --git a/.github/workflows/gh-test-downstream.yml b/.github/workflows/old/gh-test-downstream.yml similarity index 100% rename from .github/workflows/gh-test-downstream.yml rename to .github/workflows/old/gh-test-downstream.yml diff --git a/.github/workflows/meta-regression-analysis.yml b/.github/workflows/old/meta-regression-analysis.yml similarity index 100% rename from .github/workflows/meta-regression-analysis.yml rename to .github/workflows/old/meta-regression-analysis.yml diff --git a/.github/workflows/publish-ghcr.yml b/.github/workflows/old/publish-ghcr.yml similarity index 100% rename from .github/workflows/publish-ghcr.yml rename to .github/workflows/old/publish-ghcr.yml diff --git a/.github/workflows/publish-py-pypi.yml b/.github/workflows/old/publish-py-pypi.yml similarity index 100% rename from .github/workflows/publish-py-pypi.yml rename to .github/workflows/old/publish-py-pypi.yml diff --git a/.github/workflows/publish-storybook.yml b/.github/workflows/old/publish-storybook.yml similarity index 100% rename from .github/workflows/publish-storybook.yml rename to .github/workflows/old/publish-storybook.yml diff --git a/.github/workflows/publish-ts-npm.yml b/.github/workflows/old/publish-ts-npm.yml similarity index 100% rename from .github/workflows/publish-ts-npm.yml rename to .github/workflows/old/publish-ts-npm.yml diff --git a/.github/workflows/test-py-lint.yml b/.github/workflows/old/test-py-lint.yml similarity index 100% rename from .github/workflows/test-py-lint.yml rename to .github/workflows/old/test-py-lint.yml diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/old/test-py-pytest.yml similarity index 100% rename from .github/workflows/test-py-pytest.yml rename to .github/workflows/old/test-py-pytest.yml diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/old/test-pytest.yml similarity index 100% rename from .github/workflows/test-pytest.yml rename to .github/workflows/old/test-pytest.yml diff --git a/.github/workflows/test-storybook.yml b/.github/workflows/old/test-storybook.yml similarity index 100% rename from .github/workflows/test-storybook.yml rename to .github/workflows/old/test-storybook.yml diff --git a/.github/workflows/test-ts-lint.yml b/.github/workflows/old/test-ts-lint.yml similarity index 100% rename from .github/workflows/test-ts-lint.yml rename to .github/workflows/old/test-ts-lint.yml diff --git a/.github/workflows/regression-test.yml b/.github/workflows/regression-test.yml new file mode 100644 index 0000000..fde8896 --- /dev/null +++ b/.github/workflows/regression-test.yml @@ -0,0 +1,54 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Regression Test + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest pytest-reportlog black + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + - name: Test PR branch with pytest + run: | + pytest --report-log=pr_test_results.json + find . -name "*.pyc" | xargs rm + continue-on-error: true + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + path: main + - name: Test main + run: | + pytest ./main --report-log=main_test_results.json + continue-on-error: true + - name: Print test results + run: | + cat pr_test_results.json + cat main_test_results.json + - name: Compare results + run: | + python main.py pr_test_results.json main_test_results.json \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..bf707f8 --- /dev/null +++ b/main.py @@ -0,0 +1,47 @@ +import json +import pathlib +from collections import defaultdict +import sys + + +def main(): + pull_request_dict = defaultdict(list) + destination_branch_dict = defaultdict(list) + pull_request_fp = pathlib.Path(sys.argv[1]) + destination_branch_fp = pathlib.Path(sys.argv[2]) + with open(pull_request_fp, "r") as f: + pull_request_results = f.read().splitlines()[3:-1] # Trim unnecessary entries + for item in pull_request_results: + print(f"ITEM: {item}") + json_data = json.loads(item) + if "nodeid" in json_data.keys(): + pull_request_dict[json_data["outcome"]].append(json_data["nodeid"]) + with open(destination_branch_fp, "r") as f: + destination_branch_results = f.read().splitlines()[ + 3:-1 + ] # Trim unnecessary entries + for item in destination_branch_results: + json_data = json.loads(item) + if "nodeid" in json_data.keys(): + destination_branch_dict[json_data["outcome"]].append( + json_data["nodeid"] + ) + regressions = [] + warnings = [] + for item in destination_branch_dict["passed"]: + if item in pull_request_dict["failed"]: + regressions.append(item) + elif item in pull_request_dict["skipped"]: + warnings.append(item) + print(f"{len(warnings)} tests now skipped/xfailed that weren't previously") + for warning in warnings: + print(f"Warning: {warning}") + print(f"{len(regressions)} regressions were found") + for regression in regressions: + print(f"Regression: {regression}") + if len(regressions) > 0: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..99844fa --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +testpaths=*_test.py diff --git a/sanity_test.py b/sanity_test.py new file mode 100644 index 0000000..b01dc18 --- /dev/null +++ b/sanity_test.py @@ -0,0 +1,5 @@ +def test_nothing(): + assert False + +def test_new_passing_test(): + assert True \ No newline at end of file