From b642ef182e26be2b867af6b03d1cb46bb508c07d Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 4 Jun 2023 16:07:45 -0700 Subject: [PATCH 1/5] CI test --- .github/workflows/ci.yml | 162 +++++++++++++++++++++------------- tests/Auto-GPT-test-cassettes | 2 +- 2 files changed, 104 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9656fc161a2..420a333b4d15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,8 +89,12 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name }} submodules: true - - id: checkout_cassettes - name: Check out cassettes + - name: Setup git username and email + run: | + git config --global user.name "Auto-GPT-Bot" + git config --global user.email "github-bot@agpt.co" + + - name: Check out cassettes if: ${{ startsWith(github.event_name, 'pull_request') }} run: | cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" @@ -101,22 +105,12 @@ jobs: git fetch origin ${{ github.event.pull_request.base.ref }} git checkout $cassette_branch - - if git merge --no-commit --no-ff ${{ github.event.pull_request.base.ref }}; then - echo "Using cassettes from mirror branch, synced to upstream branch '${{ github.event.pull_request.base.ref }}'" - else - echo "Could not merge upstream changes to cassettes. Using cassettes from ${{ github.event.pull_request.base.ref }}." - git merge --abort - git checkout ${{ github.event.pull_request.base.ref }} - - # Delete branch to prevent conflict when re-creating it - git branch -D $cassette_branch - fi - echo "cassette_branch=$(git branch --show-current)" >> $GITHUB_OUTPUT + git merge --no-commit --strategy-option ours origin/${{ github.event.pull_request.base.ref }} + echo "Using cassettes from mirror branch, synced to upstream branch '${{ github.event.pull_request.base.ref }}' if no conflicts." else + git checkout -b $cassette_branch echo "Branch '$cassette_branch' does not exist in cassette submodule."\ "Using cassettes from ${{ github.event.pull_request.base.ref }}." - echo "cassette_branch=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT fi - name: Set up Python ${{ matrix.python-version }} @@ -124,15 +118,32 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Run pytest tests with coverage run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml + echo "test" >> tests/integration/challenges/current_score.json + env: + CI: true + PROXY: ${{ secrets.PROXY }} + AGENT_MODE: ${{ vars.AGENT_MODE }} + AGENT_TYPE: ${{ vars.AGENT_TYPE }} - - name: Run pytest tests with coverage + - id: get_cassettes_diff + name: Identify differences between cassettes of current PR and target branch + if: ${{ startsWith(github.event_name, 'pull_request') }} + run: | + cd tests/Auto-GPT-test-cassettes + cassette_diff=$(git diff --name-only --diff-filter=M -- '**/*.yaml') + + if [ -n "$cassette_diff" ]; then + echo "updated=true" >> $GITHUB_OUTPUT + fi + + - name: Attempt to beat Challenges if cassettes changed + if: ${{ startsWith(github.event_name, 'pull_request') }} && "${{ steps.push_cassettes.outputs.updated }}" == "true" run: | - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term - python tests/integration/challenges/utils/build_current_score.py + echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml + echo "test" >> tests/integration/challenges/current_score.json env: CI: true PROXY: ${{ secrets.PROXY }} @@ -145,9 +156,6 @@ jobs: - id: setup_git_auth name: Set up git token authentication run: | - git config --global user.name "Auto-GPT-Bot" - git config --global user.email "github-bot@agpt.co" - config_key="http.${{ github.server_url }}/.extraheader" base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0) @@ -173,41 +181,29 @@ jobs: echo "The challenge scores didn't change." fi - - id: push_cassettes - name: Push updated cassettes + - name: Push updated cassettes run: | if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then is_pull_request=true cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" - cassette_source_branch="${{ steps.checkout_cassettes.outputs.cassette_branch }}" - base_branch="${{ github.event.pull_request.base.ref }}" else current_branch=$(echo ${{ github.ref }} | sed -e "s/refs\/heads\///g") cassette_branch=$current_branch fi cd tests/Auto-GPT-test-cassettes - git fetch origin $cassette_source_branch:$cassette_source_branch # Commit & push changes to cassettes if any - if ! git diff --quiet $cassette_source_branch --; then - if [ "$cassette_branch" != "$cassette_source_branch" ]; then - git checkout -b $cassette_branch - fi + if ! git diff --quiet; then git add . git commit -m "Auto-update cassettes" if [ $is_pull_request ]; then - git push --force origin HEAD:$cassette_branch + git push -f origin $cassette_branch else + git pull --rebase origin $cassette_branch git push origin HEAD:$cassette_branch - fi - - cd ../.. - if [ $is_pull_request ]; then - git fetch origin $base_branch - cassette_diff=$(git diff origin/$base_branch) - else + cd ../.. git add tests/Auto-GPT-test-cassettes git commit -m "Update cassette submodule" git push origin HEAD:$current_branch @@ -216,10 +212,51 @@ jobs: echo "No cassette changes to commit" fi - if [ -n "$cassette_diff" ]; then - echo "updated=true" >> $GITHUB_OUTPUT + - id: create_pr + name: Create Pull Request if score changed + if: ${{ startsWith(github.event_name, 'pull_request') }} + run: | + base_repo_owner=${{ github.event.pull_request.base.repo.owner.login }} + base_repo_name=${{ github.event.pull_request.base.repo.name }} + updated_score_branch="update-score-${{ github.event.pull_request.number }}-$(date +'%Y%m%d%H%M%S')" + + if ! git diff --quiet -- tests/integration/challenges/current_score.json; then + git add tests/integration/challenges/current_score.json + git commit -m "Update score" + git checkout -b $updated_score_branch + + git remote add base_repo https://github.com/$base_repo_owner/$base_repo_name.git + echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token + branches_to_delete=$(gh api repos/$base_repo_owner/$base_repo_name/git/refs --jq ".[] | select(.ref | startswith(\"refs/heads/update-score-${{ github.event.pull_request.number }}\")) | .ref") + + if [ ! -z "$branches_to_delete" ]; then + for branch in $branches_to_delete + do + branch_short=${branch#refs/heads/} + prs=$(gh api repos/${{ github.repository }}/pulls -q ".[] | select(.head.ref == \"$branch_short\") | .number") + + # iterate through all previous PRs created, close them and delete the branch + for pr in $prs + do + gh api repos/${{ github.repository }}/pulls/$pr -X PATCH -F state="closed" + done + gh api -X DELETE /repos/$base_repo_owner/$base_repo_name/git/$branch + done + else + echo "No matching branches found to delete." + fi + + git push base_repo $updated_score_branch + + pr_url=$(gh pr create --title "Update Score In Pull Request Number ${{ github.event.pull_request.number }}" \ + --head "$base_repo_owner:$updated_score_branch" \ + --base "${{ github.event.pull_request.head.ref }}" \ + --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \ + --repo ${{ github.event.pull_request.head.repo.full_name }}) + + echo "pr_url=$pr_url" >> $GITHUB_OUTPUT else - echo "updated=false" >> $GITHUB_OUTPUT + echo "The current score didn't change." fi - name: Post Set up git token auth @@ -228,27 +265,34 @@ jobs: git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' - - name: Apply or remove behaviour change label and comment on PR + - name: Apply or remove behaviour change label and comment if: ${{ startsWith(github.event_name, 'pull_request') }} run: | - PR_NUMBER=${{ github.event.pull_request.number }} - TOKEN=${{ secrets.PAT_REVIEW }} REPO=${{ github.repository }} - + echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token + COMMENT_BODY="" + SCORE_CHANGED_MESSAGE="The pipeline will fail until you merge the updated score: ${{ steps.get_cassettes_diff.outputs.pr_url }}" + if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then echo "Adding label and comment..." curl -X POST \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer ${{ secrets.PAT_REVIEW }}" \ -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels \ + https://api.github.com/repos/$REPO/issues/${{ github.event.pull_request.number }}/labels \ -d '{"labels":["behaviour change"]}' - - echo $TOKEN | gh auth login --with-token - gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged." - else - echo "Removing label..." - curl -X DELETE \ - -H "Authorization: Bearer $TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/behaviour%20change + + COMMENT_BODY="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged. " + fi + + if [[ -n "${{ steps.get_cassettes_diff.outputs.pr_url }}" ]]; + COMMENT_BODY="$COMMENT_BODY$SCORE_CHANGED_MESSAGE" + fi + + if [ ! -z "$COMMENT_BODY" ]; then + gh api repos/$REPO/issues/${{ github.event.pull_request.number }}/comments -X POST -F body="$COMMENT_BODY" + fi + + if ${{ steps.get_cassettes_diff.outputs.pr_url }}; then + echo "$SCORE_CHANGED_MESSAGE" + exit 1 fi diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes index be280df43d6a..28497a8b4ef0 160000 --- a/tests/Auto-GPT-test-cassettes +++ b/tests/Auto-GPT-test-cassettes @@ -1 +1 @@ -Subproject commit be280df43d6a23b8074d9cba10d18ed8724a54c9 +Subproject commit 28497a8b4ef0e62375dd5024794426cb401a3779 From 0ee8f09857d3897f38936fd9decb73bb4c237844 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 4 Jun 2023 16:09:42 -0700 Subject: [PATCH 2/5] CI test --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 420a333b4d15..1e6177c5a46d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,7 +284,7 @@ jobs: COMMENT_BODY="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged. " fi - if [[ -n "${{ steps.get_cassettes_diff.outputs.pr_url }}" ]]; + if [[ -n "${{ steps.get_cassettes_diff.outputs.pr_url }}" ]]; then COMMENT_BODY="$COMMENT_BODY$SCORE_CHANGED_MESSAGE" fi @@ -292,7 +292,7 @@ jobs: gh api repos/$REPO/issues/${{ github.event.pull_request.number }}/comments -X POST -F body="$COMMENT_BODY" fi - if ${{ steps.get_cassettes_diff.outputs.pr_url }}; then + if [[ -n "${{ steps.get_cassettes_diff.outputs.pr_url }}" ]]; then echo "$SCORE_CHANGED_MESSAGE" exit 1 fi From 64967398635561a490bf14521b679cdbe6d23d60 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 4 Jun 2023 16:16:56 -0700 Subject: [PATCH 3/5] CI test --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e6177c5a46d..9e0ba3b62c30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -271,7 +271,7 @@ jobs: REPO=${{ github.repository }} echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token COMMENT_BODY="" - SCORE_CHANGED_MESSAGE="The pipeline will fail until you merge the updated score: ${{ steps.get_cassettes_diff.outputs.pr_url }}" + SCORE_CHANGED_MESSAGE="The pipeline will fail until you merge the updated score: ${{ steps.create_pr.outputs.pr_url }}" if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then echo "Adding label and comment..." @@ -284,7 +284,7 @@ jobs: COMMENT_BODY="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged. " fi - if [[ -n "${{ steps.get_cassettes_diff.outputs.pr_url }}" ]]; then + if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then COMMENT_BODY="$COMMENT_BODY$SCORE_CHANGED_MESSAGE" fi @@ -292,7 +292,7 @@ jobs: gh api repos/$REPO/issues/${{ github.event.pull_request.number }}/comments -X POST -F body="$COMMENT_BODY" fi - if [[ -n "${{ steps.get_cassettes_diff.outputs.pr_url }}" ]]; then + if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then echo "$SCORE_CHANGED_MESSAGE" exit 1 fi From c7ee3ba710deceaf4156b0bf53ee7e704661a27b Mon Sep 17 00:00:00 2001 From: Auto-GPT-Bot Date: Sun, 4 Jun 2023 23:17:20 +0000 Subject: [PATCH 4/5] Update challenge scores --- tests/integration/challenges/current_score.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/challenges/current_score.json b/tests/integration/challenges/current_score.json index deb4a82d33e1..028e7a44e049 100644 --- a/tests/integration/challenges/current_score.json +++ b/tests/integration/challenges/current_score.json @@ -46,3 +46,5 @@ } } } +test +test From e908e3fa15e96b4d9f54acc6c6bd91f31c83ce7e Mon Sep 17 00:00:00 2001 From: Auto-GPT-Bot Date: Sun, 4 Jun 2023 23:17:21 +0000 Subject: [PATCH 5/5] Update cassette submodule --- tests/Auto-GPT-test-cassettes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes index 28497a8b4ef0..7bc9696d7027 160000 --- a/tests/Auto-GPT-test-cassettes +++ b/tests/Auto-GPT-test-cassettes @@ -1 +1 @@ -Subproject commit 28497a8b4ef0e62375dd5024794426cb401a3779 +Subproject commit 7bc9696d70275e2587a6918c79ca0c88341ce288