fix(libs): Allow for manual rebuilding and fix commit checking (espressif#245)

Author: lucasssvaz

.github/workflows/cron.yml

@@ -12,7 +12,7 @@ on:
 #             │ │ │ │ │
 #             * * * * *
     - cron:  '0 */6 * * *'
-  workflow_dispatch: # For testing
+  workflow_dispatch: # For manually rebuilding the libraries
 
 defaults:
   run:
@@ -21,6 +21,7 @@ defaults:
 jobs:
   build-libs:
     name: Build with IDF ${{ matrix.idf_branch }}
+    if: github.repository_owner == 'espressif'
     uses: ./.github/workflows/cron_build.yml
     with:
       idf_branch: ${{ matrix.idf_branch }}

.github/workflows/cron_build.yml

@@ -132,7 +132,6 @@ jobs:
           compression-level: 0
 
       - name: Push changes
-        if: github.repository == 'espressif/esp32-arduino-lib-builder'
         env:
           GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
           GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}

tools/check-deploy-needed.sh

@@ -37,19 +37,40 @@ AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT"
 AR_NEW_PR_TITLE="IDF $IDF_BRANCH"
 
 LIBS_RELEASE_TAG="idf-"${IDF_BRANCH//\//_}""
-LIBS_VERSION="$LIBS_RELEASE_TAG-$IDF_COMMIT"
+LIBS_VERSION_PREFIX="$LIBS_RELEASE_TAG-$IDF_COMMIT-v"
+VERSION_COUNTER=1
 
 AR_HAS_BRANCH=`github_branch_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME"`
 if [ "$AR_HAS_BRANCH" == "1" ]; then
-	AR_HAS_COMMIT=`github_commit_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME" "$IDF_COMMIT"`
+	LATEST_LIBS_IDF=`github_get_libs_idf "$AR_REPO" "$AR_NEW_BRANCH_NAME" "$AR_NEW_PR_TITLE"`
 else
-	AR_HAS_COMMIT=`github_commit_exists "$AR_REPO" "$AR_BRANCH" "$IDF_COMMIT"`
+	LATEST_LIBS_IDF=`github_get_libs_idf "$AR_REPO" "$AR_BRANCH" "$AR_NEW_PR_TITLE"`
 fi
+
+echo "Current IDF commit: $IDF_COMMIT"
+echo "Latest IDF commit in $AR_BRANCH of $AR_REPO: $LATEST_LIBS_IDF"
+
+AR_HAS_COMMIT=`if [ "$LATEST_LIBS_IDF" == "$IDF_COMMIT" ]; then echo "1"; else echo "0"; fi`
 AR_HAS_PR=`github_pr_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME"`
 
 LIBS_RELEASE_ID=`github_release_id "$AR_LIBS_REPO" "$LIBS_RELEASE_TAG"`
 LIBS_HAS_RELEASE=`if [ -n "$LIBS_RELEASE_ID" ]; then echo "1"; else echo "0"; fi`
-LIBS_ASSET_ID=`github_release_asset_id "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "esp32-arduino-libs-$LIBS_VERSION.zip"`
+
+if [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
+	echo "Workflow dispatch event. Generating new libs."
+	while true; do
+		LIBS_ASSET_ID=`github_release_asset_id "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "esp32-arduino-libs-$LIBS_VERSION_PREFIX$VERSION_COUNTER.zip"`
+		if [ -n "$LIBS_ASSET_ID" ]; then
+			VERSION_COUNTER=$((VERSION_COUNTER+1))
+		else
+			break
+		fi
+	done
+else
+	LIBS_ASSET_ID=`github_release_asset_id "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "esp32-arduino-libs-$LIBS_VERSION_PREFIX$VERSION_COUNTER.zip"`
+fi
+
+LIBS_VERSION="$LIBS_VERSION_PREFIX$VERSION_COUNTER"
 LIBS_HAS_ASSET=`if [ -n "$LIBS_ASSET_ID" ]; then echo "1"; else echo "0"; fi`
 
 export IDF_COMMIT

tools/config.sh

@@ -25,7 +25,7 @@ if [ -z $IDF_TARGET ]; then
 fi
 
 # Owner of the target ESP32 Arduino repository
-AR_USER="espressif"
+AR_USER="$GITHUB_REPOSITORY_OWNER"
 
 # The full name of the repository
 AR_REPO="$AR_USER/arduino-esp32"
@@ -99,12 +99,55 @@ if [[ "$AR_OS" == "macos" ]]; then
     export SSTAT="stat -f %z"
 fi
 
+function github_get_libs_idf(){ # github_get_libs_idf <repo-path> <branch-name> <message-prefix>
+    local repo_path="$1"
+    local branch_name="$2"
+    local message_prefix="$3"
+    message_prefix=$(echo $message_prefix | sed 's/[]\/$*.^|[]/\\&/g') # Escape special characters
+    local page=1
+    local version_found=""
+    local libs_version=""
+
+    while [[ "$libs_version" == "" && "$page" -le 3 ]]; do
+        # Get the latest commit message that matches the prefix and extract the hash from the last commit message
+        version_found=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/commits?sha=$branch_name&per_page=100&page=$page" | \
+            jq -r --arg prefix "$message_prefix" '[ .[] | select(.commit.message | test($prefix + " [a-f0-9]{8}")) ][0] | .commit.message' | \
+            grep -Eo "$message_prefix [a-f0-9]{8}" | \
+            awk 'END {print $NF}'`
+        if [[ "$version_found" != "" && "$version_found" != "null" ]]; then
+            libs_version=$version_found
+        else
+            page=$((page+1))
+        fi
+    done
+
+    if [ ! "$libs_version" == "" ] && [ ! "$libs_version" == "null" ]; then echo $libs_version; else echo ""; fi
+}
+
 function github_commit_exists(){ #github_commit_exists <repo-path> <branch-name> <commit-message>
     local repo_path="$1"
     local branch_name="$2"
     local commit_message="$3"
-    local commits_found=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/commits?sha=$branch_name" | jq -r '.[].commit.message' | grep "$commit_message" | wc -l`
-    if [ ! "$commits_found" == "" ] && [ ! "$commits_found" == "null" ] && [ ! "$commits_found" == "0" ]; then echo $commits_found; else echo 0; fi
+    local page=1
+    local commits_found=0
+
+    while [ "$page" -le 3 ]; do
+        local response=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/commits?sha=$branch_name&per_page=100&page=$page"`
+
+        if [[ -z "$response" || "$response" == "[]" ]]; then
+            break
+        fi
+
+        local commits=`echo "$response" | jq -r '.[].commit.message' | grep "$commit_message" | wc -l`
+        if [ "$commits" -gt 0 ]; then
+            commits_found=1
+            break
+        fi
+
+        page=$((page+1))
+    done
+
+    echo $commits_found
 }
 
 function github_last_commit(){ # github_last_commit <repo-path> <branch-name>