From f87dbf537fa3f421e70649cc7f9a123357dd9490 Mon Sep 17 00:00:00 2001 From: Greg O'Grady Date: Tue, 12 Nov 2024 21:53:00 -0500 Subject: [PATCH 1/5] refactor: more jq and more logging --- src/jobs/e2e/collect-e2e-logs.yml | 35 ++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/jobs/e2e/collect-e2e-logs.yml b/src/jobs/e2e/collect-e2e-logs.yml index 43cf0ce4..15b862ec 100644 --- a/src/jobs/e2e/collect-e2e-logs.yml +++ b/src/jobs/e2e/collect-e2e-logs.yml @@ -84,13 +84,33 @@ steps: - run: name: Wait for smoke test jobs to complete command: | - # Loop through to check if each of the jobs have been completed - for job in $(echo "<< parameters.smoke-test-jobs >>" | tr "," "\n"); do - while [[ $(curl --location --request GET "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" --header "Circle-Token: $CIRCLE_TOKEN"| jq -r ".items[]|select(.name == \"$job\")|.status" | grep -c "running") -gt 0 ]] - do - sleep 5 - done + FILTER=$(cat \<job-status.log ; \ + curl "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" \ + --silent \ + --location \ + --request GET \ + --header "Circle-Token: $CIRCLE_TOKEN" \ + | jq -cr --arg jobstr "<< parameters.smoke-test-jobs >>" "$FILTER" \ + | tee -a job-status.log \ + | jq -sr 'map(select(.status == "running")) | length' + ) -gt 0 ]]; do + cat job-status.log + sleep 5 done + - run: echo "All required jobs have now completed" - run: name: Gather Kubernetes state after run @@ -108,8 +128,7 @@ steps: # Gather summary state of all pods in the namespace kubectl get pods -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/pods-summary-state-after-run.log" # Gather detailed state of all pods in the namespace. - for ((i = 0; i < ${#components[@]}; i++)); do - component=${components[$i]} + for component in $components; do echo "Capturing logs for component $component" kubectl describe pod $component -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/${component}-k8-state.log" & done From 80af6414a46bdd4fb942a915625cb9990dbcce97 Mon Sep 17 00:00:00 2001 From: Greg O'Grady Date: Tue, 12 Nov 2024 22:26:08 -0500 Subject: [PATCH 2/5] refactor: cleanup and add more logging to waiter --- src/jobs/e2e/waiter.yml | 75 ++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/src/jobs/e2e/waiter.yml b/src/jobs/e2e/waiter.yml index 96bb5b3c..84cf80f7 100644 --- a/src/jobs/e2e/waiter.yml +++ b/src/jobs/e2e/waiter.yml @@ -10,58 +10,79 @@ steps: - run: name: Check if all jobs have completed command: | - # Global variable to store the status code - check_jobs_status=0 ## Function to check if the API request was successful and process the response check_jobs() { + date -u echo "Fetching jobs from CircleCI API..." - response=$(curl --silent --location --request GET "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" --header "Circle-Token: $CIRCLECI_API_TOKEN") + response=$(curl "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" \ + --silent \ + --location \ + --request GET \ + --header "Circle-Token: $CIRCLECI_API_TOKEN") + if [[ $? -ne 0 ]]; then echo "Error: Failed to fetch jobs from CircleCI API" - check_jobs_status=1 - return + return 1 fi - echo "API response received:" >&2 - echo "$response" >&2 - if echo "$response" | jq -e . >/dev/null 2>&1; then - statuses=$(echo "$response" | jq -r '.items[] | select(.name != "vfcommon/waiter") | .status') - if [[ -z "$statuses" ]]; then - echo "No job statuses found or unexpected response format" - check_jobs_status=1 - return - fi - echo "Job statuses extracted:" - echo "$statuses" - else + + echo "API response received" >&2 + echo "$response" >responses.log + + if ! jq -e . >/dev/null 2>&1 \<<<"$response" ; then echo "Error: Malformed JSON response" - check_jobs_status=1 - return + cat responses.log + return 1 + fi + + STATUS_FILTER=$(cat \<> seconds due to error..." - sleep << parameters.wait-duration >> - continue elif [[ $status -eq 2 ]]; then echo "All jobs completed successfully." - exit 0 # success + exit 0 # success else echo "Jobs are still running, checking again in << parameters.wait-duration >> seconds..." - sleep << parameters.wait-duration >> fi + sleep << parameters.wait-duration >> done + - run: echo "All required jobs have now completed" From 5ecc694b83b046f3ce92c7d5748ca76945704da0 Mon Sep 17 00:00:00 2001 From: Greg O'Grady Date: Tue, 12 Nov 2024 22:42:03 -0500 Subject: [PATCH 3/5] chore: formatting --- src/jobs/e2e/collect-e2e-logs.yml | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/jobs/e2e/collect-e2e-logs.yml b/src/jobs/e2e/collect-e2e-logs.yml index 15b862ec..dad5f2cd 100644 --- a/src/jobs/e2e/collect-e2e-logs.yml +++ b/src/jobs/e2e/collect-e2e-logs.yml @@ -44,10 +44,10 @@ steps: echo "Contents of << parameters.env-name-path >>:" cat << parameters.env-name-path >> if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >> )" != "null" ]; then - DEV_ENV_NAME=$(cat << parameters.env-name-path >> ) - else - DEV_ENV_NAME=<< parameters.e2e-env-name >> - fi + DEV_ENV_NAME=$(cat << parameters.env-name-path >> ) + else + DEV_ENV_NAME=<< parameters.e2e-env-name >> + fi # Gather summary state of all pods in the namespace echo "Gathering Kubernetes state before run for env $DEV_ENV_NAME" kubectl get pods -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/pods-summary-state-before-run.log" @@ -59,8 +59,8 @@ steps: background: true command: | function capture_logs() { - if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >> )" != "null" ]; then - DEV_ENV_NAME=$(cat << parameters.env-name-path >> ) + if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >>)" != "null" ]; then + DEV_ENV_NAME=$(cat << parameters.env-name-path >>) else DEV_ENV_NAME=<< parameters.e2e-env-name >> fi @@ -71,13 +71,13 @@ steps: for ((i = 0; i < ${#components[@]} - 1; i++)); do component=${components[$i]} echo "Capturing logs for component $component" - stern -n "${DEV_ENV_NAME:?}" -l "app.kubernetes.io/name=$component" >>"${LOG_DIR:?}/${COMPONENT_LOG_DIR:?}/$component.log" --since 5m & + stern -n "${DEV_ENV_NAME:?}" -l "app.kubernetes.io/name=$component" --since 5m >>"${LOG_DIR:?}/${COMPONENT_LOG_DIR:?}/$component.log" & done # Handle the last component separately to introduce blocking.If all components' log collection is done as non blocking tasks, # the circleci step will terminate.So having last component as blocking ensures the collect logs step continues executing - last_component=${components[${#components[@]}-1]} + last_component=${components[${#components[@]} - 1]} echo "Capturing logs for last component $last_component" - stern -n "${DEV_ENV_NAME:?}" -l "app.kubernetes.io/name=$last_component" >>"${LOG_DIR:?}/${COMPONENT_LOG_DIR:?}/$last_component.log" --since 5m + stern -n "${DEV_ENV_NAME:?}" -l "app.kubernetes.io/name=$last_component" --since 5m >>"${LOG_DIR:?}/${COMPONENT_LOG_DIR:?}/$last_component.log" } capture_logs @@ -118,21 +118,21 @@ steps: LOG_DIR: *log_dir KUBE_STATE_DIR: *kube_state_dir command: | - if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >> )" != "null" ]; then - DEV_ENV_NAME=$(cat << parameters.env-name-path >> ) - else - DEV_ENV_NAME=<< parameters.e2e-env-name >> - fi + if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >>)" != "null" ]; then + DEV_ENV_NAME=$(cat << parameters.env-name-path >>) + else + DEV_ENV_NAME=<< parameters.e2e-env-name >> + fi # Read components into an array directly from the command output components=($(vfcli component list -n "${DEV_ENV_NAME:?}" | awk 'NR>3 {print $1}')) # Gather summary state of all pods in the namespace - kubectl get pods -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/pods-summary-state-after-run.log" + kubectl get pods -n $DEV_ENV_NAME >>"${LOG_DIR:?}/${KUBE_STATE_DIR:?}/pods-summary-state-after-run.log" # Gather detailed state of all pods in the namespace. for component in $components; do echo "Capturing logs for component $component" - kubectl describe pod $component -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/${component}-k8-state.log" & - done - wait + kubectl describe pod $component -n $DEV_ENV_NAME >>"${LOG_DIR:?}/${KUBE_STATE_DIR:?}/${component}-k8-state.log" & + done + wait - run: name: Gather Kubernetes events environment: From 0844b73c06cba1d4b81896d1f2295c22cfc4a65f Mon Sep 17 00:00:00 2001 From: theProf Date: Wed, 13 Nov 2024 15:53:10 +0000 Subject: [PATCH 4/5] build(fix): use bash array expansion --- src/jobs/e2e/collect-e2e-logs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/e2e/collect-e2e-logs.yml b/src/jobs/e2e/collect-e2e-logs.yml index dad5f2cd..5afdc178 100644 --- a/src/jobs/e2e/collect-e2e-logs.yml +++ b/src/jobs/e2e/collect-e2e-logs.yml @@ -128,7 +128,7 @@ steps: # Gather summary state of all pods in the namespace kubectl get pods -n $DEV_ENV_NAME >>"${LOG_DIR:?}/${KUBE_STATE_DIR:?}/pods-summary-state-after-run.log" # Gather detailed state of all pods in the namespace. - for component in $components; do + for component in "${components[@]}"; do echo "Capturing logs for component $component" kubectl describe pod $component -n $DEV_ENV_NAME >>"${LOG_DIR:?}/${KUBE_STATE_DIR:?}/${component}-k8-state.log" & done From f68c6b0429d788526286eca8ec2227ff09ea1ce3 Mon Sep 17 00:00:00 2001 From: Greg O'Grady Date: Thu, 14 Nov 2024 21:13:22 -0500 Subject: [PATCH 5/5] refactor(wip): function --- src/jobs/e2e/collect-e2e-logs.yml | 38 ++++++++++++------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/src/jobs/e2e/collect-e2e-logs.yml b/src/jobs/e2e/collect-e2e-logs.yml index 5afdc178..1a4e7cc4 100644 --- a/src/jobs/e2e/collect-e2e-logs.yml +++ b/src/jobs/e2e/collect-e2e-logs.yml @@ -84,30 +84,22 @@ steps: - run: name: Wait for smoke test jobs to complete command: | - FILTER=$(cat \<>") + function running_jobs() { + DATE=$(date -u) + FILTERED_JOBS=$( + curl "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" \ + --silent \ + --location \ + --request GET \ + --header "Circle-Token: $CIRCLE_TOKEN" \ + | jq -cr ".items[] | { status, started_at, name }" \ + | grep -E "\"name\": \"($JOBS)\"" ) - | { status, started_at, name } - EOF - ) - - while [[ $( - date -u >job-status.log ; \ - curl "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" \ - --silent \ - --location \ - --request GET \ - --header "Circle-Token: $CIRCLE_TOKEN" \ - | jq -cr --arg jobstr "<< parameters.smoke-test-jobs >>" "$FILTER" \ - | tee -a job-status.log \ - | jq -sr 'map(select(.status == "running")) | length' - ) -gt 0 ]]; do - cat job-status.log + echo "$DATE - \n${FILTERED_JOBS}" + return grep -qE '"status": "running"' \<<<"$FILTERED_JOBS" + } + while running_jobs ; do sleep 5 done