[Test] Add retry helper for flaky InSpec controls. Use such retry logic to make the yum repo check more resilient to transient failures.

Author: gmarciani

cookbooks/aws-parallelcluster-environment/test/controls/lustre_spec.rb

@@ -19,10 +19,12 @@
         its('version') { should cmp >= minimal_lustre_client_version }
       end
 
-      describe yum.repo('aws-fsx') do
-        it { should exist }
-        it { should be_enabled }
-        its('baseurl') { should include 'fsx-lustre-client-repo.s3.amazonaws.com' }
+      with_retry(retries: 3, delay: 10) do
+        describe yum.repo('aws-fsx') do
+          it { should exist }
+          it { should be_enabled }
+          its('baseurl') { should include 'fsx-lustre-client-repo.s3.amazonaws.com' }
+        end
       end
     end
   end
@@ -47,10 +49,12 @@
         its('version') { should cmp >= minimal_lustre_client_version }
       end
 
-      describe yum.repo('aws-fsx') do
-        it { should exist }
-        it { should be_enabled }
-        its('baseurl') { should include 'fsx-lustre-client-repo.s3.amazonaws.com' }
+      with_retry(retries: 3, delay: 5) do
+        describe yum.repo('aws-fsx') do
+          it { should exist }
+          it { should be_enabled }
+          its('baseurl') { should include 'fsx-lustre-client-repo.s3.amazonaws.com' }
+        end
       end
     end
   end

cookbooks/aws-parallelcluster-shared/test/libraries/retry_helper.rb

@@ -0,0 +1,15 @@
+# Helper method to retry flaky InSpec checks
+def with_retry(retries: 3, delay: 5)
+  attempts = 0
+  begin
+    attempts += 1
+    yield
+  rescue => e
+    if attempts < retries
+      sleep delay
+      retry
+    else
+      raise e
+    end
+  end
+end