From dfdd16820b6943d86e89adfbbd06048d53dde9fe Mon Sep 17 00:00:00 2001
From: skumargupta83 <skumargupta-sonata@2u.com>
Date: Fri, 3 Oct 2025 17:12:08 +0000
Subject: [PATCH] chore: Added backoff exception

---
 .../apps/course_metadata/tests/test_utils.py  | 17 ++++++++--
 .../apps/course_metadata/utils.py             | 34 +++++++++++++++++++
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index 7ce9161016..15e82b5a74 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -11,10 +11,12 @@
 import responses
 from django.conf import settings
 from django.core.exceptions import ValidationError
+from django.http.response import HttpResponse
 from django.test import TestCase
 from edx_django_utils.cache import RequestCache
 from edx_toggles.toggles.testutils import override_waffle_switch
 from slugify import slugify
+from slumber.exceptions import HttpClientError
 
 from course_discovery.apps.api.tests.mixins import SiteMixin
 from course_discovery.apps.api.v1.tests.test_views.mixins import OAuth2Mixin
@@ -44,7 +46,7 @@
     calculated_seat_upgrade_deadline, clean_html, convert_svg_to_png_from_url, create_missing_entitlement,
     download_and_save_course_image, download_and_save_program_image, ensure_draft_world, fetch_getsmarter_products,
     generate_sku, is_google_drive_url, serialize_entitlement_for_ecommerce_api, serialize_seat_for_ecommerce_api,
-    transform_skills_data, validate_slug_format
+    transform_skills_data, validate_slug_format, is_fatal_error
 )
 
 
@@ -1161,6 +1163,16 @@ def tearDown(self):
         responses.reset()
         super().tearDown()
 
+    def test_is_fatal_code(self):
+        response_with_200 = HttpResponse(status=200)
+        response_with_400 = HttpResponse(status=400)
+        response_with_429 = HttpResponse(status=429)
+        response_with_504 = HttpResponse(status=504)
+        assert not is_fatal_error(HttpClientError(response=response_with_200))
+        assert is_fatal_error(HttpClientError(response=response_with_400))
+        assert not is_fatal_error(HttpClientError(response=response_with_429))
+        assert not is_fatal_error(HttpClientError(response=response_with_504))
+
     def mock_product_api_call(self):
         """
         Mock product api with success response.
@@ -1196,8 +1208,7 @@ def test_fetch_getsmarter_products__with_invalid_credentials(self, mock_getsmart
         products = fetch_getsmarter_products()
         mock_logger.assert_called_with(f'Failed to retrieve products from getsmarter API: {exception_message}')
         assert products == []
-
-
+               
 @ddt.ddt
 class CourseSlugMethodsTests(TestCase):
     """
diff --git a/course_discovery/apps/course_metadata/utils.py b/course_discovery/apps/course_metadata/utils.py
index 2f3fe88e61..e131e8db82 100644
--- a/course_discovery/apps/course_metadata/utils.py
+++ b/course_discovery/apps/course_metadata/utils.py
@@ -1,3 +1,4 @@
+import backoff
 import datetime
 import logging
 import random
@@ -73,6 +74,26 @@ def clean_query(query):
     return query
 
 
+def is_fatal_error(ex):
+    """
+    Return True if the exception represents a fatal client error.
+
+    Fatal means:
+    - The response exists
+    - The status code is a 4XX client error (400–499)
+    - The error is not a 429 (rate limiting)
+    """
+    response = ex.response
+    if response is None:
+        return False
+
+    code = response.status_code
+    if code == 429:
+        return False
+
+    return 400 <= code < 500
+
+
 def set_official_state(obj, model, attrs=None):
     """
     Given a draft object and the model of that object, ensure that an official version is created
@@ -986,6 +1007,19 @@ def transform_skills_data(skills_data):
     return skills
 
 
+# The courses endpoint has 40 requests/minute rate limit.
+# This will back off at a rate of 60/120/240 seconds (from the factor 60 and default value of base 2).
+# This backoff code can still fail because of the concurrent requests all requesting at the same time.
+# So even in the case of entering into the next minute, if we still exceed our limit for that min,
+# any requests that failed in both limits are still approaching their max_tries limit.
+@backoff.on_exception(
+    backoff.expo,
+    (requests.exceptions.Timeout,requests.exceptions.HTTPError),
+    factor=60,
+    max_tries=4,
+    giveup=is_fatal_error,
+    max_time=300
+)
 def fetch_getsmarter_products():
     """ Returns the products details from the getsmarter API """
     products = []