diff --git a/beanprice/sources/yahoo.py b/beanprice/sources/yahoo.py index 93ad2d0..1e07972 100644 --- a/beanprice/sources/yahoo.py +++ b/beanprice/sources/yahoo.py @@ -25,11 +25,12 @@ from curl_cffi import requests from beanprice import source - +import time class YahooError(ValueError): "An error from the Yahoo API." - +RETRY_DELAYS = [1, 2, 4, 8, 16] +MAX_RETRIES = len(RETRY_DELAYS) def parse_response(response: requests.models.Response) -> Dict: """Process as response from Yahoo. @@ -53,6 +54,33 @@ def parse_response(response: requests.models.Response) -> Dict: raise YahooError("No data returned from Yahoo, ensure that the symbol is correct") return content["result"][0] +def make_yahoo_request(session: requests.Session, url: str, params: Dict) -> requests.models.Response: + """ + Helper function to make a request to Yahoo Finance with retry logic. + Handles 429 (Too Many Requests) and connection errors with exponential backoff. + """ + for i in range(MAX_RETRIES): + try: + response = session.get(url, params=params) + if response.status_code == 429: + print(f"DEBUG: Rate limit hit (429) for {url}. Retrying in {RETRY_DELAYS[i]} seconds...") + time.sleep(RETRY_DELAYS[i] + (i * 0.5)) # Adds jitter + continue + response.raise_for_status() # Raise HTTPError for other bad status codes (4xx or 5xx) + return response + except requests.exceptions.HTTPError as e: + if e.response.status_code == 429: # Catch 429 again if raise_for_status caught it + print(f"DEBUG: Rate limit hit (429) during retry {i+1}/{MAX_RETRIES}. Retrying in {RETRY_DELAYS[i]} seconds...") + time.sleep(RETRY_DELAYS[i] + (i * 0.5)) + continue + raise + except requests.exceptions.ConnectionError as e: + print(f"DEBUG: Connection error: {e}. Retrying in {RETRY_DELAYS[i]} seconds...") + time.sleep(RETRY_DELAYS[i] + (i * 0.5)) + continue + raise YahooError(f"Failed to fetch data after {MAX_RETRIES} retries due to persistent rate limiting or connection issues for {url}.") + + # Note: Feel free to suggest more here via a PR. _MARKETS = { @@ -93,7 +121,7 @@ def get_price_series( "interval": "1d", } payload.update(_DEFAULT_PARAMS) - response = session.get(url, params=payload) # Use shared session + response = make_yahoo_request(session, url, payload) # Use shared session and retry logic result = parse_response(response) meta = result["meta"] @@ -157,8 +185,7 @@ def get_latest_price(self, ticker: str) -> Optional[source.SourcePrice]: "crumb": self.crumb, # Use the session’s crumb } payload.update(_DEFAULT_PARAMS) - response = self.session.get(url, params=payload) # Use shared session - + response = make_yahoo_request(self.session, url, payload) # Use shared session and retry logic try: result = parse_response(response) except YahooError as error: