From 550617ee9ab066c0b839ef855e681f3b43fa8feb Mon Sep 17 00:00:00 2001 From: Rohan Weeden Date: Mon, 30 Dec 2024 11:37:29 -0900 Subject: [PATCH 1/2] Bump python version to 3.9 --- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 026e769..68ad951 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - run: pip install -r requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba40724..f8b93a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | From 7898c9119d570d0b1d1114e840138abc59eeccaf Mon Sep 17 00:00:00 2001 From: Rohan Weeden Date: Mon, 30 Dec 2024 11:30:27 -0900 Subject: [PATCH 2/2] Refactor EDL calls to class --- rain_api_core/edl.py | 130 ++++++++++++++++++++++++++++++++++ rain_api_core/urs_util.py | 144 ++++++++++++++++---------------------- tests/test_edl.py | 107 ++++++++++++++++++++++++++++ tests/test_urs_util.py | 71 ++++++++++--------- 4 files changed, 335 insertions(+), 117 deletions(-) create mode 100644 rain_api_core/edl.py create mode 100644 tests/test_edl.py diff --git a/rain_api_core/edl.py b/rain_api_core/edl.py new file mode 100644 index 0000000..8b1c813 --- /dev/null +++ b/rain_api_core/edl.py @@ -0,0 +1,130 @@ +import json +import logging +import os +import urllib.error +import urllib.parse +import urllib.request +from typing import Optional + +from rain_api_core.general_util import return_timing_object +from rain_api_core.timer import Timer + +log = logging.getLogger(__name__) + + +class EdlException(Exception): + def __init__( + self, + inner: Exception, + msg: dict, + payload: Optional[bytes], + ): + self.inner = inner + self.msg = msg + self.payload = payload + + +class EulaException(EdlException): + pass + + +class EdlClient: + def __init__( + self, + base_url: str = os.getenv( + 'AUTH_BASE_URL', + 'https://urs.earthdata.nasa.gov', + ), + ): + self.base_url = base_url + + def request( + self, + method: str, + endpoint: str, + params: dict = {}, + data: dict = {}, + headers: dict = {}, + ) -> dict: + if params: + params_encoded = urllib.parse.urlencode(params) + url_params = f'?{params_encoded}' + else: + url_params = '' + + # Separate variables so we can log the url without params + url = urllib.parse.urljoin(self.base_url, endpoint) + url_with_params = url + url_params + + if data: + data_encoded = urllib.parse.urlencode(data).encode() + else: + data_encoded = None + + request = urllib.request.Request( + url=url_with_params, + data=data_encoded, + headers=headers, + method=method, + ) + + log.debug( + 'Request(url=%r, data=%r, headers=%r)', + url_with_params, + data, + headers, + ) + + timer = Timer() + timer.mark(f'urlopen({url})') + try: + with urllib.request.urlopen(request) as f: + payload = f.read() + timer.mark('json.loads()') + msg = json.loads(payload) + timer.mark() + + log.info( + return_timing_object( + service='EDL', + endpoint=url, + duration=timer.total.duration() * 1000, + unit='milliseconds', + ), + ) + timer.log_all(log) + + return msg + except urllib.error.URLError as e: + log.error('Error hitting endpoint %s: %s', url, e) + timer.mark() + log.debug('ET for the attempt: %.4f', timer.total.duration()) + + self._parse_edl_error(e) + except json.JSONDecodeError as e: + raise EdlException(e, {}, payload) + + def _parse_edl_error(self, e: urllib.error.URLError): + if isinstance(e, urllib.error.HTTPError): + payload = e.read() + try: + msg = json.loads(payload) + except json.JSONDecodeError: + log.error('Could not get json message from payload: %s', payload) + msg = {} + + if ( + e.code in (403, 401) + and 'error_description' in msg + and 'eula' in msg['error_description'].lower() + ): + # sample json in this case: + # `{"status_code": 403, "error_description": "EULA Acceptance Failure", + # "resolution_url": "http://uat.urs.earthdata.nasa.gov/approve_app?client_id=LqWhtVpLmwaD4VqHeoN7ww"}` + log.warning('user needs to sign the EULA') + raise EulaException(e, msg, payload) + else: + payload = None + msg = {} + + raise EdlException(e, msg, payload) diff --git a/rain_api_core/urs_util.py b/rain_api_core/urs_util.py index ca127b1..c63308b 100644 --- a/rain_api_core/urs_util.py +++ b/rain_api_core/urs_util.py @@ -1,14 +1,11 @@ -import json import logging import os -import urllib -from time import time +from typing import Optional from rain_api_core.auth import JwtManager, UserProfile from rain_api_core.aws_util import retrieve_secret -from rain_api_core.general_util import duration, return_timing_object +from rain_api_core.edl import EdlClient, EdlException from rain_api_core.logging import log_context -from rain_api_core.timer import Timer log = logging.getLogger(__name__) @@ -27,43 +24,28 @@ def get_redirect_url(ctxt: dict = None) -> str: return f'{get_base_url(ctxt)}login' -def do_auth(code: str, redirect_url: str, aux_headers: dict = None) -> dict: - aux_headers = aux_headers or {} # A safer default - url = os.getenv('AUTH_BASE_URL', 'https://urs.earthdata.nasa.gov') + "/oauth/token" - +def do_auth(code: str, redirect_url: str, aux_headers: dict = {}) -> dict: # App U:P from URS Application auth = get_urs_creds()['UrsAuth'] - post_data = {"grant_type": "authorization_code", - "code": code, - "redirect_uri": redirect_url} + data = { + 'grant_type': 'authorization_code', + 'code': code, + 'redirect_uri': redirect_url, + } - headers = {"Authorization": "Basic " + auth} + headers = {'Authorization': 'Basic ' + auth} headers.update(aux_headers) - post_data_encoded = urllib.parse.urlencode(post_data).encode("utf-8") - post_request = urllib.request.Request(url, post_data_encoded, headers) - - timer = Timer() - timer.mark("do_auth() urlopen()") + client = EdlClient() try: - log.debug('headers: {}'.format(headers)) - log.debug('url: {}'.format(url)) - log.debug('post_data: {}'.format(post_data)) - - response = urllib.request.urlopen(post_request) # nosec URL is *always* URS. - timer.mark("do_auth() request to URS") - packet = response.read() - timer.mark() - - timer.log_all(log) - log.info(return_timing_object(service="EDL", endpoint=url, method="POST", duration=timer.total.duration())) - - return json.loads(packet) - except urllib.error.URLError as e: - log.error("Error fetching auth: %s", e) - timer.mark() - log.debug("ET for the attempt: %.4f", timer.total.duration()) + return client.request( + 'POST', + '/oauth/token', + data=data, + headers=headers, + ) + except EdlException: return {} @@ -108,35 +90,36 @@ def get_user_profile(urs_user_payload: dict, access_token) -> UserProfile: ) -def get_profile(user_id: str, token: str, temptoken: str = None, aux_headers: dict = None) -> UserProfile: - aux_headers = aux_headers or {} # Safer Default - +def get_profile( + user_id: str, + token: str, + temptoken: str = None, + aux_headers: dict = {}, +) -> Optional[UserProfile]: if not user_id or not token: return None - # get_new_token_and_profile() will pass this function a temporary token with which to fetch the profile info. We - # don't want to keep it around, just use it here, once: + # get_new_token_and_profile() will pass this function a temporary token with + # which to fetch the profile info. We don't want to keep it around, just use + # it here, once: if temptoken: headertoken = temptoken else: headertoken = token - url = os.getenv('AUTH_BASE_URL', 'https://urs.earthdata.nasa.gov') + "/api/users/{0}".format(user_id) - headers = {"Authorization": "Bearer " + headertoken} + headers = {'Authorization': 'Bearer ' + headertoken} headers.update(aux_headers) - req = urllib.request.Request(url, None, headers) + client = EdlClient() try: - timer = time() - response = urllib.request.urlopen(req) # nosec URL is *always* URS. - packet = response.read() - log.info(return_timing_object(service="EDL", endpoint=url, duration=duration(timer))) - user_profile = json.loads(packet) - + user_profile = client.request( + 'GET', + f'/api/users/{user_id}', + headers=headers, + ) return get_user_profile(user_profile, headertoken) - - except urllib.error.URLError as e: - log.warning("Error fetching profile: {0}".format(e)) + except EdlException as e: + log.warning('Error fetching profile: %s', e.inner) if not temptoken: # This keeps get_new_token_and_profile() from calling this over and over log.debug('because error above, going to get_new_token_and_profile()') return get_new_token_and_profile(user_id, token, aux_headers) @@ -148,46 +131,39 @@ def get_profile(user_id: str, token: str, temptoken: str = None, aux_headers: di return None -def get_new_token_and_profile(user_id: str, cookietoken: str, aux_headers: dict = None): - aux_headers = aux_headers or {} # A safer default - - # get a new token - url = os.getenv('AUTH_BASE_URL', 'https://urs.earthdata.nasa.gov') + "/oauth/token" - +def get_new_token_and_profile( + user_id: str, + cookietoken: str, + aux_headers: dict = {}, +) -> Optional[UserProfile]: # App U:P from URS Application auth = get_urs_creds()['UrsAuth'] - post_data = {"grant_type": "client_credentials"} - headers = {"Authorization": "Basic " + auth} - headers.update(aux_headers) + data = {'grant_type': 'client_credentials'} - # Download token - post_data_encoded = urllib.parse.urlencode(post_data).encode("utf-8") - post_request = urllib.request.Request(url, post_data_encoded, headers) + headers = {'Authorization': 'Basic ' + auth} + headers.update(aux_headers) - timer = Timer() - timer.mark("get_new_token_and_profile() urlopen()") + client = EdlClient() try: - log.info("Attempting to get new Token") - - response = urllib.request.urlopen(post_request) # nosec URL is *always* URS. + log.info('Attempting to get new Token') - timer.mark("get_new_token_and_profile() response.read()") - packet = response.read() - - timer.mark("get_new_token_and_profile() json.loads()") - log.info(return_timing_object(service="EDL", endpoint=url, duration=timer.total.duration())) - new_token = json.loads(packet)['access_token'] - timer.mark() + response = client.request( + 'POST', + '/oauth/token', + data=data, + headers=headers, + ) + new_token = response['access_token'] - log.info("Retrieved new token: {0}".format(new_token)) - timer.log_all(log) + log.info('Retrieved new token: %s', new_token) # Get user profile with new token - return get_profile(user_id, cookietoken, new_token, aux_headers=aux_headers) - - except urllib.error.URLError as e: - log.error("Error fetching auth: %s", e) - timer.mark() - log.debug("ET for the attempt: %.4f", timer.total.duration()) + return get_profile( + user_id, + cookietoken, + new_token, + aux_headers=aux_headers, + ) + except EdlException: return None diff --git a/tests/test_edl.py b/tests/test_edl.py new file mode 100644 index 0000000..a2be7cb --- /dev/null +++ b/tests/test_edl.py @@ -0,0 +1,107 @@ +import io +import json +import urllib.error +from unittest import mock + +import pytest + +from rain_api_core.edl import EdlClient, EdlException, EulaException + +MODULE = "rain_api_core.edl" + + +@pytest.fixture +def edl_client(): + return EdlClient() + + +@mock.patch(f"{MODULE}.urllib.request.urlopen", autospec=True) +def test_client_request(mock_urlopen, edl_client): + mock_urlopen(mock.ANY).__enter__().read.return_value = b'{"foo": "bar"}' + + response = edl_client.request( + "POST", + "/foo/bar", + params={ + "param_1": "value_1", + "param_2": "value_2", + }, + data={ + "data_1": "value_1", + "data_2": "value_2", + }, + headers={"header_1": "value_1"}, + ) + + request_obj = mock_urlopen.mock_calls[2].args[0] + + assert response == {"foo": "bar"} + assert request_obj.method == "POST" + assert request_obj.full_url == ( + "https://urs.earthdata.nasa.gov/foo/bar?param_1=value_1¶m_2=value_2" + ) + assert request_obj.data == b"data_1=value_1&data_2=value_2" + assert request_obj.headers == {"Header_1": "value_1"} + + +@mock.patch(f"{MODULE}.urllib.request.urlopen", autospec=True) +def test_client_request_urlerror(mock_urlopen, edl_client): + test_error = urllib.error.URLError("test error") + mock_urlopen.side_effect = test_error + + with pytest.raises(EdlException) as ex_info: + edl_client.request("GET", "/foo/bar") + + assert ex_info.value.inner is test_error + assert ex_info.value.msg == {} + assert ex_info.value.payload is None + + +@mock.patch(f"{MODULE}.urllib.request.urlopen", autospec=True) +def test_client_request_httperror(mock_urlopen, edl_client): + test_error = urllib.error.HTTPError( + url="/foo/bar", + code=500, + msg="Internal Server Error", + hdrs={}, + fp=io.BytesIO(b'{"foo": "bar"}'), + ) + mock_urlopen.side_effect = test_error + + with pytest.raises(EdlException) as ex_info: + edl_client.request("GET", "/foo/bar") + + assert ex_info.value.inner is test_error + assert ex_info.value.msg == {"foo": "bar"} + assert ex_info.value.payload == b'{"foo": "bar"}' + + +@mock.patch(f"{MODULE}.urllib.request.urlopen", autospec=True) +def test_client_request_httperror_eula(mock_urlopen, edl_client): + error_response = { + "error": "invalid_token", + "status_code": 401, + "error_description": "EULA Acceptance Failure", + "resolution_url": "https://uat.urs.earthdata.nasa.gov/approve_app", + } + error_response_encoded = json.dumps(error_response).encode() + test_error = urllib.error.HTTPError( + url="/foo/bar", + code=401, + msg="Unauthorized", + hdrs={}, + fp=io.BytesIO(error_response_encoded), + ) + mock_urlopen.side_effect = test_error + + with pytest.raises(EulaException) as ex_info: + edl_client.request("GET", "/foo/bar") + + assert ex_info.value.inner is test_error + assert ex_info.value.msg == { + "error": "invalid_token", + "status_code": 401, + "error_description": "EULA Acceptance Failure", + "resolution_url": "https://uat.urs.earthdata.nasa.gov/approve_app", + } + assert ex_info.value.payload == error_response_encoded diff --git a/tests/test_urs_util.py b/tests/test_urs_util.py index 28ffc0b..a7f69c8 100644 --- a/tests/test_urs_util.py +++ b/tests/test_urs_util.py @@ -1,10 +1,10 @@ -import json import urllib from unittest import mock import pytest from rain_api_core.auth import JwtManager, UserProfile +from rain_api_core.edl import EdlException from rain_api_core.urs_util import ( do_auth, do_login, @@ -66,22 +66,24 @@ def test_get_redirect_url(context): assert get_redirect_url(context) == "https://example.com/DEV/login" -@mock.patch(f"{MODULE}.urllib.request", autospec=True) +@mock.patch(f"{MODULE}.EdlClient", autospec=True) @mock.patch(f"{MODULE}.get_urs_creds", autospec=True) -def test_do_auth(mock_get_urs_creds, mock_request): +def test_do_auth(mock_get_urs_creds, mock_client): mock_get_urs_creds.return_value = {"UrsAuth": "URS_AUTH"} - mock_response = mock.NonCallableMock() - mock_response.read.return_value = '{"foo": "bar"}' - mock_request.urlopen.return_value = mock_response + mock_client().request.return_value = {"foo": "bar"} assert do_auth("code", "redir_url") == {"foo": "bar"} -@mock.patch(f"{MODULE}.urllib.request", autospec=True) +@mock.patch(f"{MODULE}.EdlClient", autospec=True) @mock.patch(f"{MODULE}.get_urs_creds", autospec=True) -def test_do_auth_error(mock_get_urs_creds, mock_request): +def test_do_auth_error(mock_get_urs_creds, mock_client): mock_get_urs_creds.return_value = {"UrsAuth": "URS_AUTH"} - mock_request.urlopen.side_effect = urllib.error.URLError("test error") + mock_client().request.side_effect = EdlException( + urllib.error.URLError("test error"), + msg={}, + payload=None, + ) assert do_auth("code", "redir_url") == {} @@ -122,19 +124,15 @@ def test_get_urs_url(mock_get_urs_creds, context): ) -@mock.patch(f"{MODULE}.urllib.request", autospec=True) -def test_get_profile(mock_request): - mock_response = mock.NonCallableMock() - mock_response.read.return_value = json.dumps( - { - "uid": "user_id", - "first_name": "John", - "last_name": "Smith", - "email_address": "peter.l.smith@nasa.gov", - "user_groups": [] - } - ) - mock_request.urlopen.return_value = mock_response +@mock.patch(f"{MODULE}.EdlClient", autospec=True) +def test_get_profile(mock_client): + mock_client().request.return_value = { + "uid": "user_id", + "first_name": "John", + "last_name": "Smith", + "email_address": "peter.l.smith@nasa.gov", + "user_groups": [], + } profile = get_profile("user_id", "token") assert profile.user_id == "user_id" @@ -146,11 +144,15 @@ def test_get_profile(mock_request): assert get_profile("user_id", None) is None -@mock.patch(f"{MODULE}.urllib.request", autospec=True) +@mock.patch(f"{MODULE}.EdlClient", autospec=True) @mock.patch(f"{MODULE}.get_new_token_and_profile", autospec=True) -def test_get_profile_error(mock_get_new_token_and_profile, mock_request): +def test_get_profile_error(mock_get_new_token_and_profile, mock_client): mock_get_new_token_and_profile.return_value = {"foo": "bar"} - mock_request.urlopen.side_effect = urllib.error.URLError("test error") + mock_client().request.side_effect = EdlException( + urllib.error.URLError("test error"), + msg={}, + payload=None, + ) assert get_profile("user_id", "token", "temptoken") is None mock_get_new_token_and_profile.assert_not_called() @@ -159,26 +161,29 @@ def test_get_profile_error(mock_get_new_token_and_profile, mock_request): mock_get_new_token_and_profile.assert_called_once_with("user_id", "token", {}) -@mock.patch(f"{MODULE}.urllib.request", autospec=True) +@mock.patch(f"{MODULE}.EdlClient", autospec=True) @mock.patch(f"{MODULE}.get_profile", autospec=True) @mock.patch(f"{MODULE}.get_urs_creds", autospec=True) -def test_get_new_token_and_profile(mock_get_urs_creds, mock_get_profile, mock_request): +def test_get_new_token_and_profile(mock_get_urs_creds, mock_get_profile, mock_client): mock_get_urs_creds.return_value = {"UrsAuth": "URS_AUTH"} mock_get_profile.return_value = {"foo": "bar"} - mock_response = mock.NonCallableMock() - mock_response.read.return_value = '{"access_token": "token"}' - mock_request.urlopen.return_value = mock_response + + mock_client().request.return_value = {"access_token": "token"} assert get_new_token_and_profile("user_id", "cookietoken") == {"foo": "bar"} mock_get_profile.assert_called_once_with("user_id", "cookietoken", "token", aux_headers={}) -@mock.patch(f"{MODULE}.urllib.request", autospec=True) +@mock.patch(f"{MODULE}.EdlClient", autospec=True) @mock.patch(f"{MODULE}.get_profile", autospec=True) @mock.patch(f"{MODULE}.get_urs_creds", autospec=True) -def test_get_new_token_and_profile_error(mock_get_urs_creds, mock_get_profile, mock_request): +def test_get_new_token_and_profile_error(mock_get_urs_creds, mock_get_profile, mock_client): mock_get_urs_creds.return_value = {"UrsAuth": "URS_AUTH"} - mock_request.urlopen.side_effect = urllib.error.URLError("test error") + mock_client().request.side_effect = EdlException( + urllib.error.URLError("test error"), + msg={}, + payload=None, + ) assert get_new_token_and_profile("user_id", "cookietoken") is None mock_get_profile.assert_not_called()