From d6c08cb523e7532d57ff56b8091bfca2da8dcea2 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 25 May 2021 16:19:49 +0100 Subject: [PATCH 01/17] ADDED configurable timeout for boto3 client --- ais_service_discovery/call_service.py | 20 +++++++++++++++----- ais_service_discovery/services/cloudmap.py | 5 +++-- examples.py | 13 +++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 examples.py diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index 39bef3a..651e179 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -1,3 +1,4 @@ +from os import environ as env from boto3 import client from .services import default_namespace, extract_service_parts, Services, \ CloudmapAdapter @@ -6,10 +7,18 @@ from .sqs import SqsAdaptor, Send from json import dumps -function = client('lambda') -sns = client('sns') -sqs = client('sqs') -service_discovery = client('servicediscovery') +from botocore.config import Config + + +BOTO_MAX_ATTEMPTS = env.get('BOTO_MAX_ATTEMPTS', 10) +BOTO_READ_TIMEOUT = env.get('BOTO_READ_TIMEOUT', 300) + +config = Config(read_timeout=BOTO_READ_TIMEOUT, retries={'max_attempts': BOTO_MAX_ATTEMPTS}) + +function = client('lambda', config=config) +sns = client('sns', config=config) +sqs = client('sqs', config=config) +service_discovery = client('servicediscovery', config=config) cloudmap_adaptor = CloudmapAdapter(service_discovery) services = Services(cloudmap_adaptor) @@ -48,7 +57,7 @@ def run_service(service, body, opts={}): def call_service(namespace, service, handler, body, opts={}): instances = services.discover( - namespace or default_namespace(), service, handler)['Instances'] + namespace or default_namespace(), service, handler)['instances'] if not instances: raise Exception('Service {}.{}.{} not found'.format( namespace or default_namespace(), service, handler)) @@ -58,3 +67,4 @@ def call_service(namespace, service, handler, body, opts={}): ('errorMessage' in payload) and ('errorMessage' in payload)): raise Exception(payload) return dumps(payload) + diff --git a/ais_service_discovery/services/cloudmap.py b/ais_service_discovery/services/cloudmap.py index 00c722a..91ca2a1 100644 --- a/ais_service_discovery/services/cloudmap.py +++ b/ais_service_discovery/services/cloudmap.py @@ -4,10 +4,10 @@ def __init__(self, client): def filter_instances(self, response, id): filter_function = lambda instance : instance['InstanceId'] == id - + return { **response, - 'Instances': filter(filter_function, response['Instances']) + 'instances': list(filter(filter_function, response['Instances'])) } def discover(self, namespace, name, id): @@ -15,3 +15,4 @@ def discover(self, namespace, name, id): NamespaceName=namespace, ServiceName=name ), id) + diff --git a/examples.py b/examples.py new file mode 100644 index 0000000..ab3b815 --- /dev/null +++ b/examples.py @@ -0,0 +1,13 @@ +from ais_service_discovery import call + +try: + res = call( + 'my-namespace', + 'my-service', + 'my-instance', { + 'arg': 'hello-word', + }) + print(res) +except Exception as e: + print(e) + From 36aec6fa3ff254b17b11d3a1545443f14e4c9c00 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 25 May 2021 16:28:13 +0100 Subject: [PATCH 02/17] UPDATED to leave instances key name as is --- ais_service_discovery/call_service.py | 2 +- ais_service_discovery/services/cloudmap.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index 651e179..b79aa81 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -57,7 +57,7 @@ def run_service(service, body, opts={}): def call_service(namespace, service, handler, body, opts={}): instances = services.discover( - namespace or default_namespace(), service, handler)['instances'] + namespace or default_namespace(), service, handler)['Instances'] if not instances: raise Exception('Service {}.{}.{} not found'.format( namespace or default_namespace(), service, handler)) diff --git a/ais_service_discovery/services/cloudmap.py b/ais_service_discovery/services/cloudmap.py index 91ca2a1..7967a7b 100644 --- a/ais_service_discovery/services/cloudmap.py +++ b/ais_service_discovery/services/cloudmap.py @@ -7,7 +7,7 @@ def filter_instances(self, response, id): return { **response, - 'instances': list(filter(filter_function, response['Instances'])) + 'Instances': list(filter(filter_function, response['Instances'])) } def discover(self, namespace, name, id): From 4f8fedbeb354e67c873925fe09842477ddb5ad21 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 25 May 2021 16:34:14 +0100 Subject: [PATCH 03/17] UPDATED timeout and retry as appropriate integers --- ais_service_discovery/call_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index b79aa81..9c0d808 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -10,8 +10,8 @@ from botocore.config import Config -BOTO_MAX_ATTEMPTS = env.get('BOTO_MAX_ATTEMPTS', 10) -BOTO_READ_TIMEOUT = env.get('BOTO_READ_TIMEOUT', 300) +BOTO_MAX_ATTEMPTS = int(env.get('BOTO_MAX_ATTEMPTS', 10)) +BOTO_READ_TIMEOUT = float(env.get('BOTO_READ_TIMEOUT', 300)) config = Config(read_timeout=BOTO_READ_TIMEOUT, retries={'max_attempts': BOTO_MAX_ATTEMPTS}) From 64673c2f41ae5623e48a1e954da4ece1cb6c5d9f Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 25 May 2021 16:37:52 +0100 Subject: [PATCH 04/17] UPDATED README --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ba096f7..5bf346c 100644 --- a/README.md +++ b/README.md @@ -18,15 +18,16 @@ This repository interfaces Service Discovery, in this instance CloudMap, in orde ## TODO -- Lambda (`request`). -- SQS (`listen`). - Http (`request`|`call`). - Fargate/ECS Task (`run`). -## Note: + +## Note + This library requires *Python 3.5 and above*. -## Examples: + +## Examples ### Lambda Call @@ -43,3 +44,13 @@ from ais_service_discovery import call response=call('namespace', 'service', 'handler', {}, {'InvocationType': 'Event'}) print(response) ``` + +## Configuration + +This library can utilise the following environment variables: + +``` +BOTO_MAX_ATTEMPTS=10 // Boto3 exponential back-off attemps +BOTO_READ_TIMEOUT=300 // Boto3 timeout +``` + From 267af76c6330ab73694032bb9cd0af0a303ffe80 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 26 May 2021 09:52:57 +0100 Subject: [PATCH 05/17] UPDATED README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5bf346c..b73cc94 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,10 @@ This repository interfaces Service Discovery, in this instance CloudMap, in orde ## TODO +- SQS (`listen`). - Http (`request`|`call`). - Fargate/ECS Task (`run`). +- Lambda (`request`). ## Note From aef80c828d9cf32c8e68422d2047f859925fce2f Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 26 May 2021 09:58:13 +0100 Subject: [PATCH 06/17] UPDATED total_max_retries config --- ais_service_discovery/call_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index 9c0d808..354fd82 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -10,10 +10,10 @@ from botocore.config import Config -BOTO_MAX_ATTEMPTS = int(env.get('BOTO_MAX_ATTEMPTS', 10)) -BOTO_READ_TIMEOUT = float(env.get('BOTO_READ_TIMEOUT', 300)) +BOTO_MAX_ATTEMPTS = int(env.get('BOTO_MAX_ATTEMPTS', 1)) +BOTO_READ_TIMEOUT = float(env.get('BOTO_READ_TIMEOUT', 5)) -config = Config(read_timeout=BOTO_READ_TIMEOUT, retries={'max_attempts': BOTO_MAX_ATTEMPTS}) +config = Config(read_timeout=BOTO_READ_TIMEOUT, retries={'total_max_attempts': BOTO_MAX_ATTEMPTS}) function = client('lambda', config=config) sns = client('sns', config=config) From 1536f677ebf45293098e272e50b6b0583defc6c9 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 26 May 2021 10:11:16 +0100 Subject: [PATCH 07/17] UPDATED to furnish a new release version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7c44f8..abd5dca 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='ais_service_discovery', - version='0.2.0', + version='0.2.1', author='Peak AI', author_email='infra-notifications@peak.ai', description='AIS service discovery package for python3', From 68aaebb0ca6ef442b3ac64725477e399bb001593 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Thu, 27 May 2021 09:39:43 +0100 Subject: [PATCH 08/17] UPDATED to be a release candidate --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abd5dca..cc0eddc 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='ais_service_discovery', - version='0.2.1', + version='0.2.1-rc1', author='Peak AI', author_email='infra-notifications@peak.ai', description='AIS service discovery package for python3', From 2a7434b82a33f4cb2acd8a56bc4aa5e0601f3340 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 20 Jul 2021 15:01:54 +0100 Subject: [PATCH 09/17] ADDED connection timeout FIXED a few linting errors --- ais_service_discovery/call_service.py | 35 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index 354fd82..0bfa3bf 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -1,19 +1,30 @@ +""" +call_service + +Calls services via the service discovery library +""" from os import environ as env +from json import dumps + from boto3 import client -from .services import default_namespace, extract_service_parts, Services, \ +from botocore.config import Config + +from .services import default_namespace, Services, \ CloudmapAdapter from .functions import LambdaAdaptor, Func from .sns import SnsAdaptor, Publish from .sqs import SqsAdaptor, Send -from json import dumps - -from botocore.config import Config BOTO_MAX_ATTEMPTS = int(env.get('BOTO_MAX_ATTEMPTS', 1)) BOTO_READ_TIMEOUT = float(env.get('BOTO_READ_TIMEOUT', 5)) +BOTO_CONNECTION_TIMEOUT = float(env.get('BOTO_CONNECTION_TIMEOUT', 60)) -config = Config(read_timeout=BOTO_READ_TIMEOUT, retries={'total_max_attempts': BOTO_MAX_ATTEMPTS}) +config = Config( + read_timeout=BOTO_READ_TIMEOUT, + retries={'total_max_attempts': BOTO_MAX_ATTEMPTS}, + connection_timeout=BOTO_CONNECTION_TIMEOUT +) function = client('lambda', config=config) sns = client('sns', config=config) @@ -32,9 +43,11 @@ sqs_adaptor = SqsAdaptor(sqs) send = Send(sqs_adaptor) -def run_service(service, body, opts={}): - type = service['Attributes']['type'] - if type in ['cloud-function', 'function', 'lambda']: + +def run_service(service, body, opts): + """ run_service """ + service_type = service['Attributes']['type'] + if service_type in ['cloud-function', 'function', 'lambda']: return func.call(**{ 'FunctionName': service['Attributes']['arn'], 'Payload': dumps(body), @@ -55,7 +68,8 @@ def run_service(service, body, opts={}): return None -def call_service(namespace, service, handler, body, opts={}): +def call_service(namespace, service, handler, body, opts): + """ call_service """ instances = services.discover( namespace or default_namespace(), service, handler)['Instances'] if not instances: @@ -63,8 +77,7 @@ def call_service(namespace, service, handler, body, opts={}): namespace or default_namespace(), service, handler)) [service_to_run] = instances payload = run_service(service_to_run, body, opts) - if (payload and type(payload) is dict and + if (payload and isinstance(payload) is dict and ('errorMessage' in payload) and ('errorMessage' in payload)): raise Exception(payload) return dumps(payload) - From 9877d0717992f3133969ebcc70471462b5ff1283 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 20 Jul 2021 15:04:02 +0100 Subject: [PATCH 10/17] UPDATED readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b73cc94..0b66930 100644 --- a/README.md +++ b/README.md @@ -54,5 +54,6 @@ This library can utilise the following environment variables: ``` BOTO_MAX_ATTEMPTS=10 // Boto3 exponential back-off attemps BOTO_READ_TIMEOUT=300 // Boto3 timeout +BOTO_CONNECTION_TIMEOUT=60 // Boto3 connection timeout ``` From 805e1968bde4119e8e866139d0325507f29a8c6b Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 20 Jul 2021 15:06:45 +0100 Subject: [PATCH 11/17] UPDATED version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cc0eddc..209e50e 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='ais_service_discovery', - version='0.2.1-rc1', + version='0.2.1-rc2', author='Peak AI', author_email='infra-notifications@peak.ai', description='AIS service discovery package for python3', From 91f9d8caddacf7ca8b95ca13cfddada83ed856a7 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Tue, 20 Jul 2021 15:20:18 +0100 Subject: [PATCH 12/17] UPDATED version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 209e50e..cef4525 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='ais_service_discovery', - version='0.2.1-rc2', + version='0.2.1-rc3', author='Peak AI', author_email='infra-notifications@peak.ai', description='AIS service discovery package for python3', From 741a86da77f49d979f8f233d523376a73dd33949 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 21 Jul 2021 08:47:40 +0100 Subject: [PATCH 13/17] Update ais_service_discovery/call_service.py Co-authored-by: Vishnu Agarwal --- ais_service_discovery/call_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index 0bfa3bf..61f7beb 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -23,7 +23,7 @@ config = Config( read_timeout=BOTO_READ_TIMEOUT, retries={'total_max_attempts': BOTO_MAX_ATTEMPTS}, - connection_timeout=BOTO_CONNECTION_TIMEOUT + connect_timeout=BOTO_CONNECTION_TIMEOUT ) function = client('lambda', config=config) From 21e857c24094fac496c01e2f6da07a622de008e0 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 21 Jul 2021 09:50:38 +0100 Subject: [PATCH 14/17] Update ais_service_discovery/call_service.py Co-authored-by: Vishnu Agarwal --- ais_service_discovery/call_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index 61f7beb..e15ebb5 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -44,7 +44,7 @@ send = Send(sqs_adaptor) -def run_service(service, body, opts): +def run_service(service, body, opts={}): """ run_service """ service_type = service['Attributes']['type'] if service_type in ['cloud-function', 'function', 'lambda']: From cea1c7ee73101206c6726a707d4b6f1f0871a6be Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 21 Jul 2021 09:50:43 +0100 Subject: [PATCH 15/17] Update ais_service_discovery/call_service.py Co-authored-by: Vishnu Agarwal --- ais_service_discovery/call_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index e15ebb5..24a050a 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -68,7 +68,7 @@ def run_service(service, body, opts={}): return None -def call_service(namespace, service, handler, body, opts): +def call_service(namespace, service, handler, body, opts={}): """ call_service """ instances = services.discover( namespace or default_namespace(), service, handler)['Instances'] From 7ec690e885152fd545d2542dce6ac14abda22506 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 21 Jul 2021 09:50:51 +0100 Subject: [PATCH 16/17] Update ais_service_discovery/call_service.py Co-authored-by: Vishnu Agarwal --- ais_service_discovery/call_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ais_service_discovery/call_service.py b/ais_service_discovery/call_service.py index 24a050a..4af3df1 100644 --- a/ais_service_discovery/call_service.py +++ b/ais_service_discovery/call_service.py @@ -77,7 +77,7 @@ def call_service(namespace, service, handler, body, opts={}): namespace or default_namespace(), service, handler)) [service_to_run] = instances payload = run_service(service_to_run, body, opts) - if (payload and isinstance(payload) is dict and + if (payload and isinstance(payload, dict) and ('errorMessage' in payload) and ('errorMessage' in payload)): raise Exception(payload) return dumps(payload) From 813411cdbfff7098958b243b6eaa75915618eb45 Mon Sep 17 00:00:00 2001 From: Ewan Valentine Date: Wed, 21 Jul 2021 09:55:43 +0100 Subject: [PATCH 17/17] UPDATED version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cef4525..cddee63 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='ais_service_discovery', - version='0.2.1-rc3', + version='0.2.1-rc7', author='Peak AI', author_email='infra-notifications@peak.ai', description='AIS service discovery package for python3',