From bd63b4bb366d6bd39668487dc46ba0a6397069f5 Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Mon, 20 Jan 2020 15:21:21 -0600 Subject: [PATCH 01/10] add containers to run tests and for test server --- docker/Dockerfile | 24 ++++++++++++++++++++++++ docker/startup.sh | 8 ++++++++ docker/tests/Dockerfile | 19 +++++++++++++++++++ docker/tests/run-tests.sh | 6 ++++++ nginx/nginx.conf | 27 +++++++++++++++++++++++++++ nginx/server.conf | 25 +++++++++++++++++++++++++ 6 files changed, 109 insertions(+) create mode 100644 docker/Dockerfile create mode 100755 docker/startup.sh create mode 100644 docker/tests/Dockerfile create mode 100755 docker/tests/run-tests.sh create mode 100644 nginx/nginx.conf create mode 100644 nginx/server.conf diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..90c8820 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:18.04 as build +RUN apt update -y &&\ + apt-get install -yf \ + build-essential \ + wget \ + git \ + libpcre3-dev \ + zlib1g-dev \ + libssl-dev +RUN wget https://nginx.org/download/nginx-1.16.1.tar.gz &&\ + tar -xzvf nginx-1.16.1.tar.gz +COPY . /ngx_http_aws_auth_module +WORKDIR /nginx-1.16.1 +RUN ./configure --with-compat --add-dynamic-module=../ngx_http_aws_auth_module &&\ + make modules + +FROM nginx:1.16.1-alpine +COPY nginx /etc/nginx +COPY nginx /tmp/nginx +COPY docker/startup.sh /startup.sh +COPY --from=build /nginx-1.16.1/objs/ngx_http_aws_auth_module.so /etc/nginx/modules +EXPOSE 80 +STOPSIGNAL SIGTERM +CMD [ "./startup.sh" ] \ No newline at end of file diff --git a/docker/startup.sh b/docker/startup.sh new file mode 100755 index 0000000..29121b8 --- /dev/null +++ b/docker/startup.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# substitute the domain and configure the first set of credentials +envsubst '${BUCKET},${BUCKET_DOMAIN},${_AWS_SIGNING_SCOPE},${_AWS_ACCESS_KEY},${_AWS_SIGNING_KEY}' /tmp/nginx/server-tmp.conf +cp /tmp/nginx/server-tmp.conf /etc/nginx/server.conf + +# start nginx (blocking call) +nginx -g "daemon off;" diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile new file mode 100644 index 0000000..d2cba3f --- /dev/null +++ b/docker/tests/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:18.04 as build +RUN apt update -y &&\ + apt-get install -yf \ + build-essential \ + wget \ + git \ + libpcre3-dev \ + zlib1g-dev \ + libssl-dev \ + cmake \ + sudo +RUN wget https://nginx.org/download/nginx-1.16.1.tar.gz &&\ + tar -xzvf nginx-1.16.1.tar.gz +RUN git clone git://git.cryptomilk.org/projects/cmocka.git /cmocka +COPY docker/tests/run-tests.sh /run-tests.sh +COPY . /ngx_http_aws_auth_module +ENV NGX_PATH=/nginx-1.16.1 +ENV LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib +CMD [ "./run-tests.sh" ] diff --git a/docker/tests/run-tests.sh b/docker/tests/run-tests.sh new file mode 100755 index 0000000..295229a --- /dev/null +++ b/docker/tests/run-tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd $NGX_PATH +./configure --with-http_ssl_module --with-compat --add-module=/ngx_http_aws_auth_module && make +cd /ngx_http_aws_auth_module +cp -r /cmocka vendor/ +make test \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..9db2f49 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,27 @@ +load_module modules/ngx_http_aws_auth_module.so; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + + keepalive_timeout 60; + + include server.conf; +} \ No newline at end of file diff --git a/nginx/server.conf b/nginx/server.conf new file mode 100644 index 0000000..37dff63 --- /dev/null +++ b/nginx/server.conf @@ -0,0 +1,25 @@ +server { + listen 80 default_server; + + location / { + + proxy_pass https://${BUCKET_DOMAIN}; + proxy_set_header Host ${BUCKET_DOMAIN}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + aws_sign; + aws_access_key ${_AWS_ACCESS_KEY}; + aws_key_scope ${_AWS_SIGNING_SCOPE}; + aws_signing_key ${_AWS_SIGNING_KEY}; + aws_s3_bucket ${BUCKET}; + + # add the cache status as a response header for debugging + add_header X-Cache-Status $upstream_cache_status; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} \ No newline at end of file From e1af38372882a900ed82019fd903d5c24d1b61df Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Mon, 20 Jan 2020 15:30:25 -0600 Subject: [PATCH 02/10] update reference implementation with security token --- reference-impl-py/reference_v4.py | 42 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/reference-impl-py/reference_v4.py b/reference-impl-py/reference_v4.py index c83f999..8e50958 100755 --- a/reference-impl-py/reference_v4.py +++ b/reference-impl-py/reference_v4.py @@ -55,11 +55,13 @@ def canon_querystring(qs_map): return {'cqs':'', 'qsmap':{}} # TODO: impl -def make_headers(req_time, bucket, aws_headers, content_hash): +def make_headers(req_time, bucket, aws_headers, content_hash, security_token): headers = [] headers.append(['x-amz-content-sha256', content_hash]) headers.append(['x-amz-date', req_time]) headers.append(['Host', '%s.s3.amazonaws.com' % (bucket)]) + if security_token: + headers.append(['x-amz-security-token', security_token]) hmap = {} for x in headers: @@ -77,10 +79,10 @@ def make_headers(req_time, bucket, aws_headers, content_hash): return {'hmap': hmap, 'sh': signed_headers, 'ch': canon_headers } -def canon_request(req_time, bucket, url, qs_map, aws_headers): +def canon_request(req_time, bucket, url, qs_map, token, aws_headers): qs = canon_querystring(qs_map) payload_hash = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' #hardcoded - header_info = make_headers(req_time, bucket, None, payload_hash) + header_info = make_headers(req_time, bucket, None, payload_hash, token) cr = "\n".join(('GET', url, qs['cqs'], header_info['ch'], header_info['sh'], payload_hash)) # hardcoded method print cr @@ -94,8 +96,8 @@ def sign_body(body=None): def get_scope(dt, region): return '%s/%s/s3/aws4_request' % (dt, region) -def str_to_sign_v4(req_time, scope, bucket, url, qs_map, aws_headers): - cr_info = canon_request(req_time, bucket, url, qs_map, aws_headers) +def str_to_sign_v4(req_time, scope, bucket, url, qs_map, token, aws_headers): + cr_info = canon_request(req_time, bucket, url, qs_map, token, aws_headers) h265 = sha256() h265.update(cr_info['cr_str']) hd = h265.hexdigest() @@ -103,8 +105,8 @@ def str_to_sign_v4(req_time, scope, bucket, url, qs_map, aws_headers): print s2s return {'s2s': s2s, 'headers': cr_info['headers'], 'qs':cr_info['qs'], 'scope': scope, 'sh': cr_info['sh']} -def sign(req_time, access_id, key, scope, bucket, url, qs_map, aws_headers): - s2s = str_to_sign_v4(req_time, scope, bucket, url, qs_map, aws_headers) +def sign(req_time, access_id, key, scope, bucket, url, qs_map, token, aws_headers): + s2s = str_to_sign_v4(req_time, scope, bucket, url, qs_map, token, aws_headers) retval = hmac.new(key, s2s['s2s'], sha256) sig = retval.hexdigest() auth_header = 'AWS4-HMAC-SHA256 Credential=%s/%s,SignedHeaders=%s,Signature=%s' % ( @@ -113,9 +115,9 @@ def sign(req_time, access_id, key, scope, bucket, url, qs_map, aws_headers): return {'headers': s2s['headers'], 'qs':s2s['qs'], 'sig': sig} -def get_data(req_time, access_id, key, scope, bucket, url, qs_map, aws_headers): - s = sign(req_time, access_id, key, scope, bucket, url, qs_map, aws_headers) - rurl = "http://%s.s3.amazonaws.com%s" % (bucket, url) +def get_data(req_time, access_id, key, scope, bucket, url, qs_map, token, aws_headers): + s = sign(req_time, access_id, key, scope, bucket, url, qs_map, token, aws_headers) + rurl = "https://%s.s3.amazonaws.com%s" % (bucket, url) # print rurl # print s q = Request(rurl) @@ -133,9 +135,12 @@ def get_data(req_time, access_id, key, scope, bucket, url, qs_map, aws_headers): aid = sys.argv[1] b64_key = sys.argv[2] scope = sys.argv[3] - request_time = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') if len(sys.argv) == 4 else sys.argv[4] + bucket = sys.argv[4] + path = sys.argv[5] + token = sys.argv[6] + request_time = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') if len(sys.argv) == 7 else sys.argv[7] print "Request time is %s" % request_time - get_data(request_time, aid, base64.b64decode(b64_key), scope, 'hw.anomalizer', '/lock.txt', {}, {}) + print get_data(request_time, aid, base64.b64decode(b64_key), scope, bucket, path, {}, token, {}) ''' class TestStringMethods(unittest.TestCase): @@ -144,8 +149,17 @@ def test_simple_get(self): key = base64.b64decode('k4EntTNoEN22pdavRF/KyeNx+e1BjtOGsCKu2CkBvnU=') aid = 'AKIDEXAMPLE' scope = '20150830/us-east-1/service/aws4_request' - s = sign(now, aid, key, scope, 'example', '/', {}, {}) - self.assertEqual(s['sig'], '5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31') + s = sign(now, aid, key, scope, 'example', '/', {}, None, {}) + self.assertEqual(s['sig'], '4ed4ec875ff02e55c7903339f4f24f8780b986a9cc9eff03f324d31da6a57690') + + def test_simple_get_with_security_token(self): + now = '20150830T123600Z' + key = base64.b64decode('k4EntTNoEN22pdavRF/KyeNx+e1BjtOGsCKu2CkBvnU=') + aid = 'AKIDEXAMPLE' + scope = '20150830/us-east-1/service/aws4_request' + token = 'IQoJb3JpZ2luX2VjEGsaCXVzLWVhc3QtMSJGMEQCID6TMGyw8dapyAyoqK7nRRsWfs2UcGZlNge6gD67WouHAiBbxqJ6X61HRCges6DWx538dZlZnGDRtKM1dUcIi1HllirzAwjz//////////8BEAAaDDg0NzEzMDIwNzA1MCIMQXKhCFqhuwfirKHmKscD2kA3ab0pQdqJFH7Q5X5XX5OaHyiHkwAeLNyUKK+vwafYgixxMZqVHxyeZNWkFWMPbiHfW4TVEeG6D2/jG1QGOwbLJqTdkvrJqUoLU5bfqxdYIGyDO14k6q39NCg0EpXen54uIwRrDgPQZenPDASZy+NKnNnOnQ3EbJgXFOlxAQWLcUwP5Oab0s4BxLZ4F7c2DcCMJLLCpfIr0s9sYXM3cv6rDac/agjazkIooe3JfXOSqKQK9CBLFfYqXh+/pg4VwDJ6Y64Db1imRDdXZr98okg6P6+IXerOYnw9LilKnlLSfP9A0Hx4zkMToGJeNZVLhvQXfK23Ohv4k3ZgxS8WNlvGtyh13j7xEpmCLL1MbAMXQin8Zx8hePNdfH0+oPrAEHKORmYhF7Npp97vi4fZn4rJb0wyR+tzk4BUwU8bxsqo2QdNXj3JdBCeJtbcFOTkR9VRDNFKuxcCJ4YyHwSXegpRg64D/+eNvXEai74BR0CMlXD7ixo25zM+1qhAO8wtsDRZkuLq08KkccWFMJ7mtd5hF3a44qUtjzRnW4Oirt6HAegaotLvMsWxhlKEm6THfPN0B3GqVN4dx8I2/hlcRCoA/ytapSkwutyX8QU68AETTkURQmWBx8MMe3+fdNc6o6b9TgXXxeCMEnTHwF3lFaQIzI3v+V4WHF7IEU3FiH8Qc489d64D48l71akbXN89nArzgsKXB2MmmV2lM9YeCOnsKjmX8KDM0SXiEL2zF3sXQ6cpwXdHRFLWdM5neZxBxT2NXoCh8Xjx2VEzTJ20vLfq0qS/1WmOvzxa1Z4B4GJUx9Gho/2iLHXvrBh93kk72KbzHP15ZsKixGkF4CP2qqluraym5Mv2IXV1vZhipVedNBFCngOR603MyERCw0tKnYXuduDnvEV0J9Hgf+fyeiXSXH34K5Fq525/XZDKMm4=' + s = sign(now, aid, key, scope, 'example', '/', {}, token, {}) + self.assertEqual(s['sig'], 'c0979d16460957b789c4b31048e6e008e3888666e227e749d1a0bc5d5d8ab175') if __name__ == '__main__': From e9bd1e690bac4aefdb166c5062fbaa990a57b6b0 Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Mon, 20 Jan 2020 15:31:21 -0600 Subject: [PATCH 03/10] add ability to optionally add a security token for the aws call --- aws_functions.h | 28 ++++++++++++++++++++-------- ngx_http_aws_auth.c | 11 ++++++++++- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/aws_functions.h b/aws_functions.h index deea525..cfc98b6 100644 --- a/aws_functions.h +++ b/aws_functions.h @@ -64,6 +64,7 @@ static const ngx_str_t EMPTY_STRING_SHA256 = ngx_string("e3b0c44298fc1c149afbf4c static const ngx_str_t EMPTY_STRING = ngx_null_string; static const ngx_str_t AMZ_HASH_HEADER = ngx_string("x-amz-content-sha256"); static const ngx_str_t AMZ_DATE_HEADER = ngx_string("x-amz-date"); +static const ngx_str_t AMZ_SECURITY_TOKEN_HEADER = ngx_string("x-amz-security-token"); static const ngx_str_t HOST_HEADER = ngx_string("host"); static const ngx_str_t AUTHZ_HEADER = ngx_string("authorization"); @@ -183,13 +184,14 @@ static inline struct AwsCanonicalHeaderDetails ngx_aws_auth__canonize_headers(ng const ngx_http_request_t *req, const ngx_str_t *s3_bucket, const ngx_str_t *amz_date, const ngx_str_t *content_hash, - const ngx_str_t *s3_endpoint) { + const ngx_str_t *s3_endpoint, + const ngx_str_t *security_token) { size_t header_names_size = 1, header_nameval_size = 1; size_t i, used; u_char *buf_progress; struct AwsCanonicalHeaderDetails retval; - ngx_array_t *settable_header_array = ngx_array_create(pool, 3, sizeof(header_pair_t)); + ngx_array_t *settable_header_array = ngx_array_create(pool, 4, sizeof(header_pair_t)); header_pair_t *header_ptr; header_ptr = ngx_array_push(settable_header_array); @@ -200,6 +202,12 @@ static inline struct AwsCanonicalHeaderDetails ngx_aws_auth__canonize_headers(ng header_ptr->key = AMZ_DATE_HEADER; header_ptr->value = *amz_date; + if (ngx_strncmp(security_token, &EMPTY_STRING, 1) != 0) { + header_ptr = ngx_array_push(settable_header_array); + header_ptr->key = AMZ_SECURITY_TOKEN_HEADER; + header_ptr->value = *security_token; + } + header_ptr = ngx_array_push(settable_header_array); header_ptr->key = HOST_HEADER; header_ptr->value.len = s3_bucket->len + 60; @@ -335,7 +343,8 @@ static inline const ngx_str_t* ngx_aws_auth__canon_url(ngx_pool_t *pool, const n static inline struct AwsCanonicalRequestDetails ngx_aws_auth__make_canonical_request(ngx_pool_t *pool, const ngx_http_request_t *req, - const ngx_str_t *s3_bucket_name, const ngx_str_t *amz_date, const ngx_str_t *s3_endpoint) { + const ngx_str_t *s3_bucket_name, const ngx_str_t *amz_date, + const ngx_str_t *s3_endpoint, const ngx_str_t *security_token) { struct AwsCanonicalRequestDetails retval; // canonize query string @@ -345,7 +354,7 @@ static inline struct AwsCanonicalRequestDetails ngx_aws_auth__make_canonical_req const ngx_str_t *request_body_hash = ngx_aws_auth__request_body_hash(pool, req); const struct AwsCanonicalHeaderDetails canon_headers = - ngx_aws_auth__canonize_headers(pool, req, s3_bucket_name, amz_date, request_body_hash, s3_endpoint); + ngx_aws_auth__canonize_headers(pool, req, s3_bucket_name, amz_date, request_body_hash, s3_endpoint, security_token); retval.signed_header_names = canon_headers.signed_header_names; const ngx_str_t *http_method = &(req->method_name); @@ -397,12 +406,13 @@ static inline struct AwsSignedRequestDetails ngx_aws_auth__compute_signature(ngx const ngx_str_t *signing_key, const ngx_str_t *key_scope, const ngx_str_t *s3_bucket_name, - const ngx_str_t *s3_endpoint) { + const ngx_str_t *s3_endpoint, + const ngx_str_t *security_token) { struct AwsSignedRequestDetails retval; const ngx_str_t *date = ngx_aws_auth__compute_request_time(pool, &req->start_sec); const struct AwsCanonicalRequestDetails canon_request = - ngx_aws_auth__make_canonical_request(pool, req, s3_bucket_name, date, s3_endpoint); + ngx_aws_auth__make_canonical_request(pool, req, s3_bucket_name, date, s3_endpoint, security_token); const ngx_str_t *canon_request_hash = ngx_aws_auth__hash_sha256(pool, canon_request.canon_request); // get string to sign @@ -424,8 +434,10 @@ static inline const ngx_array_t* ngx_aws_auth__sign(ngx_pool_t *pool, ngx_http_r const ngx_str_t *signing_key, const ngx_str_t *key_scope, const ngx_str_t *s3_bucket_name, - const ngx_str_t *s3_endpoint) { - const struct AwsSignedRequestDetails signature_details = ngx_aws_auth__compute_signature(pool, req, signing_key, key_scope, s3_bucket_name, s3_endpoint); + const ngx_str_t *s3_endpoint, + const ngx_str_t *security_token) { + const struct AwsSignedRequestDetails signature_details = + ngx_aws_auth__compute_signature(pool, req, signing_key, key_scope, s3_bucket_name, s3_endpoint, security_token); const ngx_str_t *auth_header_value = ngx_aws_auth__make_auth_token(pool, signature_details.signature, diff --git a/ngx_http_aws_auth.c b/ngx_http_aws_auth.c index bbd486d..d166045 100644 --- a/ngx_http_aws_auth.c +++ b/ngx_http_aws_auth.c @@ -19,6 +19,7 @@ typedef struct { ngx_str_t signing_key_decoded; ngx_str_t endpoint; ngx_str_t bucket_name; + ngx_str_t security_token; ngx_uint_t enabled; } ngx_http_aws_auth_conf_t; @@ -59,6 +60,13 @@ static ngx_command_t ngx_http_aws_auth_commands[] = { offsetof(ngx_http_aws_auth_conf_t, bucket_name), NULL }, + { ngx_string("aws_security_token"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_aws_auth_conf_t, security_token), + NULL }, + { ngx_string("aws_sign"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_NOARGS, ngx_http_aws_sign, @@ -125,6 +133,7 @@ ngx_http_aws_auth_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_str_value(conf->signing_key, prev->signing_key, ""); ngx_conf_merge_str_value(conf->endpoint, prev->endpoint, "s3.amazonaws.com"); ngx_conf_merge_str_value(conf->bucket_name, prev->bucket_name, ""); + ngx_conf_merge_str_value(conf->security_token, prev->security_token, ""); if(conf->signing_key_decoded.data == NULL) { @@ -161,7 +170,7 @@ ngx_http_aws_proxy_sign(ngx_http_request_t *r) } const ngx_array_t* headers_out = ngx_aws_auth__sign(r->pool, r, - &conf->access_key, &conf->signing_key_decoded, &conf->key_scope, &conf->bucket_name, &conf->endpoint); + &conf->access_key, &conf->signing_key_decoded, &conf->key_scope, &conf->bucket_name, &conf->endpoint, &conf->security_token); ngx_uint_t i; for(i = 0; i < headers_out->nelts; i++) From f0cab7ffcf698df651f5bf2aafffc9e0bd3dac15 Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Mon, 20 Jan 2020 15:31:52 -0600 Subject: [PATCH 04/10] add tests for the session token --- test_it.sh | 2 + test_suite.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 107 insertions(+), 8 deletions(-) create mode 100755 test_it.sh diff --git a/test_it.sh b/test_it.sh new file mode 100755 index 0000000..3efa732 --- /dev/null +++ b/test_it.sh @@ -0,0 +1,2 @@ +docker build . -f docker/tests/Dockerfile -t ngx_aws_auth_tests +docker run --rm --name ngx_aws_auth_tests ngx_aws_auth_tests \ No newline at end of file diff --git a/test_suite.c b/test_suite.c index d9a2ec2..d1a1f97 100644 --- a/test_suite.c +++ b/test_suite.c @@ -91,20 +91,21 @@ static void sha256(void **state) { static void canon_header_string(void **state) { (void) state; /* unused */ - ngx_str_t bucket, date, hash, endpoint; + ngx_str_t bucket, date, hash, endpoint, token; struct AwsCanonicalHeaderDetails retval; bucket.data = "bugait"; bucket.len = 6; date.data = "20160221T063112Z"; date.len = 16; hash.data = "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"; hash.len = 64; endpoint.data = "s3.amazonaws.com"; endpoint.len = 16; + token.data = ""; token.len=0; - retval = ngx_aws_auth__canonize_headers(pool, NULL, &bucket, &date, &hash, &endpoint); + retval = ngx_aws_auth__canonize_headers(pool, NULL, &bucket, &date, &hash, &endpoint, &token); assert_string_equal(retval.canon_header_str->data, "host:bugait.s3.amazonaws.com\nx-amz-content-sha256:f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b\nx-amz-date:20160221T063112Z\n"); } -static void signed_headers(void **state) { +static void canon_header_string_with_security_token(void **state) { (void) state; /* unused */ ngx_str_t bucket, date, hash, endpoint; @@ -114,11 +115,45 @@ static void signed_headers(void **state) { date.data = "20160221T063112Z"; date.len = 16; hash.data = "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"; hash.len = 64; endpoint.data = "s3.amazonaws.com"; endpoint.len = 16; + ngx_str_t token = ngx_string("FwoGZXIvYXdzEGIaDGSJdkH/F9YHt9L5GiKsAewV1KBD2uklClV8PHR7yW9cPh9LiqSsJGx0yZF15enXMwsOUqgIbxj0ok7i4uML4P+EabLAvLPmW2Nmvax+h8kITdit0eABAvlE6yJLi2+din9xevrKOB+Q/wM1YDAiR1LaC4JZj2TQj9nzSIQ2rLwq/8qwxnBrVdekzh3ld8eKJG3BUKWJEXYE/XScaYB/nOY6gH2tsixksfbfb+e0cqLkPCXv21DOSLLejYAonoWS8QUyLRVTzPmq6av4/UNb6vm2GpPQxTP+PW8aH6UHLDWtn1EM9qe6ot3uLjDV+3spbg=="); + + retval = ngx_aws_auth__canonize_headers(pool, NULL, &bucket, &date, &hash, &endpoint, &token); + assert_string_equal(retval.canon_header_str->data, + "host:bugait.s3.amazonaws.com\nx-amz-content-sha256:f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b\nx-amz-date:20160221T063112Z\nx-amz-security-token:FwoGZXIvYXdzEGIaDGSJdkH/F9YHt9L5GiKsAewV1KBD2uklClV8PHR7yW9cPh9LiqSsJGx0yZF15enXMwsOUqgIbxj0ok7i4uML4P+EabLAvLPmW2Nmvax+h8kITdit0eABAvlE6yJLi2+din9xevrKOB+Q/wM1YDAiR1LaC4JZj2TQj9nzSIQ2rLwq/8qwxnBrVdekzh3ld8eKJG3BUKWJEXYE/XScaYB/nOY6gH2tsixksfbfb+e0cqLkPCXv21DOSLLejYAonoWS8QUyLRVTzPmq6av4/UNb6vm2GpPQxTP+PW8aH6UHLDWtn1EM9qe6ot3uLjDV+3spbg==\n"); +} + +static void signed_headers(void **state) { + (void) state; /* unused */ + + ngx_str_t bucket, date, hash, endpoint, token; + struct AwsCanonicalHeaderDetails retval; - retval = ngx_aws_auth__canonize_headers(pool, NULL, &bucket, &date, &hash, &endpoint); + bucket.data = "bugait"; bucket.len = 6; + date.data = "20160221T063112Z"; date.len = 16; + hash.data = "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"; hash.len = 64; + endpoint.data = "s3.amazonaws.com"; endpoint.len = 16; + token.data = ""; token.len = 0; + + retval = ngx_aws_auth__canonize_headers(pool, NULL, &bucket, &date, &hash, &endpoint, &token); assert_string_equal(retval.signed_header_names->data, "host;x-amz-content-sha256;x-amz-date"); } +static void signed_headers_with_security_token(void **state) { + (void) state; /* unused */ + + ngx_str_t bucket, date, hash, endpoint; + struct AwsCanonicalHeaderDetails retval; + + bucket.data = "bugait"; bucket.len = 6; + date.data = "20160221T063112Z"; date.len = 16; + hash.data = "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"; hash.len = 64; + endpoint.data = "s3.amazonaws.com"; endpoint.len = 16; + ngx_str_t token = ngx_string("FwoGZXIvYXdzEGIaDGSJdkH/F9YHt9L5GiKsAewV1KBD2uklClV8PHR7yW9cPh9LiqSsJGx0yZF15enXMwsOUqgIbxj0ok7i4uML4P+EabLAvLPmW2Nmvax+h8kITdit0eABAvlE6yJLi2+din9xevrKOB+Q/wM1YDAiR1LaC4JZj2TQj9nzSIQ2rLwq/8qwxnBrVdekzh3ld8eKJG3BUKWJEXYE/XScaYB/nOY6gH2tsixksfbfb+e0cqLkPCXv21DOSLLejYAonoWS8QUyLRVTzPmq6av4/UNb6vm2GpPQxTP+PW8aH6UHLDWtn1EM9qe6ot3uLjDV+3spbg=="); + + retval = ngx_aws_auth__canonize_headers(pool, NULL, &bucket, &date, &hash, &endpoint, &token); + assert_string_equal(retval.signed_header_names->data, "host;x-amz-content-sha256;x-amz-date;x-amz-security-token"); +} + static void canonical_qs_empty(void **state) { (void) state; /* unused */ ngx_http_request_t request; @@ -226,7 +261,8 @@ static void canonical_request_sans_qs(void **state) { const ngx_str_t aws_date = ngx_string("20160221T063112Z"); const ngx_str_t url = ngx_string("/"); const ngx_str_t method = ngx_string("GET"); - const ngx_str_t endpoint = ngx_string("s3.amazonaws.com"); + const ngx_str_t endpoint = ngx_string("s3.amazonaws.com"); + const ngx_str_t token = ngx_string(""); struct AwsCanonicalRequestDetails result; ngx_http_request_t request; @@ -236,7 +272,7 @@ static void canonical_request_sans_qs(void **state) { request.args = EMPTY_STRING; request.connection = NULL; - result = ngx_aws_auth__make_canonical_request(pool, &request, &bucket, &aws_date, &endpoint); + result = ngx_aws_auth__make_canonical_request(pool, &request, &bucket, &aws_date, &endpoint, &token); assert_string_equal(result.canon_request->data, "GET\n\ /\n\ \n\ @@ -248,6 +284,35 @@ host;x-amz-content-sha256;x-amz-date\n\ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); } +static void canonical_request_sans_qs_with_security_token(void **state) { + (void) state; /* unused */ + const ngx_str_t bucket = ngx_string("example"); + const ngx_str_t aws_date = ngx_string("20160221T063112Z"); + const ngx_str_t url = ngx_string("/"); + const ngx_str_t method = ngx_string("GET"); + const ngx_str_t endpoint = ngx_string("s3.amazonaws.com"); + const ngx_str_t token = ngx_string("FwoGZXIvYXdzEGIaDGSJdkH/F9YHt9L5GiKsAewV1KBD2uklClV8PHR7yW9cPh9LiqSsJGx0yZF15enXMwsOUqgIbxj0ok7i4uML4P+EabLAvLPmW2Nmvax+h8kITdit0eABAvlE6yJLi2+din9xevrKOB+Q/wM1YDAiR1LaC4JZj2TQj9nzSIQ2rLwq/8qwxnBrVdekzh3ld8eKJG3BUKWJEXYE/XScaYB/nOY6gH2tsixksfbfb+e0cqLkPCXv21DOSLLejYAonoWS8QUyLRVTzPmq6av4/UNb6vm2GpPQxTP+PW8aH6UHLDWtn1EM9qe6ot3uLjDV+3spbg=="); + + struct AwsCanonicalRequestDetails result; + ngx_http_request_t request; + + request.uri = url; + request.method_name = method; + request.args = EMPTY_STRING; + request.connection = NULL; + + result = ngx_aws_auth__make_canonical_request(pool, &request, &bucket, &aws_date, &endpoint, &token); + assert_string_equal(result.canon_request->data, "GET\n\ +/\n\ +\n\ +host:example.s3.amazonaws.com\n\ +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n\ +x-amz-date:20160221T063112Z\n\ +\n\ +host;x-amz-content-sha256;x-amz-date;x-amz-security-token\n\ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); +} + static void basic_get_signature(void **state) { (void) state; /* unused */ @@ -255,7 +320,8 @@ static void basic_get_signature(void **state) { const ngx_str_t method = ngx_string("GET"); const ngx_str_t key_scope = ngx_string("20150830/us-east-1/service/aws4_request"); const ngx_str_t bucket = ngx_string("example"); - const ngx_str_t endpoint = ngx_string("s3.amazonaws.com"); + const ngx_str_t endpoint = ngx_string("s3.amazonaws.com"); + const ngx_str_t security_token = ngx_string(""); ngx_str_t signing_key, signing_key_b64e = ngx_string("k4EntTNoEN22pdavRF/KyeNx+e1BjtOGsCKu2CkBvnU="); ngx_http_request_t request; @@ -271,10 +337,38 @@ static void basic_get_signature(void **state) { ngx_decode_base64(&signing_key, &signing_key_b64e); struct AwsSignedRequestDetails result = ngx_aws_auth__compute_signature(pool, &request, - &signing_key, &key_scope, &bucket, &endpoint); + &signing_key, &key_scope, &bucket, &endpoint, &security_token); assert_string_equal(result.signature->data, "4ed4ec875ff02e55c7903339f4f24f8780b986a9cc9eff03f324d31da6a57690"); } +static void basic_get_signature_with_security_token(void **state) { + (void) state; /* unused */ + + const ngx_str_t url = ngx_string("/"); + const ngx_str_t method = ngx_string("GET"); + const ngx_str_t key_scope = ngx_string("20150830/us-east-1/service/aws4_request"); + const ngx_str_t bucket = ngx_string("example"); + const ngx_str_t endpoint = ngx_string("s3.amazonaws.com"); + const ngx_str_t security_token = ngx_string("IQoJb3JpZ2luX2VjEGsaCXVzLWVhc3QtMSJGMEQCID6TMGyw8dapyAyoqK7nRRsWfs2UcGZlNge6gD67WouHAiBbxqJ6X61HRCges6DWx538dZlZnGDRtKM1dUcIi1HllirzAwjz//////////8BEAAaDDg0NzEzMDIwNzA1MCIMQXKhCFqhuwfirKHmKscD2kA3ab0pQdqJFH7Q5X5XX5OaHyiHkwAeLNyUKK+vwafYgixxMZqVHxyeZNWkFWMPbiHfW4TVEeG6D2/jG1QGOwbLJqTdkvrJqUoLU5bfqxdYIGyDO14k6q39NCg0EpXen54uIwRrDgPQZenPDASZy+NKnNnOnQ3EbJgXFOlxAQWLcUwP5Oab0s4BxLZ4F7c2DcCMJLLCpfIr0s9sYXM3cv6rDac/agjazkIooe3JfXOSqKQK9CBLFfYqXh+/pg4VwDJ6Y64Db1imRDdXZr98okg6P6+IXerOYnw9LilKnlLSfP9A0Hx4zkMToGJeNZVLhvQXfK23Ohv4k3ZgxS8WNlvGtyh13j7xEpmCLL1MbAMXQin8Zx8hePNdfH0+oPrAEHKORmYhF7Npp97vi4fZn4rJb0wyR+tzk4BUwU8bxsqo2QdNXj3JdBCeJtbcFOTkR9VRDNFKuxcCJ4YyHwSXegpRg64D/+eNvXEai74BR0CMlXD7ixo25zM+1qhAO8wtsDRZkuLq08KkccWFMJ7mtd5hF3a44qUtjzRnW4Oirt6HAegaotLvMsWxhlKEm6THfPN0B3GqVN4dx8I2/hlcRCoA/ytapSkwutyX8QU68AETTkURQmWBx8MMe3+fdNc6o6b9TgXXxeCMEnTHwF3lFaQIzI3v+V4WHF7IEU3FiH8Qc489d64D48l71akbXN89nArzgsKXB2MmmV2lM9YeCOnsKjmX8KDM0SXiEL2zF3sXQ6cpwXdHRFLWdM5neZxBxT2NXoCh8Xjx2VEzTJ20vLfq0qS/1WmOvzxa1Z4B4GJUx9Gho/2iLHXvrBh93kk72KbzHP15ZsKixGkF4CP2qqluraym5Mv2IXV1vZhipVedNBFCngOR603MyERCw0tKnYXuduDnvEV0J9Hgf+fyeiXSXH34K5Fq525/XZDKMm4="); + + ngx_str_t signing_key, signing_key_b64e = ngx_string("k4EntTNoEN22pdavRF/KyeNx+e1BjtOGsCKu2CkBvnU="); + ngx_http_request_t request; + + request.start_sec = 1440938160; /* 20150830T123600Z */ + request.uri = url; + request.method_name = method; + request.args = EMPTY_STRING; + request.connection = NULL; + + signing_key.len = 64; + signing_key.data = ngx_palloc(pool, signing_key.len ); + ngx_decode_base64(&signing_key, &signing_key_b64e); + + struct AwsSignedRequestDetails result = ngx_aws_auth__compute_signature(pool, &request, + &signing_key, &key_scope, &bucket, &endpoint, &security_token); + assert_string_equal(result.signature->data, "c0979d16460957b789c4b31048e6e008e3888666e227e749d1a0bc5d5d8ab175"); +} + int main() { const struct CMUnitTest tests[] = { cmocka_unit_test(null_test_success), @@ -283,6 +377,7 @@ int main() { cmocka_unit_test(hmac_sha256), cmocka_unit_test(sha256), cmocka_unit_test(canon_header_string), + cmocka_unit_test(canon_header_string_with_security_token), cmocka_unit_test(canonical_qs_empty), cmocka_unit_test(canonical_qs_single_arg), cmocka_unit_test(canonical_qs_two_arg_reverse), @@ -291,8 +386,10 @@ int main() { cmocka_unit_test(canonical_url_with_qs), cmocka_unit_test(canonical_url_with_special_chars), cmocka_unit_test(signed_headers), + cmocka_unit_test(signed_headers_with_security_token), cmocka_unit_test(canonical_request_sans_qs), cmocka_unit_test(basic_get_signature), + cmocka_unit_test(basic_get_signature_with_security_token), }; pool = ngx_create_pool(1000000, NULL); From 4b168691c954b4fa2c98cc3c18941b0c687c6cf7 Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Mon, 20 Jan 2020 16:41:23 -0600 Subject: [PATCH 05/10] add security token to nginx conf docker --- docker/startup.sh | 2 +- nginx/server.conf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/startup.sh b/docker/startup.sh index 29121b8..094a5dc 100755 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -1,7 +1,7 @@ #!/bin/sh # substitute the domain and configure the first set of credentials -envsubst '${BUCKET},${BUCKET_DOMAIN},${_AWS_SIGNING_SCOPE},${_AWS_ACCESS_KEY},${_AWS_SIGNING_KEY}' /tmp/nginx/server-tmp.conf +envsubst '${BUCKET},${BUCKET_DOMAIN},${_AWS_SIGNING_SCOPE},${_AWS_ACCESS_KEY},${_AWS_SIGNING_KEY},${TOKEN}' /tmp/nginx/server-tmp.conf cp /tmp/nginx/server-tmp.conf /etc/nginx/server.conf # start nginx (blocking call) diff --git a/nginx/server.conf b/nginx/server.conf index 37dff63..01875ad 100644 --- a/nginx/server.conf +++ b/nginx/server.conf @@ -13,6 +13,7 @@ server { aws_key_scope ${_AWS_SIGNING_SCOPE}; aws_signing_key ${_AWS_SIGNING_KEY}; aws_s3_bucket ${BUCKET}; + aws_security_token ${TOKEN}; # add the cache status as a response header for debugging add_header X-Cache-Status $upstream_cache_status; From 0242b4893971580341dadf1f6f03155978daca86 Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Tue, 28 Jan 2020 10:14:01 -0600 Subject: [PATCH 06/10] reorganize files to an example of using this with token refresh; move docker tests into makefile --- Makefile | 3 +++ {docker => example/token-auto-refresh-python}/Dockerfile | 0 {nginx => example/token-auto-refresh-python/nginx}/nginx.conf | 0 {nginx => example/token-auto-refresh-python/nginx}/server.conf | 0 {docker => example/token-auto-refresh-python}/startup.sh | 0 test_it.sh | 2 -- 6 files changed, 3 insertions(+), 2 deletions(-) rename {docker => example/token-auto-refresh-python}/Dockerfile (100%) rename {nginx => example/token-auto-refresh-python/nginx}/nginx.conf (100%) rename {nginx => example/token-auto-refresh-python/nginx}/server.conf (100%) rename {docker => example/token-auto-refresh-python}/startup.sh (100%) delete mode 100755 test_it.sh diff --git a/Makefile b/Makefile index 9b95ad1..0f25f2d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ CC=gcc CFLAGS=-g -I${NGX_PATH}/src/os/unix -I${NGX_PATH}/src/core -I${NGX_PATH}/src/http -I${NGX_PATH}/src/http/modules -I${NGX_PATH}/src/event -I${NGX_PATH}/objs/ -I. +docker-test: + docker build . -f docker/tests/Dockerfile -t ngx_aws_auth_tests \ + && docker run --rm --name ngx_aws_auth_tests ngx_aws_auth_tests all: diff --git a/docker/Dockerfile b/example/token-auto-refresh-python/Dockerfile similarity index 100% rename from docker/Dockerfile rename to example/token-auto-refresh-python/Dockerfile diff --git a/nginx/nginx.conf b/example/token-auto-refresh-python/nginx/nginx.conf similarity index 100% rename from nginx/nginx.conf rename to example/token-auto-refresh-python/nginx/nginx.conf diff --git a/nginx/server.conf b/example/token-auto-refresh-python/nginx/server.conf similarity index 100% rename from nginx/server.conf rename to example/token-auto-refresh-python/nginx/server.conf diff --git a/docker/startup.sh b/example/token-auto-refresh-python/startup.sh similarity index 100% rename from docker/startup.sh rename to example/token-auto-refresh-python/startup.sh diff --git a/test_it.sh b/test_it.sh deleted file mode 100755 index 3efa732..0000000 --- a/test_it.sh +++ /dev/null @@ -1,2 +0,0 @@ -docker build . -f docker/tests/Dockerfile -t ngx_aws_auth_tests -docker run --rm --name ngx_aws_auth_tests ngx_aws_auth_tests \ No newline at end of file From 38913bc0a88ede69e9e73a04e7c3f009c8346261 Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Tue, 28 Jan 2020 11:31:42 -0600 Subject: [PATCH 07/10] allow logs to be disabled by passing -D NO_AWS_AUTH_LOGS flag --- aws_functions.h | 11 ++++++++++- example/token-auto-refresh-python/Dockerfile | 8 ++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/aws_functions.h b/aws_functions.h index cfc98b6..b7f95fa 100644 --- a/aws_functions.h +++ b/aws_functions.h @@ -55,10 +55,14 @@ struct AwsSignedRequestDetails { // mainly useful to avoid having to full instantiate request structures for // tests... +#ifndef NO_AWS_AUTH_LOGS #define safe_ngx_log_error(req, ...) \ if (req->connection) { \ - ngx_log_error(NGX_LOG_ERR, req->connection->log, 0, __VA_ARGS__); \ + ngx_log_error(NGX_LOG_ERR, req->connection->log, 0, __VA_ARGS__); \ } +#else +#define safe_ngx_log_error(req, ...) +#endif static const ngx_str_t EMPTY_STRING_SHA256 = ngx_string("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); static const ngx_str_t EMPTY_STRING = ngx_null_string; @@ -71,6 +75,11 @@ static const ngx_str_t AUTHZ_HEADER = ngx_string("authorization"); static inline char* __CHAR_PTR_U(u_char* ptr) {return (char*)ptr;} static inline const char* __CONST_CHAR_PTR_U(const u_char* ptr) {return (const char*)ptr;} +static inline void ngx_conditional_log(const ngx_http_request_t *req, ...) { + #ifndef NO_AWS_AUTH_LOGS + #endif +} + static inline const ngx_str_t* ngx_aws_auth__compute_request_time(ngx_pool_t *pool, const time_t *timep) { ngx_str_t *const retval = ngx_palloc(pool, sizeof(ngx_str_t)); retval->data = ngx_palloc(pool, AMZ_DATE_MAX_LEN); diff --git a/example/token-auto-refresh-python/Dockerfile b/example/token-auto-refresh-python/Dockerfile index 90c8820..3add31f 100644 --- a/example/token-auto-refresh-python/Dockerfile +++ b/example/token-auto-refresh-python/Dockerfile @@ -11,13 +11,13 @@ RUN wget https://nginx.org/download/nginx-1.16.1.tar.gz &&\ tar -xzvf nginx-1.16.1.tar.gz COPY . /ngx_http_aws_auth_module WORKDIR /nginx-1.16.1 -RUN ./configure --with-compat --add-dynamic-module=../ngx_http_aws_auth_module &&\ +RUN ./configure --with-compat --add-dynamic-module=../ngx_http_aws_auth_module --with-cc-opt="-D NO_AWS_AUTH_LOGS" &&\ make modules FROM nginx:1.16.1-alpine -COPY nginx /etc/nginx -COPY nginx /tmp/nginx -COPY docker/startup.sh /startup.sh +COPY example/token-auto-refresh-python/nginx /etc/nginx +COPY example/token-auto-refresh-python/nginx /tmp/nginx +COPY example/token-auto-refresh-python/startup.sh /startup.sh COPY --from=build /nginx-1.16.1/objs/ngx_http_aws_auth_module.so /etc/nginx/modules EXPOSE 80 STOPSIGNAL SIGTERM From 5a412ab05ecfd6853e0c82d50f8e632e3cfe5f4e Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Tue, 28 Jan 2020 12:42:12 -0600 Subject: [PATCH 08/10] add python credential refresh script --- example/token-auto-refresh-python/Dockerfile | 8 ++ .../refresh_credentials.py | 122 ++++++++++++++++++ example/token-auto-refresh-python/startup.sh | 8 +- 3 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 example/token-auto-refresh-python/refresh_credentials.py diff --git a/example/token-auto-refresh-python/Dockerfile b/example/token-auto-refresh-python/Dockerfile index 3add31f..6646f62 100644 --- a/example/token-auto-refresh-python/Dockerfile +++ b/example/token-auto-refresh-python/Dockerfile @@ -15,10 +15,18 @@ RUN ./configure --with-compat --add-dynamic-module=../ngx_http_aws_auth_module - make modules FROM nginx:1.16.1-alpine +RUN apk add --update \ + python3 \ + curl +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py &&\ + python3 get-pip.py &&\ + pip install boto3 &&\ + pip install watchtower COPY example/token-auto-refresh-python/nginx /etc/nginx COPY example/token-auto-refresh-python/nginx /tmp/nginx COPY example/token-auto-refresh-python/startup.sh /startup.sh COPY --from=build /nginx-1.16.1/objs/ngx_http_aws_auth_module.so /etc/nginx/modules +COPY example/token-auto-refresh/refresh_credentials.py /refresh_credentials.py EXPOSE 80 STOPSIGNAL SIGTERM CMD [ "./startup.sh" ] \ No newline at end of file diff --git a/example/token-auto-refresh-python/refresh_credentials.py b/example/token-auto-refresh-python/refresh_credentials.py new file mode 100644 index 0000000..9e0e2e3 --- /dev/null +++ b/example/token-auto-refresh-python/refresh_credentials.py @@ -0,0 +1,122 @@ +#!/usr/bin/python3 +import os +import http.client +import json +import hmac +import hashlib +import base64 +import subprocess +from datetime import datetime +import io +import logging +import sys + +level = logging.DEBUG if '-d' in sys.argv or '--debug' in sys.argv else logging.INFO +logging.basicConfig(stream=sys.stdout, level=level, format='%(asctime)s - %(name)s - %(levelname)s\n%(message)s') + +credentials_host = '169.254.170.2' +credentials_path = os.environ['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] +aws_region = os.environ['AWS_REGION'] +aws_bucket_domain = os.environ['BUCKET_DOMAIN'] +aws_bucket = aws_bucket_domain.split('.')[0] +aws_service = 's3' + +def get_timestamps(): + if os.path.exists('/etc/timestamps'): + with open('/etc/timestamps') as fp: + exp = fp.readline().strip() + created = fp.readline().strip() + + exp = datetime.strptime(exp, '%Y-%m-%dT%H:%M:%SZ') + created = datetime.strptime(created, '%Y-%m-%dT%H:%M:%SZ') + + return (exp, created) + return (None, None) + +def get_current_creds(): + try: + conn = http.client.HTTPConnection(credentials_host, timeout=5) + conn.request('GET', credentials_path) + res = conn.getresponse() + + creds = json.loads(res.readline().decode('utf8')) + except: + logging.error("Unexpected error: %s" % (sys.exc_info()[0])) + raise + + return (creds['AccessKeyId'], creds['SecretAccessKey'], creds['Token'], creds['Expiration']) + +def sign(key, val): + return hmac.new(key, val.encode('utf-8'), hashlib.sha256).digest() + +def get_signature_key(key, dateStamp, regionName, serviceName): + kDate = sign(("AWS4" + key).encode("utf-8"), dateStamp) + kRegion = sign(kDate, regionName) + kService = sign(kRegion, serviceName) + kSigning = sign(kService, "aws4_request") + return kSigning + +def encode_signature(signature): + return base64.b64encode(signature).decode('ascii') + +def get_key_scope(ymd, region, service): + return '%s/%s/%s/aws4_request' % (ymd, region, service) + +def refresh_credentials(access_key, secret, security_token, signing_key, scope, expiration): + + def export_credentials(): + os.environ['_AWS_ACCESS_KEY'] = access_key + os.environ['_AWS_SIGNING_KEY'] = signing_key + os.environ['_AWS_SIGNING_SCOPE'] = scope + os.environ['_AWS_SECURITY_TOKEN'] = security_token + os.environ['_AWS_BUCKET'] = aws_bucket + os.environ['_AWS_BUCKET_DOMAIN'] = aws_bucket_domain + with open('/etc/timestamps', 'w') as fp: + fp.write('%s\n%s\n' % (expiration, datetime.utcnow())) + + def substitue_credentials(): + subprocess.run('envsubst \'${_AWS_BUCKET},${_AWS_BUCKET_DOMAIN},${_AWS_SIGNING_SCOPE},${_AWS_ACCESS_KEY},${_AWS_SIGNING_KEY},${_AWS_SECURITY_TOKEN}\' /etc/nginx/s3proxy.conf', shell=True) + + def logConfig(): + with open('/etc/nginx/s3proxy.conf') as fp: + logging.debug(fp.read()) + + export_credentials() + substitue_credentials() + logConfig() + +def signal_nginx_reload(): + subprocess.run('nginx -s reload', shell=True) + +def format_date(date): + return '%04d%02d%02d' % (date.year, date.month, date.day) + +def has_key_expired(exiration, last_created): + now = datetime.utcnow() + return now >= expiration or now.date() > last_created.date() + +logging.info('Checking Access Credentials...') + +(expiration, last_created) = get_timestamps() +now = datetime.utcnow() + +logging.debug('Expiration: %s\nNow: %s\n' %\ + (expiration, now)) + +if has_key_expired(expiration, last_created): + (aws_access_key, aws_secret_key, aws_security_token, expiration) = get_current_creds() + ymd = format_date(datetime.utcnow().date()) + signature = get_signature_key(aws_secret_key, ymd, aws_region, aws_service) + signature = encode_signature(signature) + scope = get_key_scope(ymd, aws_region, aws_service) + + refresh_credentials(aws_access_key, aws_secret_key, aws_security_token, signature, scope, expiration) + logging.info('Access Credentials Updated') + + logging.debug('Signature (base64): %s\nScope: %s' % (signature, scope)) + + if '--no-reload' not in sys.argv: + logging.info('Reloading Nginx Configuration') + signal_nginx_reload() +else: + logging.info('Access Credentials Current') \ No newline at end of file diff --git a/example/token-auto-refresh-python/startup.sh b/example/token-auto-refresh-python/startup.sh index 094a5dc..1351e95 100755 --- a/example/token-auto-refresh-python/startup.sh +++ b/example/token-auto-refresh-python/startup.sh @@ -1,8 +1,10 @@ #!/bin/sh -# substitute the domain and configure the first set of credentials -envsubst '${BUCKET},${BUCKET_DOMAIN},${_AWS_SIGNING_SCOPE},${_AWS_ACCESS_KEY},${_AWS_SIGNING_KEY},${TOKEN}' /tmp/nginx/server-tmp.conf -cp /tmp/nginx/server-tmp.conf /etc/nginx/server.conf +python3 ./refresh_credentials.py -d --no-reload + +# schedule check for new credentials every minute via cron +echo "* * * * * /refresh_credentials.py" >> /etc/crontabs/root +crond # start nginx (blocking call) nginx -g "daemon off;" From f071c11cf9d7330d0945a935daf2c3f688a81a3f Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Tue, 28 Jan 2020 13:39:54 -0600 Subject: [PATCH 09/10] fixes to python auto-refresh example --- example/token-auto-refresh-python/Dockerfile | 2 +- example/token-auto-refresh-python/nginx/server.conf | 8 ++++---- example/token-auto-refresh-python/refresh_credentials.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/example/token-auto-refresh-python/Dockerfile b/example/token-auto-refresh-python/Dockerfile index 6646f62..b9f20d4 100644 --- a/example/token-auto-refresh-python/Dockerfile +++ b/example/token-auto-refresh-python/Dockerfile @@ -26,7 +26,7 @@ COPY example/token-auto-refresh-python/nginx /etc/nginx COPY example/token-auto-refresh-python/nginx /tmp/nginx COPY example/token-auto-refresh-python/startup.sh /startup.sh COPY --from=build /nginx-1.16.1/objs/ngx_http_aws_auth_module.so /etc/nginx/modules -COPY example/token-auto-refresh/refresh_credentials.py /refresh_credentials.py +COPY example/token-auto-refresh-python/refresh_credentials.py /refresh_credentials.py EXPOSE 80 STOPSIGNAL SIGTERM CMD [ "./startup.sh" ] \ No newline at end of file diff --git a/example/token-auto-refresh-python/nginx/server.conf b/example/token-auto-refresh-python/nginx/server.conf index 01875ad..1eb8d9f 100644 --- a/example/token-auto-refresh-python/nginx/server.conf +++ b/example/token-auto-refresh-python/nginx/server.conf @@ -3,8 +3,8 @@ server { location / { - proxy_pass https://${BUCKET_DOMAIN}; - proxy_set_header Host ${BUCKET_DOMAIN}; + proxy_pass https://${_AWS_BUCKET_DOMAIN}; + proxy_set_header Host ${_AWS_BUCKET_DOMAIN}; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -12,8 +12,8 @@ server { aws_access_key ${_AWS_ACCESS_KEY}; aws_key_scope ${_AWS_SIGNING_SCOPE}; aws_signing_key ${_AWS_SIGNING_KEY}; - aws_s3_bucket ${BUCKET}; - aws_security_token ${TOKEN}; + aws_s3_bucket ${_AWS_BUCKET}; + aws_security_token ${_AWS_SECURITY_TOKEN}; # add the cache status as a response header for debugging add_header X-Cache-Status $upstream_cache_status; diff --git a/example/token-auto-refresh-python/refresh_credentials.py b/example/token-auto-refresh-python/refresh_credentials.py index 9e0e2e3..f3af86e 100644 --- a/example/token-auto-refresh-python/refresh_credentials.py +++ b/example/token-auto-refresh-python/refresh_credentials.py @@ -75,10 +75,10 @@ def export_credentials(): fp.write('%s\n%s\n' % (expiration, datetime.utcnow())) def substitue_credentials(): - subprocess.run('envsubst \'${_AWS_BUCKET},${_AWS_BUCKET_DOMAIN},${_AWS_SIGNING_SCOPE},${_AWS_ACCESS_KEY},${_AWS_SIGNING_KEY},${_AWS_SECURITY_TOKEN}\' /etc/nginx/s3proxy.conf', shell=True) + subprocess.run('envsubst \'${_AWS_BUCKET},${_AWS_BUCKET_DOMAIN},${_AWS_SIGNING_SCOPE},${_AWS_ACCESS_KEY},${_AWS_SIGNING_KEY},${_AWS_SECURITY_TOKEN}\' /etc/nginx/server.conf', shell=True) def logConfig(): - with open('/etc/nginx/s3proxy.conf') as fp: + with open('/etc/nginx/server.conf') as fp: logging.debug(fp.read()) export_credentials() @@ -93,7 +93,7 @@ def format_date(date): def has_key_expired(exiration, last_created): now = datetime.utcnow() - return now >= expiration or now.date() > last_created.date() + return expiration == None or now >= expiration or now.date() > last_created.date() logging.info('Checking Access Credentials...') From 5e26a8b4e090a9a1fc0c2c1ee34301fcb72fdbf4 Mon Sep 17 00:00:00 2001 From: Kory Taborn Date: Tue, 28 Jan 2020 18:08:39 -0600 Subject: [PATCH 10/10] add README documentation and support for mocked credentials --- README.md | 19 ++++++++++++ example/token-auto-refresh-python/.env | 7 +++++ example/token-auto-refresh-python/README.md | 30 +++++++++++++++++++ .../refresh_credentials.py | 29 +++++++++++------- 4 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 example/token-auto-refresh-python/.env create mode 100644 example/token-auto-refresh-python/README.md diff --git a/README.md b/README.md index bf480e3..b2dbf96 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,21 @@ Implements proxying of authenticated requests to S3. } ``` +## Security Token Usage + +If you are using temporary credentials through something like an IAM role, this module +supports this by using the `aws_security_token` directive. You specify this the same as +you do the other directives, however, **you are responsible for recycling the credentials.** + +This _should_ be the normal usage of this module, as even with static credentials you +need to regenerate the signing scope and key after a date change. An example of doing +this via python and the `envsubst` command can be found in the examples folder as a +docker image. The original use case for the example is to pull the credentials from +a Fargate container environment, but it can be adapted to support EC2 +instances. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html +for more details on how to retrieve temporary credentials for EC2 instances assigned an +IAM role. + ## Security considerations The V4 protocol does not need access to the actual secret keys that one obtains from the IAM service. The correct way to use the IAM key is to actually generate @@ -103,7 +118,11 @@ L4vRLWAO92X5L3Sqk5QydUSdB0nC9+1wfqLMOKLbRp4= The 2.x version of the module currently only has support for GET and HEAD calls. This is because signing request body is complex and has not yet been implemented. +## Running Tests +You should be able to run all of the tests for this module via a Docker container. If you run +`make docker-test` from the root of this project, it will build and run the tests for this project +via container. ## Credits Original idea based on http://nginx.org/pipermail/nginx/2010-February/018583.html and suggestion of moving to variables rather than patching the proxy module. diff --git a/example/token-auto-refresh-python/.env b/example/token-auto-refresh-python/.env new file mode 100644 index 0000000..d324a38 --- /dev/null +++ b/example/token-auto-refresh-python/.env @@ -0,0 +1,7 @@ +BUCKET_DOMAIN= +AWS_ACCESS_KEY= +AWS_SECRET_ACCESS_KEY= +AWS_SECURITY_TOKEN= +AWS_TOKEN_EXPIRATION=2020-01-29T05:08:57Z +AWS_REGION= +USE_MOCK=false \ No newline at end of file diff --git a/example/token-auto-refresh-python/README.md b/example/token-auto-refresh-python/README.md new file mode 100644 index 0000000..2336a5c --- /dev/null +++ b/example/token-auto-refresh-python/README.md @@ -0,0 +1,30 @@ +# Automatically Recycling IAM Credentials via Python + +This example sets up the nginx module to support passing the `x-amz-security-token` header when using +temporary IAM credentials. It's original use case is to retrieve credentials from a Fargate container, +but it can be adapted to support any environment. + +## Build + +You can build the environment with the following **from the root of the project**: + +`docker build . -f example/token-auto-refresh-python/Dockerfile -t nginx-aws-auth-refresh` + +## Run + +When deploying to Fargate, the only environment variable required is the BUCKET_DOMAIN, which +should be your full bucket domain url. + +For example: + +`docker run --rm -p 5000:80 -e BUCKET_DOMAIN=public-encrypted-s3.s3.amazonaws.com nginx-aws-auth` + +### Run Locally + +If you would like to run this locally, you will need to retrieve your temporary credentials from +your hosting instance. If you are using Fargate, you can try deploying the image from here (https://github.com/BrutalSimplicity/fargate-ssh), to +allow you to ssh into the instance and view the credentials from that environment. Be sure to +read on how IAM roles are managed for details on how to access that information (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html). + +After you have the credentials fill in the .env file in the root directory, and you can then run +it locally with something like `docker run --rm -p 5000:80 --env-file=example/token-auto-refresh-python/.env nginx-aws-auth-refresh:latest` \ No newline at end of file diff --git a/example/token-auto-refresh-python/refresh_credentials.py b/example/token-auto-refresh-python/refresh_credentials.py index f3af86e..1caef4f 100644 --- a/example/token-auto-refresh-python/refresh_credentials.py +++ b/example/token-auto-refresh-python/refresh_credentials.py @@ -15,7 +15,8 @@ logging.basicConfig(stream=sys.stdout, level=level, format='%(asctime)s - %(name)s - %(levelname)s\n%(message)s') credentials_host = '169.254.170.2' -credentials_path = os.environ['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] +credentials_path = os.environ.get('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI', None) +use_mock = os.environ.get('USE_MOCK', '').lower() in ['1','t','true','yes'] aws_region = os.environ['AWS_REGION'] aws_bucket_domain = os.environ['BUCKET_DOMAIN'] aws_bucket = aws_bucket_domain.split('.')[0] @@ -34,15 +35,23 @@ def get_timestamps(): return (None, None) def get_current_creds(): - try: - conn = http.client.HTTPConnection(credentials_host, timeout=5) - conn.request('GET', credentials_path) - res = conn.getresponse() - - creds = json.loads(res.readline().decode('utf8')) - except: - logging.error("Unexpected error: %s" % (sys.exc_info()[0])) - raise + if not use_mock: + try: + conn = http.client.HTTPConnection(credentials_host, timeout=5) + conn.request('GET', credentials_path) + res = conn.getresponse() + + creds = json.loads(res.readline().decode('utf8')) + except: + logging.error("Unexpected error: %s" % (sys.exc_info()[0])) + raise + else: + creds = { + 'AccessKeyId': os.environ['AWS_ACCESS_KEY'], + 'SecretAccessKey': os.environ['AWS_SECRET_ACCESS_KEY'], + 'Token': os.environ['AWS_SECURITY_TOKEN'], + 'Expiration': os.environ['AWS_TOKEN_EXPIRATION'] + } return (creds['AccessKeyId'], creds['SecretAccessKey'], creds['Token'], creds['Expiration'])