From 1e5f0ebdfb141672e6fd95a2db2deb625252b65f Mon Sep 17 00:00:00 2001 From: Sergio Garcia Murillo Date: Tue, 14 Feb 2023 19:18:55 +0100 Subject: [PATCH 1/3] add s4 sigv4 authentication --- libavformat/aviobuf.c | 2 +- libavformat/http.c | 164 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 163 insertions(+), 3 deletions(-) diff --git a/libavformat/aviobuf.c b/libavformat/aviobuf.c index 4ad734a3c3e28..32059d921bc64 100644 --- a/libavformat/aviobuf.c +++ b/libavformat/aviobuf.c @@ -1015,7 +1015,7 @@ URLContext* ffio_geturlcontext(AVIOContext *s) int ffio_copy_url_options(AVIOContext* pb, AVDictionary** avio_opts) { const char *opts[] = { - "headers", "user_agent", "cookies", "http_proxy", "referer", "rw_timeout", "icy", NULL }; + "headers", "user_agent", "cookies", "http_proxy", "referer", "rw_timeout", "icy", "s3_access_key", "s3_secret_key", NULL}; const char **opt = opts; uint8_t *buf = NULL; int ret = 0; diff --git a/libavformat/http.c b/libavformat/http.c index 7bce8215354ac..248f717109da8 100644 --- a/libavformat/http.c +++ b/libavformat/http.c @@ -31,9 +31,32 @@ #include "libavutil/bprint.h" #include "libavutil/getenv_utf8.h" #include "libavutil/opt.h" +#include "libavutil/sha.h" #include "libavutil/time.h" #include "libavutil/parseutils.h" + +#define SIGV4_DO_NOT_USE_CUSTOM_CONFIG +#include "sigv4/sigv4.h" + + +static int32_t valid_sha256_init(void* ctx) +{ + return av_sha_init((struct AVSHA*)ctx, 256); +} + +static int32_t valid_sha256_update(void* ctx, const uint8_t* data, size_t len) +{ + av_sha_update((struct AVSHA*)ctx, data, len); + return 0; +} + +static int32_t valid_sha256_final(void* ctx, uint8_t* digest, size_t digestLen) +{ + av_sha_final((struct AVSHA*)ctx, digest); + return 0; +} + #include "avformat.h" #include "http.h" #include "httpauth.h" @@ -136,6 +159,9 @@ typedef struct HTTPContext { char *new_location; AVDictionary *redirect_cache; uint64_t filesize_from_content_range; + /* aws sigv4*/ + char* s3_access_key; + char* s3_secret_key; } HTTPContext; #define OFFSET(x) offsetof(HTTPContext, x) @@ -178,6 +204,8 @@ static const AVOption options[] = { { "resource", "The resource requested by a client", OFFSET(resource), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, E }, { "reply_code", "The http status code to return to a client", OFFSET(reply_code), AV_OPT_TYPE_INT, { .i64 = 200}, INT_MIN, 599, E}, { "short_seek_size", "Threshold to favor readahead over seek.", OFFSET(short_seek_size), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, D }, + { "s3_access_key", "AWS S3 access key", OFFSET(s3_access_key), AV_OPT_TYPE_STRING, {.str = NULL }, 0, 0, AV_OPT_FLAG_EXPORT }, + { "s3_secret_key", "AWS S3 secret key", OFFSET(s3_secret_key), AV_OPT_TYPE_STRING, {.str = NULL }, 0, 0, AV_OPT_FLAG_EXPORT }, { NULL } }; @@ -1407,6 +1435,7 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, uint64_t off = s->off; const char *method; int send_expect_100 = 0; + size_t headers = 0; av_bprint_init_for_buffer(&request, s->buffer, sizeof(s->buffer)); @@ -1446,10 +1475,12 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, } } + av_bprintf(&request, "%s ", method); bprint_escaped_path(&request, path); av_bprintf(&request, " HTTP/1.1\r\n"); + if (post && s->chunked_post) av_bprintf(&request, "Transfer-Encoding: chunked\r\n"); /* set default headers if needed */ @@ -1477,8 +1508,6 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, if (!has_header(s->headers, "\r\nConnection: ")) av_bprintf(&request, "Connection: %s\r\n", s->multiple_requests ? "keep-alive" : "close"); - if (!has_header(s->headers, "\r\nHost: ")) - av_bprintf(&request, "Host: %s\r\n", hoststr); if (!has_header(s->headers, "\r\nContent-Length: ") && s->post_data) av_bprintf(&request, "Content-Length: %d\r\n", s->post_datalen); @@ -1494,10 +1523,141 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, if (!has_header(s->headers, "\r\nIcy-MetaData: ") && s->icy) av_bprintf(&request, "Icy-MetaData: 1\r\n"); + /* Get pointer to signed headers start */ + headers = request.len; + + if (!has_header(s->headers, "\r\nHost: ")) + av_bprintf(&request, "Host: %s\r\n", hoststr); + /* now add in custom headers */ if (s->headers) av_bprintf(&request, "%s", s->headers); + if (s->s3_access_key != NULL && s->s3_secret_key != NULL) { + struct SigV4Credentials sigv4Creds; + struct SigV4HttpParameters sigv4HttpParams; + struct SigV4CryptoInterface cryptoInterface; + struct SigV4Parameters sigv4Params; + SigV4Status_t status = SigV4Success; + struct AVSHA* content_sha256_ctx; + char content_sha256_digest[32]; + char content_sha256_digest_str[65]; + + const char* s3_service_name = "s3"; + char *s3_bucket = NULL, *s3_region = NULL, *host_copy = NULL, *end = NULL; + + // Buffer to hold the authorization header. + char pSigv4Auth[2048U]; + size_t sigv4AuthLen = sizeof(pSigv4Auth); + + // Pointer to signature in the Authorization header that will be populated in + // pSigv4Auth by the SigV4_GenerateHTTPAuthorization API function. + char* signature = NULL; + size_t signatureLen = 0; + + char utc_date[200]; + char date_ISO8601[SIGV4_ISO_STRING_LEN+1]; + time_t t; + struct tm* tmp; + + /* Calculate iso8601 data*/ + t = time(NULL); + tmp = gmtime(&t); + + if (!strftime(utc_date, sizeof(utc_date), "%FT%TZ", tmp)) { + av_log(NULL, AV_LOG_DEBUG, "strftime error in http_connect\n"); + return AVERROR_UNKNOWN; + } + + status = SigV4_AwsIotDateToIso8601(utc_date, strlen(utc_date), date_ISO8601, SIGV4_ISO_STRING_LEN); + date_ISO8601[SIGV4_ISO_STRING_LEN] = 0; + + if (status != SigV4Success) { + av_log(h, AV_LOG_ERROR, "Failed to parse Date header: %s\n", utc_date); + err = AVERROR(EINVAL); + goto done; + } + + av_bprintf(&request, "x-amz-date: %s\r\n", date_ISO8601); + + /* Calculate SHA256 of the payload*/ + content_sha256_ctx = av_sha_alloc(); + av_sha_init(content_sha256_ctx, 256); + av_sha_update(content_sha256_ctx, post ? s->post_data : NULL, post ? s->post_datalen : 0U); + av_sha_final(content_sha256_ctx, content_sha256_digest); + ff_data_to_hex(content_sha256_digest_str, content_sha256_digest, sizeof(content_sha256_digest), 1); + av_freep(&content_sha256_ctx); + + av_bprintf(&request, "x-amz-content-sha256: %s\r\n", content_sha256_digest_str); + + /* Get bucket and region from host string*/ + host_copy = strdup(hoststr); + s3_bucket = av_strtok(host_copy, ".", &end); + s3_region = av_strtok(NULL, ".", &end); + + if (s3_bucket == NULL || s3_region == NULL) { + av_log(h, AV_LOG_ERROR, "Cannot parse s3 bucket and region from host: %s\n", hoststr); + av_freep(&host_copy); + av_freep(&cryptoInterface.pHashContext); + err = AVERROR(EINVAL); + goto done; + } + + sigv4HttpParams.pHttpMethod = method; + sigv4HttpParams.httpMethodLen = strlen(method); + sigv4HttpParams.pPath = path; + sigv4HttpParams.pathLen = strlen(path); + sigv4HttpParams.pQuery = NULL; + sigv4HttpParams.queryLen = 0; + sigv4HttpParams.flags = 0; + sigv4HttpParams.pHeaders = request.str + headers; + sigv4HttpParams.headersLen = request.len - headers; + sigv4HttpParams.pPayload = post ? s->post_data : NULL; + sigv4HttpParams.payloadLen = post ? s->post_datalen : 0U; + + cryptoInterface.pHashContext = (void*) av_sha_alloc(); + cryptoInterface.hashInit = valid_sha256_init; + cryptoInterface.hashUpdate = valid_sha256_update; + cryptoInterface.hashFinal = valid_sha256_final; + cryptoInterface.hashBlockLen = SIGV4_HASH_MAX_BLOCK_LENGTH; + cryptoInterface.hashDigestLen = SIGV4_HASH_MAX_DIGEST_LENGTH; + + sigv4Creds.pAccessKeyId = s->s3_access_key; + sigv4Creds.accessKeyIdLen = strlen(s->s3_access_key); + sigv4Creds.pSecretAccessKey = s->s3_secret_key; + sigv4Creds.secretAccessKeyLen = strlen(s->s3_secret_key); + + sigv4Params.pAlgorithm = SIGV4_AWS4_HMAC_SHA256; + sigv4Params.algorithmLen = SIGV4_AWS4_HMAC_SHA256_LENGTH; + // Parsed temporary credentials obtained from AWS IoT Credential Provider. + sigv4Params.pCredentials = &sigv4Creds; + // Date in ISO8601 format. + sigv4Params.pDateIso8601 = date_ISO8601; + // The AWS region for the request. + sigv4Params.pRegion = s3_region; + sigv4Params.regionLen = strlen(s3_region), + // The AWS service for the request. + sigv4Params.pService = s3_service_name; + sigv4Params.serviceLen = strlen(s3_service_name); + // SigV4 crypto interface. See SigV4CryptoInterface_t interface documentation. + sigv4Params.pCryptoInterface = &cryptoInterface; + // HTTP parameters for the HTTP request to generate a SigV4 authorization header for. + sigv4Params.pHttpParameters = &sigv4HttpParams; + + status = SigV4_GenerateHTTPAuthorization(&sigv4Params, pSigv4Auth, &sigv4AuthLen, &signature, &signatureLen); + + av_freep(&host_copy); + av_freep(&cryptoInterface.pHashContext); + + if (status != SigV4Success) { + av_log(h, AV_LOG_ERROR, "Failed to generate authorization header: %d bucket:%s region:%s\n%s\n", status, s3_bucket, s3_region, sigv4HttpParams.pHeaders); + err = AVERROR(EINVAL); + goto done; + } + + av_bprintf(&request, "Authorization: %s\r\n", pSigv4Auth); + } + if (authstr) av_bprintf(&request, "%s", authstr); if (proxyauthstr) From 5ea696b313704db24122773f51e8f76857cbc96d Mon Sep 17 00:00:00 2001 From: Sergio Garcia Murillo Date: Wed, 1 Mar 2023 19:15:37 +0100 Subject: [PATCH 2/3] add s3 bucket and region config --- libavformat/aviobuf.c | 2 +- libavformat/http.c | 27 ++++++++------------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/libavformat/aviobuf.c b/libavformat/aviobuf.c index 32059d921bc64..433758a6c9f8d 100644 --- a/libavformat/aviobuf.c +++ b/libavformat/aviobuf.c @@ -1015,7 +1015,7 @@ URLContext* ffio_geturlcontext(AVIOContext *s) int ffio_copy_url_options(AVIOContext* pb, AVDictionary** avio_opts) { const char *opts[] = { - "headers", "user_agent", "cookies", "http_proxy", "referer", "rw_timeout", "icy", "s3_access_key", "s3_secret_key", NULL}; + "headers", "user_agent", "cookies", "http_proxy", "referer", "rw_timeout", "icy", "s3_access_key", "s3_secret_key", "s3_bucket", "s3_region", NULL}; const char **opt = opts; uint8_t *buf = NULL; int ret = 0; diff --git a/libavformat/http.c b/libavformat/http.c index 248f717109da8..85963db11cae5 100644 --- a/libavformat/http.c +++ b/libavformat/http.c @@ -162,6 +162,8 @@ typedef struct HTTPContext { /* aws sigv4*/ char* s3_access_key; char* s3_secret_key; + char* s3_bucket; + char* s3_region; } HTTPContext; #define OFFSET(x) offsetof(HTTPContext, x) @@ -206,6 +208,8 @@ static const AVOption options[] = { { "short_seek_size", "Threshold to favor readahead over seek.", OFFSET(short_seek_size), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, D }, { "s3_access_key", "AWS S3 access key", OFFSET(s3_access_key), AV_OPT_TYPE_STRING, {.str = NULL }, 0, 0, AV_OPT_FLAG_EXPORT }, { "s3_secret_key", "AWS S3 secret key", OFFSET(s3_secret_key), AV_OPT_TYPE_STRING, {.str = NULL }, 0, 0, AV_OPT_FLAG_EXPORT }, + { "s3_bucket", "AWS S3 bucket name", OFFSET(s3_bucket), AV_OPT_TYPE_STRING, {.str = NULL }, 0, 0, AV_OPT_FLAG_EXPORT }, + { "s3_region", "AWS S3 region name", OFFSET(s3_region), AV_OPT_TYPE_STRING, {.str = NULL }, 0, 0, AV_OPT_FLAG_EXPORT }, { NULL } }; @@ -1533,7 +1537,7 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, if (s->headers) av_bprintf(&request, "%s", s->headers); - if (s->s3_access_key != NULL && s->s3_secret_key != NULL) { + if (s->s3_access_key != NULL && s->s3_secret_key != NULL && s->s3_region != NULL && s->s3_bucket != NULL) { struct SigV4Credentials sigv4Creds; struct SigV4HttpParameters sigv4HttpParams; struct SigV4CryptoInterface cryptoInterface; @@ -1544,7 +1548,6 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, char content_sha256_digest_str[65]; const char* s3_service_name = "s3"; - char *s3_bucket = NULL, *s3_region = NULL, *host_copy = NULL, *end = NULL; // Buffer to hold the authorization header. char pSigv4Auth[2048U]; @@ -1589,19 +1592,6 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, av_freep(&content_sha256_ctx); av_bprintf(&request, "x-amz-content-sha256: %s\r\n", content_sha256_digest_str); - - /* Get bucket and region from host string*/ - host_copy = strdup(hoststr); - s3_bucket = av_strtok(host_copy, ".", &end); - s3_region = av_strtok(NULL, ".", &end); - - if (s3_bucket == NULL || s3_region == NULL) { - av_log(h, AV_LOG_ERROR, "Cannot parse s3 bucket and region from host: %s\n", hoststr); - av_freep(&host_copy); - av_freep(&cryptoInterface.pHashContext); - err = AVERROR(EINVAL); - goto done; - } sigv4HttpParams.pHttpMethod = method; sigv4HttpParams.httpMethodLen = strlen(method); @@ -1634,8 +1624,8 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, // Date in ISO8601 format. sigv4Params.pDateIso8601 = date_ISO8601; // The AWS region for the request. - sigv4Params.pRegion = s3_region; - sigv4Params.regionLen = strlen(s3_region), + sigv4Params.pRegion = s->s3_region; + sigv4Params.regionLen = strlen(s->s3_region), // The AWS service for the request. sigv4Params.pService = s3_service_name; sigv4Params.serviceLen = strlen(s3_service_name); @@ -1646,11 +1636,10 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, status = SigV4_GenerateHTTPAuthorization(&sigv4Params, pSigv4Auth, &sigv4AuthLen, &signature, &signatureLen); - av_freep(&host_copy); av_freep(&cryptoInterface.pHashContext); if (status != SigV4Success) { - av_log(h, AV_LOG_ERROR, "Failed to generate authorization header: %d bucket:%s region:%s\n%s\n", status, s3_bucket, s3_region, sigv4HttpParams.pHeaders); + av_log(h, AV_LOG_ERROR, "Failed to generate authorization header: %d bucket:%s region:%s\n%s\n", status, s->s3_bucket, s->s3_region, sigv4HttpParams.pHeaders); err = AVERROR(EINVAL); goto done; } From 4a13785198c0a982dd46c5ebaaf4b0ab4470a640 Mon Sep 17 00:00:00 2001 From: Sergio Garcia Murillo Date: Mon, 6 Mar 2023 10:40:04 +0100 Subject: [PATCH 3/3] fix auth header length --- libavformat/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/http.c b/libavformat/http.c index 85963db11cae5..03ab53b4ee338 100644 --- a/libavformat/http.c +++ b/libavformat/http.c @@ -1644,7 +1644,7 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, goto done; } - av_bprintf(&request, "Authorization: %s\r\n", pSigv4Auth); + av_bprintf(&request, "Authorization: %.*s\r\n", (int)sigv4AuthLen, pSigv4Auth); } if (authstr)