From 90abef0fb9920bf72c97aa70dc48b2687a12a0a5 Mon Sep 17 00:00:00 2001 From: Daniel Mack Date: Mon, 22 Aug 2016 11:03:59 +0200 Subject: [PATCH 1/4] Set iContentLength to kNoContentLengthHeader initially Other checks in the code rely on the possibility that there is no iContentLength set, but that condition is currently never given. Hence, in resetState(), do this assignment explicitly, so that in transfers without a Content-Length field, iContentLength remains -1. --- src/HttpClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HttpClient.cpp b/src/HttpClient.cpp index 498d26c..a7f6a9d 100644 --- a/src/HttpClient.cpp +++ b/src/HttpClient.cpp @@ -32,8 +32,8 @@ void HttpClient::resetState() { iState = eIdle; iStatusCode = 0; - iContentLength = 0; iBodyLengthConsumed = 0; + iContentLength = kNoContentLengthHeader; iContentLengthPtr = kContentLengthPrefix; iHttpResponseTimeout = kHttpResponseTimeout; } From 63d44175e0356669bc29af5a3efdcda3b4a62591 Mon Sep 17 00:00:00 2001 From: Daniel Mack Date: Mon, 22 Aug 2016 11:05:46 +0200 Subject: [PATCH 2/4] Add support for chunked transfer encoding Support for chunked transfer encoding is mandated by many web servers. For the protocol details, see https://en.wikipedia.org/wiki/Chunked_transfer_encoding and RFC7203, section 4.1: https://tools.ietf.org/html/rfc7230#section-4.1 Support for this transfer scheme is now built in to HttpClient::responseBody(), and the state machine handles Content-Length transmissions as well. That has the benefit that timing delays are handled fine for both cases. Note that this replaces #7. Fixes #3 --- src/HttpClient.cpp | 116 +++++++++++++++++++++++++++++++++++++++++++-- src/HttpClient.h | 4 ++ 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/HttpClient.cpp b/src/HttpClient.cpp index a7f6a9d..2f42593 100644 --- a/src/HttpClient.cpp +++ b/src/HttpClient.cpp @@ -514,19 +514,127 @@ int HttpClient::contentLength() return iContentLength; } +int HttpClient::hexToDec(char c) +{ + if (c >= '0' && c <= '9') + { + return c - '0'; + } + + if (c >= 'A' && c <= 'F') + { + return c - 'A' + 10; + } + + if (c >= 'a' && c <= 'f') + { + return c - 'a' + 10; + } + + return -1; +} + String HttpClient::responseBody() { int bodyLength = contentLength(); String response; - if (bodyLength > 0) + int skipBytes = 0; + int chunkSize; + + enum { + CHUNK_READER_HEADER, + CHUNK_READER_CONTENT, + CHUNK_READER_TRAIL_LF, + } state; + + unsigned long timeoutStart = millis(); + + if (bodyLength == kNoContentLengthHeader) { - response.reserve(bodyLength); + // If we didn't get the Content-Length header, assume the server is + // sending in chunked encoding. + // + // See https://en.wikipedia.org/wiki/Chunked_transfer_encoding + + chunkSize = 0; + state = CHUNK_READER_HEADER; + } + else + { + chunkSize = bodyLength; + response.reserve(chunkSize); + state = CHUNK_READER_CONTENT; } - while (available()) + while ((millis() - timeoutStart) < kHttpWaitForDataDelay) { - response += (char)read(); + // For Content-Length based transmission, stop reading after the + // first chunk is completed + if (bodyLength != kNoContentLengthHeader && chunkSize == 0) + { + break; + } + + // In chunked encoding, we don't know how much data we can expect. + // The transfer ends when the connection is closed. + if (!connected()) + { + break; + } + + // Transfers are subject to timing delays, so let's wait a bit if no data is + // currently available. + if (!available()) + { + delay(1); + continue; + } + + char c = (char)read(); + + switch (state) + { + case CHUNK_READER_HEADER: { + int v = hexToDec(c); + + if (v >= 0) + { + chunkSize <<= 4; + chunkSize += hexToDec(c); + } + + if (c == '\n') + { + response.reserve(response.length() + chunkSize); + state = CHUNK_READER_CONTENT; + } + + break; + } + + case CHUNK_READER_CONTENT: + if (chunkSize--) + { + response += c; + } + else + { + skipBytes = 2; + state = CHUNK_READER_TRAIL_LF; + } + + break; + + case CHUNK_READER_TRAIL_LF: + if (skipBytes-- == 0) + { + chunkSize = 0; + state = CHUNK_READER_HEADER; + } + + break; + } } return response; diff --git a/src/HttpClient.h b/src/HttpClient.h index 141b2df..1529fc3 100644 --- a/src/HttpClient.h +++ b/src/HttpClient.h @@ -305,6 +305,10 @@ class HttpClient : public Client */ void flushClientRx(); + /** Helper function to convert a hex nibble to an interger value + */ + int hexToDec(char c); + // Number of milliseconds that we wait each time there isn't any data // available to be read (during status code and header processing) static const int kHttpWaitForDataDelay = 1000; From 5b0c4c04ed707e2b4379171ee5ad625e9084101d Mon Sep 17 00:00:00 2001 From: Daniel Mack Date: Mon, 22 Aug 2016 11:14:58 +0200 Subject: [PATCH 3/4] Write body in chunks of 1024 There seems to be a limit in how many bytes Client::write() may take at once, and data is not sent if it is exceeded. I'm not entriely sure where that limit comes from, but I have the suspicion is has to do with the MTU. A value of 1024 works fine in my test. If that turns out to not cover all cases, the threshold can be made configurable later on. --- src/HttpClient.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/HttpClient.cpp b/src/HttpClient.cpp index 2f42593..fec9c0b 100644 --- a/src/HttpClient.cpp +++ b/src/HttpClient.cpp @@ -132,7 +132,10 @@ int HttpClient::startRequest(const char* aURLPath, const char* aHttpMethod, if (hasBody) { - write(aBody, aContentLength); + for (int i = 0; i < aContentLength; i += 1024) + { + write(aBody + i, min(aContentLength - i, 1024)); + } } } From e4bc11f84db5c41ab0d100651afed7180d9604a0 Mon Sep 17 00:00:00 2001 From: Daniel Mack Date: Mon, 22 Aug 2016 11:18:43 +0200 Subject: [PATCH 4/4] cosemetic trailing white space cleanups --- src/HttpClient.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/HttpClient.cpp b/src/HttpClient.cpp index fec9c0b..722f4d8 100644 --- a/src/HttpClient.cpp +++ b/src/HttpClient.cpp @@ -59,7 +59,7 @@ void HttpClient::beginRequest() iState = eRequestStarted; } -int HttpClient::startRequest(const char* aURLPath, const char* aHttpMethod, +int HttpClient::startRequest(const char* aURLPath, const char* aHttpMethod, const char* aContentType, int aContentLength, const byte aBody[]) { if (iState == eReadingBody) @@ -96,7 +96,7 @@ int HttpClient::startRequest(const char* aURLPath, const char* aHttpMethod, Serial.println("Connection failed"); #endif return HTTP_ERROR_CONNECTION_FAILED; - } + } } } else @@ -385,7 +385,7 @@ int HttpClient::responseStatusCode() const char* statusPrefix = "HTTP/*.* "; const char* statusPtr = statusPrefix; // Whilst we haven't timed out & haven't reached the end of the headers - while ((c != '\n') && + while ((c != '\n') && ( (millis() - timeoutStart) < iHttpResponseTimeout )) { if (available()) @@ -478,7 +478,7 @@ int HttpClient::skipResponseHeaders() // Just keep reading until we finish reading the headers or time out unsigned long timeoutStart = millis(); // Whilst we haven't timed out & haven't reached the end of the headers - while ((!endOfHeadersReached()) && + while ((!endOfHeadersReached()) && ( (millis() - timeoutStart) < iHttpResponseTimeout )) { if (available()) @@ -508,7 +508,7 @@ int HttpClient::skipResponseHeaders() int HttpClient::contentLength() { - // skip the response headers, if they haven't been read already + // skip the response headers, if they haven't been read already if (!endOfHeadersReached()) { skipResponseHeaders(); @@ -684,7 +684,7 @@ bool HttpClient::headerAvailable() { // end of the line, all done break; - } + } else { // ignore any CR or LF characters