From e274f86bf5e8650a7ed2af3b9a15f3982083e2e5 Mon Sep 17 00:00:00 2001 From: Adam Chlupacek Date: Sun, 9 Jun 2019 19:22:23 +0200 Subject: [PATCH] Add support for http multipart data. --- .../protocol/http/codec/HttpHeaderCodec.scala | 32 ++++++++++++ .../http/codec/HttpMimeHeaderCodec.scala | 22 ++++++++ .../http/header/Content-Disposition.scala | 4 +- .../protocol/http/header/Content-Type.scala | 6 +-- .../protocol/http/header/GenericHeader.scala | 2 +- .../protocol/http/header/HttpHeader.scala | 2 + .../header/`Content-Transfer-Encoding`.scala | 19 +++++++ .../header/value/HeaderCodecDefinition.scala | 18 ++++++- .../protocol/http/mime/MIMEHeader.scala | 45 ++++++++++++++++ .../http/codec/HttpMimeHeaderCodecSpec.scala | 51 +++++++++++++++++++ .../spinoco/protocol/mail/EmailHeader.scala | 3 +- .../header/`Content-Transfer-Encoding`.scala | 2 +- .../protocol/mail/EmailHeaderSpec.scala | 3 +- .../protocol}/mime/TransferEncoding.scala | 4 +- 14 files changed, 199 insertions(+), 14 deletions(-) create mode 100644 http/src/main/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodec.scala create mode 100644 http/src/main/scala/spinoco/protocol/http/header/`Content-Transfer-Encoding`.scala create mode 100644 http/src/main/scala/spinoco/protocol/http/mime/MIMEHeader.scala create mode 100644 http/src/test/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodecSpec.scala rename {mail/src/main/scala/spinoco/protocol/mail => mime/src/main/scala/spinoco/protocol}/mime/TransferEncoding.scala (97%) diff --git a/http/src/main/scala/spinoco/protocol/http/codec/HttpHeaderCodec.scala b/http/src/main/scala/spinoco/protocol/http/codec/HttpHeaderCodec.scala index d1f2815..bffe4e7 100644 --- a/http/src/main/scala/spinoco/protocol/http/codec/HttpHeaderCodec.scala +++ b/http/src/main/scala/spinoco/protocol/http/codec/HttpHeaderCodec.scala @@ -38,6 +38,30 @@ object HttpHeaderCodec { ) } + /** + * Encodes// decodes arbitrary header line for mime header. + * @param otherHeaders If you need to supply own headers to be encoded/decoded, + * Supply them here. First is header name (may be upper/lowercase) and second + * is decoder of the header. + * This may also override default supplied codecs. + * @return + */ + def contentCodec(maxHeaderLength: Int, otherHeaders: (String, Codec[ContentHeaderField]) *): Codec[ContentHeaderField] = { + val allCodecs = allContentHeadersCodecs ++ otherHeaders.map { case (hdr,codec) => hdr.toLowerCase -> codec }.toMap + implicit val ascii = Charset.forName("ASCII") // only ascii allowed in http header + + takeWhile(ByteVector(':'), ByteVector(':',' '), string, maxHeaderLength).flatZip[ContentHeaderField] { name => + val trimmed = name.trim + ignoreWS ~> + (allCodecs.get(trimmed.toLowerCase) match { + case Some(codec) => codec + case None => utf8.xmap[GenericHeader](s => GenericHeader(trimmed, s), _.value).upcast[ContentHeaderField] + }) + }.xmap ( + { case (_, header) => header } + , (header: ContentHeaderField) => header.name -> header + ) + } val allHeaderCodecs : Map[String, Codec[HttpHeader]] = Seq[HeaderCodecDefinition[HttpHeader]]( Accept.codec @@ -96,4 +120,12 @@ object HttpHeaderCodec { ).map { codec => codec.headerName.toLowerCase -> codec.headerCodec }.toMap + val allContentHeadersCodecs: Map[String, Codec[ContentHeaderField]] = { + Seq( + `Content-Disposition`.codec + , `Content-Type`.codec + , `Content-Transfer-Encoding`.codec + ).map { codec => codec.headerName.toLowerCase -> codec.contentField}.toMap + } + } diff --git a/http/src/main/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodec.scala b/http/src/main/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodec.scala new file mode 100644 index 0000000..c35c93c --- /dev/null +++ b/http/src/main/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodec.scala @@ -0,0 +1,22 @@ +package spinoco.protocol.http.codec + +import scodec.{Codec, codecs} +import spinoco.protocol.http.codec.helper.crlf +import spinoco.protocol.http.header.ContentHeaderField +import spinoco.protocol.http.mime.MIMEHeader + +object HttpMimeHeaderCodec { + + val defaultCodec: Codec[MIMEHeader] = + codec(HttpHeaderCodec.contentCodec(Int.MaxValue)) + + def codec( + headerCodec: Codec[ContentHeaderField] + ): Codec[MIMEHeader] = { + codecs.listDelimited[ContentHeaderField](crlf.bits, headerCodec).xmap[MIMEHeader]( + headers => MIMEHeader(headers) + , header => header.fields + ) + } + +} diff --git a/http/src/main/scala/spinoco/protocol/http/header/Content-Disposition.scala b/http/src/main/scala/spinoco/protocol/http/header/Content-Disposition.scala index f594176..52b7dc2 100644 --- a/http/src/main/scala/spinoco/protocol/http/header/Content-Disposition.scala +++ b/http/src/main/scala/spinoco/protocol/http/header/Content-Disposition.scala @@ -9,9 +9,9 @@ import spinoco.protocol.mime.ContentDisposition * * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition */ -sealed case class `Content-Disposition`(value: ContentDisposition) extends DefaultHeader +sealed case class `Content-Disposition`(value: ContentDisposition) extends DefaultHeader with ContentHeaderField object `Content-Disposition` { val codec = - HeaderCodecDefinition[ `Content-Disposition`](ContentDisposition.htmlCodec.xmap (cd => `Content-Disposition`(cd), _.value)) + HeaderCodecDefinition.contentField[ `Content-Disposition`](ContentDisposition.htmlCodec.xmap (cd => `Content-Disposition`(cd), _.value)) } diff --git a/http/src/main/scala/spinoco/protocol/http/header/Content-Type.scala b/http/src/main/scala/spinoco/protocol/http/header/Content-Type.scala index 49df7e4..58ad3a2 100644 --- a/http/src/main/scala/spinoco/protocol/http/header/Content-Type.scala +++ b/http/src/main/scala/spinoco/protocol/http/header/Content-Type.scala @@ -9,9 +9,9 @@ import spinoco.protocol.mime.ContentType * RFC 7231 section 3.1.1.5 * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type */ -sealed case class `Content-Type`(value: ContentType) extends DefaultHeader +sealed case class `Content-Type`(value: ContentType) extends DefaultHeader with ContentHeaderField -object `Content-Type` { val codec = - HeaderCodecDefinition[`Content-Type`](ContentType.codec.xmap (`Content-Type`.apply, _.value)) +object `Content-Type` { + val codec = HeaderCodecDefinition.contentField[`Content-Type`](ContentType.codec.xmap (`Content-Type`.apply, _.value)) } diff --git a/http/src/main/scala/spinoco/protocol/http/header/GenericHeader.scala b/http/src/main/scala/spinoco/protocol/http/header/GenericHeader.scala index 4cfda37..ada0790 100644 --- a/http/src/main/scala/spinoco/protocol/http/header/GenericHeader.scala +++ b/http/src/main/scala/spinoco/protocol/http/header/GenericHeader.scala @@ -3,5 +3,5 @@ package spinoco.protocol.http.header /** * Generic unrecognized header */ -case class GenericHeader(name: String, value: String) extends HttpHeader +case class GenericHeader(name: String, value: String) extends HttpHeader with ContentHeaderField diff --git a/http/src/main/scala/spinoco/protocol/http/header/HttpHeader.scala b/http/src/main/scala/spinoco/protocol/http/header/HttpHeader.scala index f035ec1..6d50eca 100644 --- a/http/src/main/scala/spinoco/protocol/http/header/HttpHeader.scala +++ b/http/src/main/scala/spinoco/protocol/http/header/HttpHeader.scala @@ -15,3 +15,5 @@ trait HttpHeader { } +/** denotes header fields that may be used with MIME parts **/ +trait ContentHeaderField extends HttpHeader \ No newline at end of file diff --git a/http/src/main/scala/spinoco/protocol/http/header/`Content-Transfer-Encoding`.scala b/http/src/main/scala/spinoco/protocol/http/header/`Content-Transfer-Encoding`.scala new file mode 100644 index 0000000..1ad69df --- /dev/null +++ b/http/src/main/scala/spinoco/protocol/http/header/`Content-Transfer-Encoding`.scala @@ -0,0 +1,19 @@ +package spinoco.protocol.http.header + +import spinoco.protocol.http.header.value.HeaderCodecDefinition +import spinoco.protocol.mime.TransferEncoding + +/** + * https://tools.ietf.org/html/rfc2045#section-6 + * + * Although this does not seem to be HTTP header, in https://tools.ietf.org/html/rfc7578#section-4.5 is shown that it can + * be used together with form data. + * + */ +case class `Content-Transfer-Encoding`(value: TransferEncoding) extends DefaultHeader with ContentHeaderField + +object `Content-Transfer-Encoding` { + + val codec = HeaderCodecDefinition.contentField[`Content-Transfer-Encoding`](TransferEncoding.codec.xmap(`Content-Transfer-Encoding`.apply, _.value)) + +} \ No newline at end of file diff --git a/http/src/main/scala/spinoco/protocol/http/header/value/HeaderCodecDefinition.scala b/http/src/main/scala/spinoco/protocol/http/header/value/HeaderCodecDefinition.scala index 7460453..701c2d4 100644 --- a/http/src/main/scala/spinoco/protocol/http/header/value/HeaderCodecDefinition.scala +++ b/http/src/main/scala/spinoco/protocol/http/header/value/HeaderCodecDefinition.scala @@ -1,7 +1,8 @@ package spinoco.protocol.http.header.value import scodec.Codec -import spinoco.protocol.http.header.HttpHeader +import shapeless.Typeable +import spinoco.protocol.http.header.{ContentHeaderField, HttpHeader} import scala.reflect.ClassTag @@ -30,4 +31,19 @@ object HeaderCodecDefinition { clz.getSimpleName.replace("$minus","-") } + trait ContentHeaderCodecDefinition[A <: HttpHeader] extends HeaderCodecDefinition[A]{ + + def contentField: Codec[ContentHeaderField] + + } + + def contentField[A <: ContentHeaderField: Typeable](codec: Codec[A])(implicit ev: ClassTag[A]): ContentHeaderCodecDefinition[HttpHeader] = + new ContentHeaderCodecDefinition[HttpHeader] { + def headerName: String = nameFromClass(ev.runtimeClass) + + def headerCodec: Codec[HttpHeader] = codec.asInstanceOf[Codec[HttpHeader]].withContext(headerName) + + def contentField: Codec[ContentHeaderField] = codec.upcast + } + } \ No newline at end of file diff --git a/http/src/main/scala/spinoco/protocol/http/mime/MIMEHeader.scala b/http/src/main/scala/spinoco/protocol/http/mime/MIMEHeader.scala new file mode 100644 index 0000000..9639d64 --- /dev/null +++ b/http/src/main/scala/spinoco/protocol/http/mime/MIMEHeader.scala @@ -0,0 +1,45 @@ +package spinoco.protocol.http.mime + +import shapeless.Typeable +import spinoco.protocol.http.header._ +import spinoco.protocol.mime.{ContentDisposition, ContentType, TransferEncoding} + + +/** + * The header for HTTP multipart data. + * + * @param fields The header fields. + */ +case class MIMEHeader(fields: List[ContentHeaderField]) { self => + + def getField[A <: ContentHeaderField](implicit T: Typeable[A]): Option[A] = + fields.collectFirst(Function.unlift(T.cast)) + + def appendField(field: ContentHeaderField): MIMEHeader = + self.copy(fields = self.fields :+ field) + + def getContentType: Option[ContentType] = + getField[`Content-Type`].map(_.value) + + def getTransferEncoding: Option[TransferEncoding] = + getField[`Content-Transfer-Encoding`].map(_.value) + + def getContentDisposition: Option[ContentDisposition] = + getField[`Content-Disposition`].map(_.value) + + def contentType(tpe: ContentType): MIMEHeader = + appendField(`Content-Type`(tpe)) + + def transferEncoding(enc: TransferEncoding): MIMEHeader = + appendField(`Content-Transfer-Encoding`(enc)) + + def contentDisposition(disp: ContentDisposition): MIMEHeader = + appendField(`Content-Disposition`(disp)) + +} + + + + + + diff --git a/http/src/test/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodecSpec.scala b/http/src/test/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodecSpec.scala new file mode 100644 index 0000000..ee1096a --- /dev/null +++ b/http/src/test/scala/spinoco/protocol/http/codec/HttpMimeHeaderCodecSpec.scala @@ -0,0 +1,51 @@ +package spinoco.protocol.http.codec + +import org.scalacheck.Properties +import org.scalacheck.Prop._ +import scodec.{Attempt, DecodeResult} +import scodec.bits.BitVector +import spinoco.protocol.http.mime.MIMEHeader +import spinoco.protocol.http.header._ +import spinoco.protocol.mime._ + +object HttpMimeHeaderCodecSpec extends Properties("HttpMimeHeaderCodec"){ + + + property("decode") = secure { + HttpMimeHeaderCodec.defaultCodec.decode(BitVector( + Seq( + "Content-Disposition: form-data; name=\"field1\"" + , "Content-Type: text/plain;charset=UTF-8" + , "Content-Transfer-Encoding: quoted-printable" + ).mkString("\r\n").getBytes() + )) ?= Attempt.successful(DecodeResult( + MIMEHeader( + List( + `Content-Disposition`(ContentDisposition(ContentDispositionType.IETFToken("form-data"), Map("name" -> "field1"))) + , `Content-Type`(ContentType.TextContent(MediaType.`text/plain`, Some(MIMECharset.`UTF-8`))) + , `Content-Transfer-Encoding`(TransferEncoding.QuotedPrintable) + ) + ) + , BitVector.empty)) + } + + property("encode") = secure { + HttpMimeHeaderCodec.defaultCodec.encode( + MIMEHeader( + List( + `Content-Disposition`(ContentDisposition(ContentDispositionType.IETFToken("form-data"), Map("name" -> "field1"))) + , `Content-Type`(ContentType.TextContent(MediaType.`text/plain`, Some(MIMECharset.`UTF-8`))) + , `Content-Transfer-Encoding`(TransferEncoding.QuotedPrintable) + ) + ) + ).map(_.decodeAscii) ?= Attempt.successful(Right( + Seq( + "Content-Disposition: form-data; name=field1" + , "Content-Type: text/plain; charset=utf-8" + , "Content-Transfer-Encoding: quoted-printable" + ).mkString("\r\n") + )) + + } + +} diff --git a/mail/src/main/scala/spinoco/protocol/mail/EmailHeader.scala b/mail/src/main/scala/spinoco/protocol/mail/EmailHeader.scala index 3892dc5..fbfda7e 100644 --- a/mail/src/main/scala/spinoco/protocol/mail/EmailHeader.scala +++ b/mail/src/main/scala/spinoco/protocol/mail/EmailHeader.scala @@ -5,8 +5,7 @@ import java.time.ZonedDateTime import shapeless.Typeable import spinoco.protocol.mail.header._ -import spinoco.protocol.mail.mime.TransferEncoding -import spinoco.protocol.mime.ContentType +import spinoco.protocol.mime.{ContentType, TransferEncoding} /** * Represents Email header that consists of arbitrary fields. diff --git a/mail/src/main/scala/spinoco/protocol/mail/header/`Content-Transfer-Encoding`.scala b/mail/src/main/scala/spinoco/protocol/mail/header/`Content-Transfer-Encoding`.scala index 5840a70..1879a12 100644 --- a/mail/src/main/scala/spinoco/protocol/mail/header/`Content-Transfer-Encoding`.scala +++ b/mail/src/main/scala/spinoco/protocol/mail/header/`Content-Transfer-Encoding`.scala @@ -1,7 +1,7 @@ package spinoco.protocol.mail.header import scodec.Codec -import spinoco.protocol.mail.mime.TransferEncoding +import spinoco.protocol.mime.TransferEncoding case class `Content-Transfer-Encoding`(value: TransferEncoding) extends ContentHeaderField with DefaultEmailHeaderField diff --git a/mail/src/test/scala/spinoco/protocol/mail/EmailHeaderSpec.scala b/mail/src/test/scala/spinoco/protocol/mail/EmailHeaderSpec.scala index 41f24e8..f0f35fd 100644 --- a/mail/src/test/scala/spinoco/protocol/mail/EmailHeaderSpec.scala +++ b/mail/src/test/scala/spinoco/protocol/mail/EmailHeaderSpec.scala @@ -9,8 +9,7 @@ import scodec.bits.ByteVector import shapeless.tag import spinoco.protocol.mail.header._ import spinoco.protocol.mail.header.codec.EmailHeaderCodec -import spinoco.protocol.mail.mime.TransferEncoding -import spinoco.protocol.mime.{ContentType, MIMECharset, MediaType} +import spinoco.protocol.mime.{ContentType, MIMECharset, MediaType, TransferEncoding} /** * Created by pach on 23/10/17. diff --git a/mail/src/main/scala/spinoco/protocol/mail/mime/TransferEncoding.scala b/mime/src/main/scala/spinoco/protocol/mime/TransferEncoding.scala similarity index 97% rename from mail/src/main/scala/spinoco/protocol/mail/mime/TransferEncoding.scala rename to mime/src/main/scala/spinoco/protocol/mime/TransferEncoding.scala index 6b28c33..96a07c1 100644 --- a/mail/src/main/scala/spinoco/protocol/mail/mime/TransferEncoding.scala +++ b/mime/src/main/scala/spinoco/protocol/mime/TransferEncoding.scala @@ -1,7 +1,7 @@ -package spinoco.protocol.mail.mime +package spinoco.protocol.mime -import scodec.{Attempt, Codec} import scodec.codecs._ +import scodec.{Attempt, Codec} import spinoco.protocol.common.util.attempt