diff --git a/src/openms/include/OpenMS/FORMAT/Base64.h b/src/openms/include/OpenMS/FORMAT/Base64.h index 23abe32abfa..d97e43e9258 100644 --- a/src/openms/include/OpenMS/FORMAT/Base64.h +++ b/src/openms/include/OpenMS/FORMAT/Base64.h @@ -73,7 +73,7 @@ namespace OpenMS You have to specify the byte order of the input and if it is zlib-compressed. */ template - static void decode(const String & in, ByteOrder from_byte_order, std::vector & out, bool zlib_compression = false); + static void decode(const String & in, ByteOrder from_byte_order, std::vector & out, bool zlib_compression = false, size_t default_arr_length = 0); /** @brief Encodes a vector of integer point numbers to a Base64 string @@ -83,7 +83,7 @@ namespace OpenMS @note @p in will be empty after this method */ template - static void encodeIntegers(std::vector & in, ByteOrder to_byte_order, String & out, bool zlib_compression = false); + static void encodeIntegers(std::vector& in, ByteOrder to_byte_order, String& out, bool zlib_compression = false); /** @brief Decodes a Base64 string to a vector of integer numbers @@ -91,7 +91,7 @@ namespace OpenMS You have to specify the byte order of the input and if it is zlib-compressed. */ template - static void decodeIntegers(const String & in, ByteOrder from_byte_order, std::vector & out, bool zlib_compression = false); + static void decodeIntegers(const String & in, ByteOrder from_byte_order, std::vector & out, bool zlib_compression = false, size_t default_arr_length = 0); /** @brief Encodes a vector of strings to a Base64 string @@ -116,7 +116,7 @@ namespace OpenMS @param out A vector containing the decoded data (split at null "\0") bytes @param zlib_compression Whether the data should be decompressed with zlib after decoding in Base64 */ - static void decodeStrings(const String & in, std::vector & out, bool zlib_compression = false); + static void decodeStrings(const String& in, std::vector& out, bool zlib_compression = false, size_t default_arr_length = 0); /** @brief Decodes a Base64 string to a QByteArray @@ -125,7 +125,7 @@ namespace OpenMS @param base64_uncompressed A ByteArray containing the decoded data @param zlib_compression Whether the data should be decompressed with zlib after decoding in Base64 */ - static void decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression); + static void decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression, size_t default_arr_length = 0); private: @@ -151,7 +151,7 @@ namespace OpenMS ///Decodes a compressed Base64 string to a vector of floating point numbers template - static void decodeCompressed_(const String & in, ByteOrder from_byte_order, std::vector & out); + static void decodeCompressed_(const String& in, ByteOrder from_byte_order, std::vector& out, size_t default_arr_length); /// Decodes a Base64 string to a vector of integer numbers template @@ -159,7 +159,7 @@ namespace OpenMS ///Decodes a compressed Base64 string to a vector of integer numbers template - static void decodeIntegersCompressed_(const String & in, ByteOrder from_byte_order, std::vector & out); + static void decodeIntegersCompressed_(const String& in, ByteOrder from_byte_order, std::vector& out, size_t default_arr_length); static void stringSimdEncoder_(std::string& in, std::string& out); @@ -246,11 +246,11 @@ namespace OpenMS } template ////////////////////////////////////////////nothing to change here, magic happenes elsewhere - void Base64::decode(const String & in, ByteOrder from_byte_order, std::vector & out, bool zlib_compression) + void Base64::decode(const String& in, ByteOrder from_byte_order, std::vector& out, bool zlib_compression, size_t default_arr_length) { if (zlib_compression) { - decodeCompressed_(in, from_byte_order, out); + decodeCompressed_(in, from_byte_order, out, default_arr_length); } else { @@ -275,18 +275,17 @@ namespace OpenMS template - void Base64::decodeCompressed_(const String & in, ByteOrder from_byte_order, std::vector & out) + void Base64::decodeCompressed_(const String& in, ByteOrder from_byte_order, std::vector& out, size_t default_arr_length) { out.clear(); if (in.empty()) return; constexpr Size element_size = sizeof(ToType); - String decompressed; String s; stringSimdDecoder_(in, s); - QByteArray bazip = QByteArray::fromRawData(s.c_str(), (int) s.size()); + /* QByteArray bazip = QByteArray::fromRawData(s.c_str(), (int)s.size()); /////////////////////////////////////////////////////////////////////////////////////if faster: first encode then call fromRawData // QByteArray qt_byte_array = QByteArray::fromRawData(in.c_str(), (int) in.size()); @@ -298,16 +297,15 @@ namespace OpenMS czip[2] = (bazip.size() & 0x0000ff00) >> 8; czip[3] = (bazip.size() & 0x000000ff); czip += bazip; - QByteArray base64_uncompressed = qUncompress(czip); + */ + String decompressed; + + ZlibCompression::uncompressString((const uchar*)s.data(), s.size(), decompressed, default_arr_length); - if (base64_uncompressed.isEmpty()) + if (decompressed.empty()) { throw Exception::ConversionError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression error?"); } - decompressed.resize(base64_uncompressed.size()); - - std::copy(base64_uncompressed.begin(), base64_uncompressed.end(), decompressed.begin()); - void* byte_buffer = reinterpret_cast(&decompressed[0]); Size buffer_size = decompressed.size(); @@ -417,11 +415,11 @@ namespace OpenMS } template - void Base64::decodeIntegers(const String & in, ByteOrder from_byte_order, std::vector & out, bool zlib_compression) + void Base64::decodeIntegers(const String& in, ByteOrder from_byte_order, std::vector& out, bool zlib_compression, size_t default_arr_length) { if (zlib_compression) { - decodeIntegersCompressed_(in, from_byte_order, out); + decodeIntegersCompressed_(in, from_byte_order, out, default_arr_length); } else { @@ -430,7 +428,7 @@ namespace OpenMS } template - void Base64::decodeIntegersCompressed_(const String & in, ByteOrder from_byte_order, std::vector & out) + void Base64::decodeIntegersCompressed_(const String& in, ByteOrder from_byte_order, std::vector& out, size_t default_arr_length) { out.clear(); if (in.empty()) @@ -441,24 +439,11 @@ namespace OpenMS constexpr Size element_size = sizeof(ToType); String decompressed; - - QByteArray qt_byte_array = QByteArray::fromRawData(in.c_str(), (int) in.size()); - QByteArray bazip = QByteArray::fromBase64(qt_byte_array); - QByteArray czip; - czip.resize(4); - czip[0] = (bazip.size() & 0xff000000) >> 24; - czip[1] = (bazip.size() & 0x00ff0000) >> 16; - czip[2] = (bazip.size() & 0x0000ff00) >> 8; - czip[3] = (bazip.size() & 0x000000ff); - czip += bazip; - QByteArray base64_uncompressed = qUncompress(czip); - if (base64_uncompressed.isEmpty()) + ZlibCompression::uncompressString((const void*)in.data(), in.size(), decompressed, default_arr_length); + if (decompressed.empty()) { throw Exception::ConversionError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression error?"); } - decompressed.resize(base64_uncompressed.size()); - - std::copy(base64_uncompressed.begin(), base64_uncompressed.end(), decompressed.begin()); byte_buffer = reinterpret_cast(&decompressed[0]); buffer_size = decompressed.size(); diff --git a/src/openms/include/OpenMS/FORMAT/HANDLERS/MzMLHandlerHelper.h b/src/openms/include/OpenMS/FORMAT/HANDLERS/MzMLHandlerHelper.h index 4eb4d9cf0bc..0e61cdc7f8f 100644 --- a/src/openms/include/OpenMS/FORMAT/HANDLERS/MzMLHandlerHelper.h +++ b/src/openms/include/OpenMS/FORMAT/HANDLERS/MzMLHandlerHelper.h @@ -126,7 +126,7 @@ namespace OpenMS @param data_ The input and output @param skipXMLCheck whether to skip cleaning the Base64 arrays and remove whitespaces */ - static void decodeBase64Arrays(std::vector & data_, const bool skipXMLCheck = false); + static void decodeBase64Arrays(std::vector& data_, size_t default_arr_length, const bool skipXMLCheck = false); /** @brief Identify a data array from a list. diff --git a/src/openms/include/OpenMS/FORMAT/ZlibCompression.h b/src/openms/include/OpenMS/FORMAT/ZlibCompression.h index acd2063cafb..f3402a73ed3 100644 --- a/src/openms/include/OpenMS/FORMAT/ZlibCompression.h +++ b/src/openms/include/OpenMS/FORMAT/ZlibCompression.h @@ -61,25 +61,34 @@ namespace OpenMS */ static void compressString(const QByteArray& raw_data, QByteArray& compressed_data); + /** - * @brief Uncompresses data using Qt (wrapper around Qt function) + * @brief Uncompresses data using zlib + * Compared to variant of this function, which does not use output_size parameter, this is faster and preferred. + * Be aware that this function preallocates amount of memory corresponding to ouput_size. * * @param compressed_data Compressed data * @param nr_bytes Number of bytes in compressed data * @param raw_data Uncompressed result data + * @param output_size Size of uncompressed data + * @throws Exception::InvalidValue if output_size specified turns out to be smaller than actual size of uncompressed data. + * @throws Exception::InternalToolError if zlib tools used by function fail for unknown reason. * */ - static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data); + static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data, size_t output_size); /** - * @brief Uncompresses data using Qt + * @brief Uncompresses data using zlib + * When size of data after decompression is known, it is recommended to use another variant of this function, which allows specifying output size. + * Does not support gzip format decompression. * * @param compressed_data Compressed data + * @param nr_bytes Number of bytes in compressed data * @param raw_data Uncompressed result data + * @throws Exception::InternalToolError if zlib tools used by function fail for unknown reason. This might happen if decompression from unsupported gzip format is attempted. * */ - static void uncompressString(const QByteArray& compressed_data, QByteArray& raw_data); - + static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data); }; } // namespace OpenMS diff --git a/src/openms/source/FORMAT/Base64.cpp b/src/openms/source/FORMAT/Base64.cpp index b3c37ddda30..053cd88c412 100644 --- a/src/openms/source/FORMAT/Base64.cpp +++ b/src/openms/source/FORMAT/Base64.cpp @@ -237,7 +237,7 @@ namespace OpenMS } - void Base64::decodeStrings(const String& in, std::vector& out, bool zlib_compression) + void Base64::decodeStrings(const String& in, std::vector& out, bool zlib_compression, size_t default_arr_length) { out.clear(); @@ -249,7 +249,7 @@ namespace OpenMS } QByteArray base64_uncompressed; - decodeSingleString(in, base64_uncompressed, zlib_compression); //////////////////////////////////////////////the magic happenes here + decodeSingleString(in, base64_uncompressed, zlib_compression, default_arr_length); //////////////////////////////////////////////the magic happenes here QList null_strings = base64_uncompressed.split('\0'); for (QList::iterator it = null_strings.begin(); it < null_strings.end(); ++it) { @@ -260,7 +260,7 @@ namespace OpenMS } } - void Base64::decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression) + void Base64::decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression, size_t default_arr_length) { // The length of a base64 string is a always a multiple of 4 (always 3 // bytes are encoded as 4 characters) @@ -273,19 +273,10 @@ namespace OpenMS base64_uncompressed = QByteArray::fromBase64(herewego); if (zlib_compression) { - QByteArray czip; - czip.resize(4); - czip[0] = (base64_uncompressed.size() & 0xff000000) >> 24; - czip[1] = (base64_uncompressed.size() & 0x00ff0000) >> 16; - czip[2] = (base64_uncompressed.size() & 0x0000ff00) >> 8; - czip[3] = (base64_uncompressed.size() & 0x000000ff); - czip += base64_uncompressed; - base64_uncompressed = qUncompress(czip); - - if (base64_uncompressed.isEmpty()) - { - throw Exception::ConversionError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression error?"); - } + String uncompressed; + ZlibCompression::uncompressString(base64_uncompressed, base64_uncompressed.size(), uncompressed, default_arr_length); + base64_uncompressed = QByteArray::fromRawData(uncompressed.c_str(), (int)uncompressed.size()); + } } diff --git a/src/openms/source/FORMAT/HANDLERS/MzMLHandler.cpp b/src/openms/source/FORMAT/HANDLERS/MzMLHandler.cpp index 958b5a1510d..6014be6ba8c 100644 --- a/src/openms/source/FORMAT/HANDLERS/MzMLHandler.cpp +++ b/src/openms/source/FORMAT/HANDLERS/MzMLHandler.cpp @@ -121,42 +121,36 @@ namespace OpenMS::Internal // Whether spectrum should be populated with data if (options_.getFillData()) { - size_t errCount = 0; + std::atomic errCount = 0; String error_message; -#ifdef _OPENMP -#pragma omp parallel for -#endif + #pragma omp parallel for for (SignedSize i = 0; i < (SignedSize)spectrum_data_.size(); i++) { // parallel exception catching and re-throwing business - if (!errCount) // no need to parse further if already an error was encountered + try { - try + populateSpectraWithData_(spectrum_data_[i].data, + spectrum_data_[i].default_array_length, + options_, + spectrum_data_[i].spectrum); + if (options_.getSortSpectraByMZ() && !spectrum_data_[i].spectrum.isSorted()) { - populateSpectraWithData_(spectrum_data_[i].data, - spectrum_data_[i].default_array_length, - options_, - spectrum_data_[i].spectrum); - if (options_.getSortSpectraByMZ() && !spectrum_data_[i].spectrum.isSorted()) - { - spectrum_data_[i].spectrum.sortByPosition(); - } + spectrum_data_[i].spectrum.sortByPosition(); } + } - catch (OpenMS::Exception::BaseException& e) - { -#pragma omp critical(MZMLErrorHandling) - { - ++errCount; - error_message = e.what(); - } - } - catch (...) + catch (OpenMS::Exception::BaseException& e) + { + #pragma omp critical(MZMLErrorHandling) { -#pragma omp atomic ++errCount; + error_message = e.what(); } } + catch (...) + { + ++errCount; + } } if (errCount != 0) { @@ -264,7 +258,7 @@ namespace OpenMS::Internal typedef SpectrumType::PeakType PeakType; // decode all base64 arrays - MzMLHandlerHelper::decodeBase64Arrays(input_data, options_.getSkipXMLChecks()); + MzMLHandlerHelper::decodeBase64Arrays(input_data, default_arr_length, options_.getSkipXMLChecks()); //look up the precision and the index of the intensity and m/z array bool mz_precision_64 = true; @@ -521,7 +515,7 @@ namespace OpenMS::Internal typedef ChromatogramType::PeakType ChromatogramPeakType; //decode all base64 arrays - MzMLHandlerHelper::decodeBase64Arrays(input_data, options_.getSkipXMLChecks()); + MzMLHandlerHelper::decodeBase64Arrays(input_data, options_.getSkipXMLChecks(), default_arr_length); //look up the precision and the index of the intensity and m/z array bool int_precision_64 = true; diff --git a/src/openms/source/FORMAT/HANDLERS/MzMLHandlerHelper.cpp b/src/openms/source/FORMAT/HANDLERS/MzMLHandlerHelper.cpp index c9748462c03..c912a6ca614 100644 --- a/src/openms/source/FORMAT/HANDLERS/MzMLHandlerHelper.cpp +++ b/src/openms/source/FORMAT/HANDLERS/MzMLHandlerHelper.cpp @@ -135,7 +135,8 @@ namespace OpenMS::Internal } } - void MzMLHandlerHelper::decodeBase64Arrays(std::vector& data, const bool skipXMLCheck) + void + MzMLHandlerHelper::decodeBase64Arrays(std::vector& data, size_t default_arr_length, const bool skipXMLCheck) { // decode all base64 arrays for (auto& bindata : data) @@ -178,7 +179,7 @@ namespace OpenMS::Internal // decoder always works with 64 bit (takes std::vector) MSNumpressCoder::NumpressConfig config; config.np_compression = bindata.np_compression; - MSNumpressCoder().decodeNP(bindata.base64, bindata.floats_64, bindata.compression, config); + MSNumpressCoder().decodeNP(bindata.base64, bindata.floats_64, bindata.compression, config); // todo: use default_arr_length // Next, ensure that we only look at the float array even if the // mzML tags say 32 bit data (I am looking at you, proteowizard) @@ -186,7 +187,7 @@ namespace OpenMS::Internal } else if (bindata.precision == BinaryData::PRE_64) { - Base64::decode(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.floats_64, bindata.compression); + Base64::decode(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.floats_64, bindata.compression, default_arr_length * 8); if (bindata.size != bindata.floats_64.size()) { MzMLHandlerHelper::warning(0, String("Float binary data array '") + bindata.meta.getName() + @@ -196,7 +197,7 @@ namespace OpenMS::Internal } else if (bindata.precision == BinaryData::PRE_32) { - Base64::decode(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.floats_32, bindata.compression); + Base64::decode(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.floats_32, bindata.compression, default_arr_length * 4); if (bindata.size != bindata.floats_32.size()) { MzMLHandlerHelper::warning(0, String("Float binary data array '") + bindata.meta.getName() + @@ -226,7 +227,7 @@ namespace OpenMS::Internal { if (bindata.precision == BinaryData::PRE_64) { - Base64::decodeIntegers(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.ints_64, bindata.compression); + Base64::decodeIntegers(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.ints_64, bindata.compression, default_arr_length * 8); if (bindata.size != bindata.ints_64.size()) { MzMLHandlerHelper::warning(0, String("Integer binary data array '") + bindata.meta.getName() + @@ -236,7 +237,7 @@ namespace OpenMS::Internal } else if (bindata.precision == BinaryData::PRE_32) { - Base64::decodeIntegers(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.ints_32, bindata.compression); + Base64::decodeIntegers(bindata.base64, Base64::BYTEORDER_LITTLEENDIAN, bindata.ints_32, bindata.compression, default_arr_length * 4); if (bindata.size != bindata.ints_32.size()) { MzMLHandlerHelper::warning(0, String("Integer binary data array '") + bindata.meta.getName() + @@ -247,7 +248,7 @@ namespace OpenMS::Internal } else if (bindata.data_type == BinaryData::DT_STRING) { - Base64::decodeStrings(bindata.base64, bindata.decoded_char, bindata.compression); + Base64::decodeStrings(bindata.base64, bindata.decoded_char, bindata.compression, default_arr_length); if (bindata.size != bindata.decoded_char.size()) { MzMLHandlerHelper::warning(0, String("String binary data array '") + bindata.meta.getName() + diff --git a/src/openms/source/FORMAT/ZlibCompression.cpp b/src/openms/source/FORMAT/ZlibCompression.cpp index 5bc6dd17b33..9a71d9cbcef 100644 --- a/src/openms/source/FORMAT/ZlibCompression.cpp +++ b/src/openms/source/FORMAT/ZlibCompression.cpp @@ -10,6 +10,9 @@ #include +#include + +#include #include using namespace std; @@ -31,20 +34,18 @@ namespace OpenMS sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + 11; // taken from zlib's compress.c, as we cannot use compressBound* int zlib_error; - do - { - compressed.resize(compressed_length); // reserve enough space -- we may not need all of it - zlib_error = compress(reinterpret_cast(&compressed[0]), &compressed_length, (Bytef*)raw_data, sourceLen); - switch (zlib_error) - { - case Z_MEM_ERROR: - throw Exception::OutOfMemory(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, compressed_length); + compressed.resize(compressed_length); // reserve enough space -- we may not need all of it + zlib_error = compress(reinterpret_cast(&compressed[0]), &compressed_length, (Bytef*)raw_data, sourceLen); - case Z_BUF_ERROR: - compressed_length *= 2; - } - } while (zlib_error == Z_BUF_ERROR); + switch (zlib_error) + { + case Z_MEM_ERROR: + case Z_BUF_ERROR: + throw Exception::OutOfMemory(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, compressed_length); + case Z_OK: // ok + break; + } if (zlib_error != Z_OK) { @@ -59,35 +60,70 @@ namespace OpenMS compressed_data.remove(0, 4); } - void ZlibCompression::uncompressString(const void * tt, size_t blob_bytes, std::string& uncompressed) + void ZlibCompression::uncompressString(const void* compressed_data, size_t nr_bytes, std::string& raw_data, size_t output_size) { - // take a leap of faith and assume the input is valid - QByteArray compressed_data = QByteArray::fromRawData((const char*)tt, (int)blob_bytes); - QByteArray raw_data; + raw_data.resize(output_size); + uLongf uncompressedSize = output_size; + int ret = uncompress((Bytef*)raw_data.data(), &uncompressedSize, (Bytef*)compressed_data, nr_bytes); - ZlibCompression::uncompressString(compressed_data, raw_data); + if (ret == Z_OK) + { + if (uncompressedSize != raw_data.size()) + { + OPENMS_LOG_INFO << "ZlibCompression::uncompressString: Warning: decompressed data was smaller (" << std::to_string(uncompressedSize) << ") than anticipated: " << std::to_string(output_size) << std::endl; + raw_data.resize(output_size); + } + } + else if (ret == Z_BUF_ERROR) + { + throw Exception::InvalidValue(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression failed because specified output_size was too small. Size of data after decompression is larger than anticipated: " + std::to_string(uncompressedSize), std::to_string(output_size)); + } else + { + throw Exception::InternalToolError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "zlib::inflate failed with code " + std::to_string(ret) + " ."); + } - // Note that we may have zero bytes in the string, so we cannot use QString - uncompressed.clear(); - uncompressed = std::string(raw_data.data(), raw_data.size()); } - void ZlibCompression::uncompressString(const QByteArray& compressed_data, QByteArray& raw_data) + void ZlibCompression::uncompressString(const void* compressed_data, size_t nr_bytes, std::string& uncompressed) { - QByteArray czip; - czip.resize(4); - czip[0] = (compressed_data.size() & 0xff000000) >> 24; - czip[1] = (compressed_data.size() & 0x00ff0000) >> 16; - czip[2] = (compressed_data.size() & 0x0000ff00) >> 8; - czip[3] = (compressed_data.size() & 0x000000ff); - czip += compressed_data; - raw_data = qUncompress(czip); - - if (raw_data.isEmpty()) + const size_t CHUNK_SIZE = 16384; + uncompressed.clear(); + z_stream strm = {}; + + // Setup input + strm.next_in = (Bytef*)(compressed_data); + strm.avail_in = nr_bytes; + + int ret; + + // Initialize zlib (use inflateInit2 for gzip or raw deflate) + ret = inflateInit(&strm); + if (ret != Z_OK) { - throw Exception::ConversionError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression error?"); + throw Exception::InternalToolError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "zlib::inflateInit failed with code " + std::to_string(ret) + " ."); } - } + // Decompress loop + std::array buffer; + + do + { + strm.avail_out = CHUNK_SIZE; + strm.next_out = (Bytef*)buffer.data(); + + ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) + { + inflateEnd(&strm); + throw Exception::InternalToolError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "zlib::inflate failed with code " + std::to_string(ret) + " ."); + } + + size_t bytesDecompressed = CHUNK_SIZE - strm.avail_out; + uncompressed.insert(uncompressed.end(), buffer.begin(), buffer.begin() + bytesDecompressed); + + } while (ret != Z_STREAM_END); + + inflateEnd(&strm); + } } diff --git a/src/tests/class_tests/openms/source/ZlibCompression_test.cpp b/src/tests/class_tests/openms/source/ZlibCompression_test.cpp index 3de5597fc17..684d97bff7d 100644 --- a/src/tests/class_tests/openms/source/ZlibCompression_test.cpp +++ b/src/tests/class_tests/openms/source/ZlibCompression_test.cpp @@ -106,78 +106,82 @@ START_SECTION((static void compressString(const QByteArray& raw_data, QByteArray } END_SECTION -START_SECTION((static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data))) +START_SECTION((static void uncompressString(const void* compressed_data, size_t nr_bytes, std::string& raw_data, size_t output_size))) { std::string compressed_data; std::string uncompressed_data; ZlibCompression::compressString(raw_data, compressed_data); - ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data, raw_data.size()); TEST_EQUAL(raw_data.size(), 58) - TEST_TRUE(compressed_data.size() < raw_data.size()) + TEST_TRUE(compressed_data.size() < raw_data.size()) TEST_EQUAL(uncompressed_data.size(), 58) TEST_TRUE(uncompressed_data == raw_data) ZlibCompression::compressString(raw_data2, compressed_data); - ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data, raw_data2.size()); TEST_EQUAL(raw_data2.size(), 64) - TEST_TRUE(compressed_data.size() >= raw_data2.size()) // "ABCD..." string is difficult to compress + TEST_TRUE(compressed_data.size() >= raw_data2.size()) // difficult to compress... TEST_EQUAL(uncompressed_data.size(), 64) TEST_TRUE(uncompressed_data == raw_data2) ZlibCompression::compressString(raw_data3, compressed_data); - ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data, raw_data3.size()); TEST_EQUAL(raw_data3.size(), 105) - TEST_TRUE(compressed_data.size() < raw_data3.size()) + TEST_TRUE(compressed_data.size() < raw_data3.size()) TEST_EQUAL(uncompressed_data.size(), 105) TEST_TRUE(uncompressed_data == raw_data3) ZlibCompression::compressString(raw_data4, compressed_data); - ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data, raw_data4.size()); TEST_EQUAL(raw_data4.size(), 1052) - TEST_TRUE(compressed_data.size() < raw_data4.size()) + TEST_TRUE(compressed_data.size() < raw_data4.size()) TEST_EQUAL(uncompressed_data.size(), 1052) TEST_TRUE(uncompressed_data == raw_data4) + + //////////////// + // Exceptions // + //////////////// + ZlibCompression::compressString(raw_data, compressed_data); + // Invalid output_size + TEST_EXCEPTION(Exception::InvalidValue, ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data, 10);) + // Truncated data + TEST_EXCEPTION(Exception::InternalToolError, ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size()-10, uncompressed_data, raw_data.size());) } END_SECTION - -START_SECTION((static void uncompressString(const QByteArray& compressed_data, QByteArray& raw_data))) -{ - QByteArray raw_data_q = QByteArray::fromRawData(&raw_data[0], raw_data.size()); - QByteArray raw_data_q2 = QByteArray::fromRawData(&raw_data2[0], raw_data2.size()); - QByteArray raw_data_q3 = QByteArray::fromRawData(&raw_data3[0], raw_data3.size()); - QByteArray raw_data_q4 = QByteArray::fromRawData(&raw_data4[0], raw_data4.size()); - QByteArray compressed_data; - QByteArray uncompressed_data; +START_SECTION((static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data))) +{ + std::string compressed_data; + std::string uncompressed_data; - ZlibCompression::compressString(raw_data_q, compressed_data); - ZlibCompression::uncompressString(compressed_data, uncompressed_data); - TEST_EQUAL(raw_data_q.size(), 58) - TEST_TRUE(compressed_data.size() < raw_data_q.size()) + ZlibCompression::compressString(raw_data, compressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + TEST_EQUAL(raw_data.size(), 58) + TEST_TRUE(compressed_data.size() < raw_data.size()) TEST_EQUAL(uncompressed_data.size(), 58) - TEST_TRUE(uncompressed_data == raw_data_q) + TEST_TRUE(uncompressed_data == raw_data) - ZlibCompression::compressString(raw_data_q2, compressed_data); - ZlibCompression::uncompressString(compressed_data, uncompressed_data); - TEST_EQUAL(raw_data_q2.size(), 64) - TEST_TRUE(compressed_data.size() >= raw_data_q2.size()) // difficult to compress... + ZlibCompression::compressString(raw_data2, compressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + TEST_EQUAL(raw_data2.size(), 64) + TEST_TRUE(compressed_data.size() >= raw_data2.size()) // "ABCD..." string is difficult to compress TEST_EQUAL(uncompressed_data.size(), 64) - TEST_TRUE(uncompressed_data == raw_data_q2) + TEST_TRUE(uncompressed_data == raw_data2) - ZlibCompression::compressString(raw_data_q3, compressed_data); - ZlibCompression::uncompressString(compressed_data, uncompressed_data); - TEST_EQUAL(raw_data_q3.size(), 105) - TEST_TRUE(compressed_data.size() < raw_data_q3.size()) + ZlibCompression::compressString(raw_data3, compressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + TEST_EQUAL(raw_data3.size(), 105) + TEST_TRUE(compressed_data.size() < raw_data3.size()) TEST_EQUAL(uncompressed_data.size(), 105) - TEST_TRUE(uncompressed_data == raw_data_q3) + TEST_TRUE(uncompressed_data == raw_data3) - ZlibCompression::compressString(raw_data_q4, compressed_data); - ZlibCompression::uncompressString(compressed_data, uncompressed_data); - TEST_EQUAL(raw_data_q4.size(), 1052) - TEST_TRUE(compressed_data.size() < raw_data_q4.size()) + ZlibCompression::compressString(raw_data4, compressed_data); + ZlibCompression::uncompressString(&compressed_data[0], compressed_data.size(), uncompressed_data); + TEST_EQUAL(raw_data4.size(), 1052) + TEST_TRUE(compressed_data.size() < raw_data4.size()) TEST_EQUAL(uncompressed_data.size(), 1052) - TEST_TRUE(uncompressed_data == raw_data_q4) + TEST_TRUE(uncompressed_data == raw_data4) } END_SECTION