Zlib noqt

src/openms/include/OpenMS/FORMAT/Base64.h

@@ -73,7 +73,7 @@ namespace OpenMS
         You have to specify the byte order of the input and if it is zlib-compressed.
     */
     template <typename ToType>
-    static void decode(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out, bool zlib_compression = false);
+    static void decode(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out, bool zlib_compression = false, size_t default_arr_length = 0);
 
     /**
         @brief Encodes a vector of integer point numbers to a Base64 string
@@ -83,15 +83,15 @@ namespace OpenMS
         @note @p in will be empty after this method
     */
     template <typename FromType>
-    static void encodeIntegers(std::vector<FromType> & in, ByteOrder to_byte_order, String & out, bool zlib_compression = false);
+    static void encodeIntegers(std::vector<FromType>& in, ByteOrder to_byte_order, String& out, bool zlib_compression = false);
 
     /**
         @brief Decodes a Base64 string to a vector of integer numbers
 
         You have to specify the byte order of the input and if it is zlib-compressed.
     */
     template <typename ToType>
-    static void decodeIntegers(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out, bool zlib_compression = false);
+    static void decodeIntegers(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out, bool zlib_compression = false, size_t default_arr_length = 0);
 
     /**
         @brief Encodes a vector of strings to a Base64 string
@@ -116,7 +116,7 @@ namespace OpenMS
         @param out A vector containing the decoded data (split at null "\0") bytes
         @param zlib_compression Whether the data should be decompressed with zlib after decoding in Base64
     */
-    static void decodeStrings(const String & in, std::vector<String> & out, bool zlib_compression = false);
+    static void decodeStrings(const String& in, std::vector<String>& out, bool zlib_compression = false, size_t default_arr_length = 0);
 
     /**
         @brief Decodes a Base64 string to a QByteArray
@@ -125,7 +125,7 @@ namespace OpenMS
         @param base64_uncompressed A ByteArray containing the decoded data
         @param zlib_compression Whether the data should be decompressed with zlib after decoding in Base64
     */
-    static void decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression);
+    static void decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression, size_t default_arr_length = 0);
 
 private:
 
@@ -151,15 +151,15 @@ namespace OpenMS
 
     ///Decodes a compressed Base64 string to a vector of floating point numbers
     template <typename ToType>
-    static void decodeCompressed_(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out);
+    static void decodeCompressed_(const String& in, ByteOrder from_byte_order, std::vector<ToType>& out, size_t default_arr_length);
 
     /// Decodes a Base64 string to a vector of integer numbers
     template <typename ToType>
     static void decodeIntegersUncompressed_(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out);
 
     ///Decodes a compressed Base64 string to a vector of integer numbers
     template <typename ToType>
-    static void decodeIntegersCompressed_(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out);
+    static void decodeIntegersCompressed_(const String& in, ByteOrder from_byte_order, std::vector<ToType>& out, size_t default_arr_length);
 
     static void stringSimdEncoder_(std::string& in, std::string& out);
 
@@ -246,11 +246,11 @@ namespace OpenMS
   }
 
   template <typename ToType>  ////////////////////////////////////////////nothing to change here, magic happenes elsewhere
-  void Base64::decode(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out, bool zlib_compression)
+  void Base64::decode(const String& in, ByteOrder from_byte_order, std::vector<ToType>& out, bool zlib_compression, size_t default_arr_length)
   {
     if (zlib_compression)
     {
-      decodeCompressed_(in, from_byte_order, out);
+      decodeCompressed_(in, from_byte_order, out, default_arr_length);
     }
     else
     {
@@ -275,18 +275,17 @@ namespace OpenMS
 
 
   template <typename ToType>
-  void Base64::decodeCompressed_(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out)
+  void Base64::decodeCompressed_(const String& in, ByteOrder from_byte_order, std::vector<ToType>& out, size_t default_arr_length)
   {
     out.clear();
     if (in.empty()) return;
 
     constexpr Size element_size = sizeof(ToType);
 
-    String decompressed;
 
     String s;
     stringSimdDecoder_(in, s);
-    QByteArray bazip = QByteArray::fromRawData(s.c_str(), (int) s.size());
+    /* QByteArray bazip = QByteArray::fromRawData(s.c_str(), (int)s.size());
 
    /////////////////////////////////////////////////////////////////////////////////////if faster: first encode then call fromRawData
    // QByteArray qt_byte_array = QByteArray::fromRawData(in.c_str(), (int) in.size());
@@ -298,16 +297,15 @@ namespace OpenMS
     czip[2] = (bazip.size() & 0x0000ff00) >> 8;
     czip[3] = (bazip.size() & 0x000000ff);
     czip += bazip;
-    QByteArray base64_uncompressed = qUncompress(czip);
+    */
+    String decompressed;
+
+    ZlibCompression::uncompressString((const uchar*)s.data(), s.size(), decompressed, default_arr_length);
 
-    if (base64_uncompressed.isEmpty())
+    if (decompressed.empty())
     {
       throw Exception::ConversionError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression error?");
     }
-    decompressed.resize(base64_uncompressed.size());
-
-    std::copy(base64_uncompressed.begin(), base64_uncompressed.end(), decompressed.begin());
-
     void* byte_buffer = reinterpret_cast<void *>(&decompressed[0]);
     Size buffer_size = decompressed.size();
 
@@ -417,11 +415,11 @@ namespace OpenMS
   }
 
   template <typename ToType>
-  void Base64::decodeIntegers(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out, bool zlib_compression)
+  void Base64::decodeIntegers(const String& in, ByteOrder from_byte_order, std::vector<ToType>& out, bool zlib_compression, size_t default_arr_length)
   {
     if (zlib_compression)
     {
-      decodeIntegersCompressed_(in, from_byte_order, out);
+      decodeIntegersCompressed_(in, from_byte_order, out, default_arr_length);
     }
     else
     {
@@ -430,7 +428,7 @@ namespace OpenMS
   }
 
   template <typename ToType>
-  void Base64::decodeIntegersCompressed_(const String & in, ByteOrder from_byte_order, std::vector<ToType> & out)
+  void Base64::decodeIntegersCompressed_(const String& in, ByteOrder from_byte_order, std::vector<ToType>& out, size_t default_arr_length)
   {
     out.clear();
     if (in.empty())
@@ -441,24 +439,11 @@ namespace OpenMS
     constexpr Size element_size = sizeof(ToType);
 
     String decompressed;
-
-    QByteArray qt_byte_array = QByteArray::fromRawData(in.c_str(), (int) in.size());
-    QByteArray bazip = QByteArray::fromBase64(qt_byte_array);
-    QByteArray czip;
-    czip.resize(4);
-    czip[0] = (bazip.size() & 0xff000000) >> 24;
-    czip[1] = (bazip.size() & 0x00ff0000) >> 16;
-    czip[2] = (bazip.size() & 0x0000ff00) >> 8;
-    czip[3] = (bazip.size() & 0x000000ff);
-    czip += bazip;
-    QByteArray base64_uncompressed = qUncompress(czip);
-    if (base64_uncompressed.isEmpty())
+    ZlibCompression::uncompressString((const void*)in.data(), in.size(), decompressed, default_arr_length);
+    if (decompressed.empty())
     {
       throw Exception::ConversionError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression error?");
     }
-    decompressed.resize(base64_uncompressed.size());
-
-    std::copy(base64_uncompressed.begin(), base64_uncompressed.end(), decompressed.begin());
 
     byte_buffer = reinterpret_cast<void *>(&decompressed[0]);
     buffer_size = decompressed.size();

src/openms/include/OpenMS/FORMAT/HANDLERS/MzMLHandlerHelper.h

@@ -126,7 +126,7 @@ namespace OpenMS
         @param data_ The input and output
         @param skipXMLCheck whether to skip cleaning the Base64 arrays and remove whitespaces
       */
-      static void decodeBase64Arrays(std::vector<BinaryData> & data_, const bool skipXMLCheck = false);
+      static void decodeBase64Arrays(std::vector<BinaryData>& data_, size_t default_arr_length, const bool skipXMLCheck = false);
 
       /**
         @brief Identify a data array from a list.

src/openms/include/OpenMS/FORMAT/ZlibCompression.h

@@ -61,25 +61,34 @@ namespace OpenMS
     */
     static void compressString(const QByteArray& raw_data, QByteArray& compressed_data);
 
+
     /**
-      * @brief Uncompresses data using Qt (wrapper around Qt function)
+      * @brief Uncompresses data using zlib
+      * Compared to variant of this function, which does not use output_size parameter, this is faster and preferred.
+      * Be aware that this function preallocates amount of memory corresponding to ouput_size.
       *
       * @param compressed_data Compressed data
       * @param nr_bytes Number of bytes in compressed data
       * @param raw_data Uncompressed result data
+      * @param output_size Size of uncompressed data
+      * @throws Exception::InvalidValue if output_size specified turns out to be smaller than actual size of uncompressed data.
+      * @throws Exception::InternalToolError if zlib tools used by function fail for unknown reason.
       * 
     */
-    static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data);
+    static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data, size_t output_size);
 
     /**
-      * @brief Uncompresses data using Qt
+      * @brief Uncompresses data using zlib
+      * When size of data after decompression is known, it is recommended to use another variant of this function, which allows specifying output size.
+      * Does not support gzip format decompression.
       *
       * @param compressed_data Compressed data
+      * @param nr_bytes Number of bytes in compressed data
       * @param raw_data Uncompressed result data
+      * @throws Exception::InternalToolError if zlib tools used by function fail for unknown reason. This might happen if decompression from unsupported gzip format is attempted.
       * 
     */
-    static void uncompressString(const QByteArray& compressed_data, QByteArray& raw_data);
-
+    static void uncompressString(const void * compressed_data, size_t nr_bytes, std::string& raw_data);
   };
 
 } // namespace OpenMS

src/openms/source/FORMAT/Base64.cpp

@@ -237,7 +237,7 @@ namespace OpenMS
   }
 
 
-  void Base64::decodeStrings(const String& in, std::vector<String>& out, bool zlib_compression)
+  void Base64::decodeStrings(const String& in, std::vector<String>& out, bool zlib_compression, size_t default_arr_length)
   {
     out.clear();
 
@@ -249,7 +249,7 @@ namespace OpenMS
     }
 
     QByteArray base64_uncompressed;
-    decodeSingleString(in, base64_uncompressed, zlib_compression);    //////////////////////////////////////////////the magic happenes here
+    decodeSingleString(in, base64_uncompressed, zlib_compression, default_arr_length);    //////////////////////////////////////////////the magic happenes here
     QList<QByteArray> null_strings = base64_uncompressed.split('\0');
     for (QList<QByteArray>::iterator it = null_strings.begin(); it < null_strings.end(); ++it)
     {
@@ -260,7 +260,7 @@ namespace OpenMS
     }
   }
 
-  void Base64::decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression)
+  void Base64::decodeSingleString(const String& in, QByteArray& base64_uncompressed, bool zlib_compression, size_t default_arr_length)
   {
     // The length of a base64 string is a always a multiple of 4 (always 3
     // bytes are encoded as 4 characters)
@@ -273,19 +273,10 @@ namespace OpenMS
     base64_uncompressed = QByteArray::fromBase64(herewego);
     if (zlib_compression)
     {
-      QByteArray czip;
-      czip.resize(4);
-      czip[0] = (base64_uncompressed.size() & 0xff000000) >> 24;
-      czip[1] = (base64_uncompressed.size() & 0x00ff0000) >> 16;
-      czip[2] = (base64_uncompressed.size() & 0x0000ff00) >> 8;
-      czip[3] = (base64_uncompressed.size() & 0x000000ff);
-      czip += base64_uncompressed;
-      base64_uncompressed = qUncompress(czip);
-
-      if (base64_uncompressed.isEmpty())
-      {
-        throw Exception::ConversionError(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Decompression error?");
-      }
+      String uncompressed;
+      ZlibCompression::uncompressString(base64_uncompressed, base64_uncompressed.size(), uncompressed, default_arr_length);
+      base64_uncompressed = QByteArray::fromRawData(uncompressed.c_str(), (int)uncompressed.size());
+
     }
   }
 

src/openms/source/FORMAT/HANDLERS/MzMLHandler.cpp

@@ -121,42 +121,36 @@ namespace OpenMS::Internal
       // Whether spectrum should be populated with data
       if (options_.getFillData())
       {
-        size_t errCount = 0;
+        std::atomic<int> errCount = 0;
         String error_message;
-#ifdef _OPENMP
-#pragma omp parallel for
-#endif
+        #pragma omp parallel for
         for (SignedSize i = 0; i < (SignedSize)spectrum_data_.size(); i++)
         {
           // parallel exception catching and re-throwing business
-          if (!errCount) // no need to parse further if already an error was encountered
+          try
           {
-            try
+            populateSpectraWithData_(spectrum_data_[i].data,
+                                      spectrum_data_[i].default_array_length,
+                                      options_,
+                                      spectrum_data_[i].spectrum);
+            if (options_.getSortSpectraByMZ() && !spectrum_data_[i].spectrum.isSorted())
             {
-              populateSpectraWithData_(spectrum_data_[i].data,
-                                       spectrum_data_[i].default_array_length,
-                                       options_,
-                                       spectrum_data_[i].spectrum);
-              if (options_.getSortSpectraByMZ() && !spectrum_data_[i].spectrum.isSorted())
-              {
-                spectrum_data_[i].spectrum.sortByPosition();
-              }
+              spectrum_data_[i].spectrum.sortByPosition();
             }
+          }
 
-            catch (OpenMS::Exception::BaseException& e)
-            {
-#pragma omp critical(MZMLErrorHandling)
-              {
-                ++errCount;
-                error_message = e.what();
-              }
-            }
-            catch (...)
+          catch (OpenMS::Exception::BaseException& e)
+          {
+            #pragma omp critical(MZMLErrorHandling)
             {
-#pragma omp atomic
               ++errCount;
+              error_message = e.what();
             }
           }
+          catch (...)
+          {
+            ++errCount;
+          }
         }
         if (errCount != 0)
         {
@@ -264,7 +258,7 @@ namespace OpenMS::Internal
       typedef SpectrumType::PeakType PeakType;
 
       // decode all base64 arrays
-      MzMLHandlerHelper::decodeBase64Arrays(input_data, options_.getSkipXMLChecks());
+      MzMLHandlerHelper::decodeBase64Arrays(input_data, default_arr_length, options_.getSkipXMLChecks());
 
       //look up the precision and the index of the intensity and m/z array
       bool mz_precision_64 = true;
@@ -521,7 +515,7 @@ namespace OpenMS::Internal
       typedef ChromatogramType::PeakType ChromatogramPeakType;
 
       //decode all base64 arrays
-      MzMLHandlerHelper::decodeBase64Arrays(input_data, options_.getSkipXMLChecks());
+      MzMLHandlerHelper::decodeBase64Arrays(input_data, options_.getSkipXMLChecks(), default_arr_length);
 
       //look up the precision and the index of the intensity and m/z array
       bool int_precision_64 = true;