From abb632fb0f778258e9a8c8c1b3a262f804755fdb Mon Sep 17 00:00:00 2001
From: Arne Welzel <arne.welzel@corelight.com>
Date: Thu, 8 May 2025 19:21:50 +0200
Subject: [PATCH] IXWebSocketTransport: Avoid bloating _rxbuf

Buffering too much data into _rxbuf results in dispatch() processing to
run into a performance issue due to calling _rxbuf.erase() for each
frame.

Concretely, if _rxbuf grows large due to a client sending frames very
fast, each processing in dispatch() results in moving the remaining
buffer to the front.

Instead of restructuring to avoid .erase(), this patch limits the
maximum size of _rxbuf to kChunkSize to alleviate the O^2 erase()
overhead. It also has the side-effect of keeping the received data
in the OS's TCP stack, building up back pressure to the client
earlier if the server code can't keep up.

Fixes #429
---
 ixwebsocket/IXWebSocketTransport.cpp | 15 +++++++++++++++
 ixwebsocket/IXWebSocketTransport.h   |  4 ++++
 2 files changed, 19 insertions(+)

diff --git a/ixwebsocket/IXWebSocketTransport.cpp b/ixwebsocket/IXWebSocketTransport.cpp
index 23072f34..1d381ecb 100644
--- a/ixwebsocket/IXWebSocketTransport.cpp
+++ b/ixwebsocket/IXWebSocketTransport.cpp
@@ -547,9 +547,12 @@ namespace ix
 
             if (_rxbuf.size() < ws.header_size + ws.N)
             {
+                _rxbufWanted = ws.header_size + ws.N;
                 return; /* Need: ws.header_size+ws.N - _rxbuf.size() */
             }
 
+            _rxbufWanted = 0;
+
             if (!ws.fin && (ws.opcode == wsheader_type::PING || ws.opcode == wsheader_type::PONG ||
                             ws.opcode == wsheader_type::CLOSE))
             {
@@ -1098,6 +1101,18 @@ namespace ix
     {
         while (true)
         {
+            // If _rxbufWanted isn't set, don't attempt to read more than kChunkSize
+            // into _rxbuf. If a client is sending frames faster than they can be
+            // processed this would otherwise bloat _rxbuf and further introduce
+            // unnecessary processing overhead due to .erase() in dispatch().
+            //
+            // Further, not reading everything from the socket will eventually
+            // result in back pressure for the client.
+            if (_rxbufWanted == 0 && _rxbuf.size() >= kChunkSize) break;
+
+            // There's also no point in reading more bytes than needed.
+            if (_rxbufWanted > 0 && _rxbuf.size() >= _rxbufWanted) break;
+
             ssize_t ret = _socket->recv((char*) &_readbuf[0], _readbuf.size());
 
             if (ret < 0 && Socket::isWaitNeeded())
diff --git a/ixwebsocket/IXWebSocketTransport.h b/ixwebsocket/IXWebSocketTransport.h
index 97cfc15a..c89e6015 100644
--- a/ixwebsocket/IXWebSocketTransport.h
+++ b/ixwebsocket/IXWebSocketTransport.h
@@ -158,6 +158,10 @@ namespace ix
         // data messages. That buffer is resized
         std::vector<uint8_t> _rxbuf;
 
+        // If set to a positive value, only read bytes from the socket until
+        // _rxbuf has reached this size to avoid unnecessary erase churn.
+        uint64_t _rxbufWanted = 0;
+
         // Contains all messages that are waiting to be sent
         std::vector<uint8_t> _txbuf;
         mutable std::mutex _txbufMutex;