From 77ac809329cf94833677dada799a43d94955f3c0 Mon Sep 17 00:00:00 2001 From: Sy Brand Date: Wed, 29 Apr 2026 15:05:37 +0100 Subject: [PATCH 1/3] Fix body read all for large sizes --- .../fixtures/module-mode/src/kv-store.js | 18 ++++++++++++++++++ .../js-compute/fixtures/module-mode/tests.json | 1 + 2 files changed, 19 insertions(+) diff --git a/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js b/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js index 961b476a01..42702f2208 100644 --- a/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js +++ b/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js @@ -1985,6 +1985,24 @@ async function kvStoreInterfaceTests() { ); } +// Test that HttpBody::read_all() correctly handles large list responses +// (i.e. that it uses a growable buffer and doesn't overflow). +// We populate the store with enough 1024-char keys that the list response JSON +// would have exceeded the old 500000-byte fixed buffer. +routes.set('/kv-store/list/large-response', async () => { + const store = new KVStore(KV_STORE_NAME); + // 490 keys × ~1024-char names ≈ 507 KB of JSON + const prefix = 'large-list-' + 'a'.repeat(1013); + await Promise.all( + Array.from({ length: 490 }, (_, i) => + store.put(prefix + String(i).padStart(4, '0'), 'x'), + ), + ); + const result = await store.list({}); + strictEqual(typeof result, 'object', 'list result is an object'); + return new Response('ok'); +}); + function iteratableToStream(iterable) { return new ReadableStream({ async pull(controller) { diff --git a/integration-tests/js-compute/fixtures/module-mode/tests.json b/integration-tests/js-compute/fixtures/module-mode/tests.json index a5e2ff72cc..a82160a89c 100644 --- a/integration-tests/js-compute/fixtures/module-mode/tests.json +++ b/integration-tests/js-compute/fixtures/module-mode/tests.json @@ -412,6 +412,7 @@ "GET /kv-store-options/gen-invalid": { "flake": true }, "GET /kv-store-entry/body": { "flake": true }, "GET /kv-store-entry/bodyUsed": { "flake": true }, + "GET /kv-store/list/large-response": { "flake": true }, "GET /transform-stream/identity": { "downstream_response": { "body": "hello" } }, From 32bbd27868565d90b2a33c363d807526f617c2c9 Mon Sep 17 00:00:00 2001 From: Sy Brand Date: Wed, 29 Apr 2026 15:08:13 +0100 Subject: [PATCH 2/3] Host API changes --- runtime/fastly/host-api/host_api.cpp | 30 ++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/runtime/fastly/host-api/host_api.cpp b/runtime/fastly/host-api/host_api.cpp index e80aed88bc..023e065576 100644 --- a/runtime/fastly/host-api/host_api.cpp +++ b/runtime/fastly/host-api/host_api.cpp @@ -998,13 +998,27 @@ Result HttpBody::read_into(uint8_t *ptr, size_t chunk_size) const { } constexpr size_t HANDLE_READ_CHUNK_SIZE = 8192; -constexpr size_t HANDLE_READ_BUFFER_SIZE = 500000; Result HttpBody::read_all() const { Result res; + size_t buf_cap = HANDLE_READ_CHUNK_SIZE; size_t buf_len = 0; - uint8_t *buf = static_cast(malloc(HANDLE_READ_BUFFER_SIZE)); + uint8_t *buf = static_cast(malloc(buf_cap)); + if (!buf) { + res.emplace_err(FASTLY_HOST_ERROR_GENERIC_ERROR); + return res; + } do { + if (buf_len + HANDLE_READ_CHUNK_SIZE > buf_cap) { + buf_cap *= 2; + uint8_t *new_buf = static_cast(realloc(buf, buf_cap)); + if (!new_buf) { + free(buf); + res.emplace_err(FASTLY_HOST_ERROR_GENERIC_ERROR); + return res; + } + buf = new_buf; + } host_api::Result chunk = this->read_into((buf + buf_len), HANDLE_READ_CHUNK_SIZE); if (auto *err = chunk.to_err()) { free(buf); @@ -1013,15 +1027,15 @@ Result HttpBody::read_all() const { } size_t len = chunk.unwrap(); if (len == 0) { - buf = static_cast(realloc(buf, buf_len)); + if (buf_len == 0) { + free(buf); + buf = nullptr; + } else { + buf = static_cast(realloc(buf, buf_len)); + } break; } buf_len += len; - if (buf_len > HANDLE_READ_BUFFER_SIZE) { - free(buf); - res.emplace_err(FASTLY_HOST_ERROR_BUFFER_LEN); - return res; - } } while (true); res.emplace(make_host_bytes(buf, buf_len)); return res; From a1a9c85e4ab733083f66a49fcecca61a85de8b4c Mon Sep 17 00:00:00 2001 From: Sy Brand Date: Wed, 29 Apr 2026 15:37:22 +0100 Subject: [PATCH 3/3] Fix key length --- .../js-compute/fixtures/module-mode/src/kv-store.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js b/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js index 42702f2208..7f84296c26 100644 --- a/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js +++ b/integration-tests/js-compute/fixtures/module-mode/src/kv-store.js @@ -1992,7 +1992,7 @@ async function kvStoreInterfaceTests() { routes.set('/kv-store/list/large-response', async () => { const store = new KVStore(KV_STORE_NAME); // 490 keys × ~1024-char names ≈ 507 KB of JSON - const prefix = 'large-list-' + 'a'.repeat(1013); + const prefix = 'large-list-' + 'a'.repeat(1009); await Promise.all( Array.from({ length: 490 }, (_, i) => store.put(prefix + String(i).padStart(4, '0'), 'x'),