From 6b6909db250332b5941b86f99f7f640621b46783 Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 11:44:06 +0000 Subject: [PATCH 1/9] improve debugging for failed requests --- src/publish/publish_plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index 3ed29d756..1c1655d99 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -72,7 +72,7 @@ export const publishPlugin: InternalPlugin = { new URL(uploadUrl).origin } with status ${uploadResponse.status}` ) - logger.debug(uploadResponse) + logger.debug(await uploadResponse.text()) } resolve() }) From 52375881d5fce43aa99a7ddc8804f454ff12a077 Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 11:44:31 +0000 Subject: [PATCH 2/9] gzip report content when publishing --- src/publish/publish_plugin.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index 1c1655d99..847c9abb1 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -1,9 +1,10 @@ import { Writable } from 'node:stream' import { stripVTControlCharacters } from 'node:util' -import { mkdtemp, stat } from 'node:fs/promises' +import { mkdtemp } from 'node:fs/promises' import path from 'node:path' import { tmpdir } from 'node:os' import { createReadStream, createWriteStream } from 'node:fs' +import { createGzip } from 'node:zlib' import { supportsColor } from 'supports-color' import hasAnsi from 'has-ansi' import { InternalPlugin } from '../plugin' @@ -53,13 +54,13 @@ export const publishPlugin: InternalPlugin = { return () => { return new Promise((resolve) => { tempFileStream.end(async () => { - const stats = await stat(tempFilePath) const uploadResponse = await fetch(uploadUrl, { method: 'PUT', headers: { - 'Content-Length': stats.size.toString(), + 'Content-Type': 'application/jsonl', + 'Content-Encoding': 'gzip', }, - body: createReadStream(tempFilePath, { encoding: 'utf-8' }), + body: createReadStream(tempFilePath).pipe(createGzip()), duplex: 'half', }) if (uploadResponse.ok) { From 24c98497d9a537ea252e7299dd4888bdc7ebcfbe Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 11:49:26 +0000 Subject: [PATCH 3/9] gzip on file system, then stream to service --- src/publish/publish_plugin.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index 847c9abb1..658addef9 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -1,6 +1,7 @@ -import { Writable } from 'node:stream' +import { Writable, pipeline } from 'node:stream' +import { promisify } from 'node:util' import { stripVTControlCharacters } from 'node:util' -import { mkdtemp } from 'node:fs/promises' +import { mkdtemp, stat } from 'node:fs/promises' import path from 'node:path' import { tmpdir } from 'node:os' import { createReadStream, createWriteStream } from 'node:fs' @@ -10,6 +11,8 @@ import hasAnsi from 'has-ansi' import { InternalPlugin } from '../plugin' import { IPublishConfig } from './types' +const pipelineAsync = promisify(pipeline) + const DEFAULT_CUCUMBER_PUBLISH_URL = 'https://messages.cucumber.io/api/reports' export const publishPlugin: InternalPlugin = { @@ -54,13 +57,21 @@ export const publishPlugin: InternalPlugin = { return () => { return new Promise((resolve) => { tempFileStream.end(async () => { + const gzippedFilePath = `${tempFilePath}.gz` + await pipelineAsync( + createReadStream(tempFilePath), + createGzip(), + createWriteStream(gzippedFilePath) + ) + const stats = await stat(gzippedFilePath) const uploadResponse = await fetch(uploadUrl, { method: 'PUT', headers: { 'Content-Type': 'application/jsonl', 'Content-Encoding': 'gzip', + 'Content-Length': stats.size.toString(), }, - body: createReadStream(tempFilePath).pipe(createGzip()), + body: createReadStream(gzippedFilePath), duplex: 'half', }) if (uploadResponse.ok) { From 19b1e82186dd67005e2a5ca50552fa1f61d20917 Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 11:52:54 +0000 Subject: [PATCH 4/9] gzip as we write messages to the file --- src/publish/publish_plugin.ts | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index 658addef9..de2dfcfea 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -1,5 +1,5 @@ -import { Writable, pipeline } from 'node:stream' -import { promisify } from 'node:util' +import { Writable } from 'node:stream' +import { pipeline } from 'node:stream/promises' import { stripVTControlCharacters } from 'node:util' import { mkdtemp, stat } from 'node:fs/promises' import path from 'node:path' @@ -11,8 +11,6 @@ import hasAnsi from 'has-ansi' import { InternalPlugin } from '../plugin' import { IPublishConfig } from './types' -const pipelineAsync = promisify(pipeline) - const DEFAULT_CUCUMBER_PUBLISH_URL = 'https://messages.cucumber.io/api/reports' export const publishPlugin: InternalPlugin = { @@ -48,22 +46,19 @@ export const publishPlugin: InternalPlugin = { const uploadUrl = touchResponse.headers.get('Location') const tempDir = await mkdtemp(path.join(tmpdir(), `cucumber-js-publish-`)) - const tempFilePath = path.join(tempDir, 'envelopes.ndjson') - const tempFileStream = createWriteStream(tempFilePath, { - encoding: 'utf-8', - }) - on('message', (value) => tempFileStream.write(JSON.stringify(value) + '\n')) + const tempFilePath = path.join(tempDir, 'envelopes.jsonl.gz') + const writeStream = createGzip() + const finishedWriting = pipeline( + writeStream, + createWriteStream(tempFilePath) + ) + on('message', (value) => writeStream.write(JSON.stringify(value) + '\n')) return () => { return new Promise((resolve) => { - tempFileStream.end(async () => { - const gzippedFilePath = `${tempFilePath}.gz` - await pipelineAsync( - createReadStream(tempFilePath), - createGzip(), - createWriteStream(gzippedFilePath) - ) - const stats = await stat(gzippedFilePath) + writeStream.end(async () => { + await finishedWriting + const stats = await stat(tempFilePath) const uploadResponse = await fetch(uploadUrl, { method: 'PUT', headers: { @@ -71,7 +66,7 @@ export const publishPlugin: InternalPlugin = { 'Content-Encoding': 'gzip', 'Content-Length': stats.size.toString(), }, - body: createReadStream(gzippedFilePath), + body: createReadStream(tempFilePath), duplex: 'half', }) if (uploadResponse.ok) { From 437ea61cab321980698534c8a560d72bd47bb1cb Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 12:14:59 +0000 Subject: [PATCH 5/9] better debug logging for size --- src/publish/publish_plugin.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index de2dfcfea..adc7056a7 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -59,12 +59,14 @@ export const publishPlugin: InternalPlugin = { writeStream.end(async () => { await finishedWriting const stats = await stat(tempFilePath) + const contentLength = stats.size.toString() + logger.debug('Uploading envelopes to Cucumber Reports with content length:', contentLength) const uploadResponse = await fetch(uploadUrl, { method: 'PUT', headers: { 'Content-Type': 'application/jsonl', 'Content-Encoding': 'gzip', - 'Content-Length': stats.size.toString(), + 'Content-Length': contentLength, }, body: createReadStream(tempFilePath), duplex: 'half', From 8f3e3e9662440629d20cf2ce4280c27bd4ffca33 Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 12:46:19 +0000 Subject: [PATCH 6/9] update step definition to do gunzip --- features/step_definitions/report_server_steps.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/features/step_definitions/report_server_steps.ts b/features/step_definitions/report_server_steps.ts index 78c91b870..588ceabcf 100644 --- a/features/step_definitions/report_server_steps.ts +++ b/features/step_definitions/report_server_steps.ts @@ -1,5 +1,6 @@ import { URL } from 'node:url' import assert from 'node:assert' +import { gunzipSync } from 'node:zlib' import { expect } from 'chai' import { Given, Then, DataTable } from '../..' import { World } from '../support/world' @@ -30,7 +31,7 @@ Then( .map((row) => row[0]) const receivedBodies = await this.reportServer.stop() - const ndjson = receivedBodies.toString('utf-8').trim() + const ndjson = gunzipSync(receivedBodies).toString('utf-8').trim() if (ndjson === '') assert.fail('Server received nothing') const receivedMessageTypes = ndjson From 15f1e278e9882e6fa616341d16e29176d5ba91aa Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 12:52:26 +0000 Subject: [PATCH 7/9] formatting etc --- src/publish/publish_plugin.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index adc7056a7..2b2a3fabb 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -60,7 +60,10 @@ export const publishPlugin: InternalPlugin = { await finishedWriting const stats = await stat(tempFilePath) const contentLength = stats.size.toString() - logger.debug('Uploading envelopes to Cucumber Reports with content length:', contentLength) + logger.debug( + 'Uploading envelopes to Cucumber Reports with content length:', + contentLength + ) const uploadResponse = await fetch(uploadUrl, { method: 'PUT', headers: { From c4fc62dae0eb04c5d0e3395b95ec3b75656cd5cf Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 14:47:39 +0000 Subject: [PATCH 8/9] restore missing encoding declaration --- src/publish/publish_plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index 2b2a3fabb..2bdbc129f 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -71,7 +71,7 @@ export const publishPlugin: InternalPlugin = { 'Content-Encoding': 'gzip', 'Content-Length': contentLength, }, - body: createReadStream(tempFilePath), + body: createReadStream(tempFilePath, { encoding: 'utf-8' }), duplex: 'half', }) if (uploadResponse.ok) { From 66887a3a7ad6c999ab64c6855b1eed1226fdb159 Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 10 Nov 2025 16:32:11 +0000 Subject: [PATCH 9/9] remove errant encoding declaration --- src/publish/publish_plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/publish/publish_plugin.ts b/src/publish/publish_plugin.ts index 2bdbc129f..2b2a3fabb 100644 --- a/src/publish/publish_plugin.ts +++ b/src/publish/publish_plugin.ts @@ -71,7 +71,7 @@ export const publishPlugin: InternalPlugin = { 'Content-Encoding': 'gzip', 'Content-Length': contentLength, }, - body: createReadStream(tempFilePath, { encoding: 'utf-8' }), + body: createReadStream(tempFilePath), duplex: 'half', }) if (uploadResponse.ok) {