From 2a315c0d92488ff7329a352cdd963fac15681caf Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Wed, 21 Jan 2026 12:20:14 +0000 Subject: [PATCH 01/13] Adding progress tracking for upload and zip --- src/utils/index.ts | 2 ++ src/utils/query/useShip.ts | 59 ++++++++++++++++++++----------- src/utils/upload.ts | 72 ++++++++++++++++++++++++++++++++++++++ src/utils/zip.ts | 69 ++++++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 21 deletions(-) create mode 100644 src/utils/upload.ts create mode 100644 src/utils/zip.ts diff --git a/src/utils/index.ts b/src/utils/index.ts index 68c9dbb..4d6269b 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -16,6 +16,8 @@ export * from './godot.js' export * from './help.js' export * from './hooks/index.js' export * from './query/index.js' +export * from './upload.js' +export * from './zip.js' /** * Works the same way that git short commits are generated. diff --git a/src/utils/query/useShip.ts b/src/utils/query/useShip.ts index 2d6db06..808a6e7 100644 --- a/src/utils/query/useShip.ts +++ b/src/utils/query/useShip.ts @@ -2,16 +2,14 @@ import fs from 'node:fs' import {Command} from '@oclif/core' import {useMutation} from '@tanstack/react-query' -import axios from 'axios' import fg from 'fast-glob' import {v4 as uuid} from 'uuid' -import {ZipFile} from 'yazl' import {getNewUploadTicket, getProject, startJobsFromUpload} from '@cli/api/index.js' import {BaseCommand} from '@cli/baseCommands/index.js' import {DEFAULT_IGNORED_FILES_GLOBS, DEFAULT_SHIPPED_FILES_GLOBS, cacheKeys} from '@cli/constants/index.js' import {Job, Platform, ProjectConfig, ShipGameFlags, UploadDetails} from '@cli/types' -import {getCWDGitInfo, getFileHash, queryClient} from '@cli/utils/index.js' +import {createZip, getCWDGitInfo, getFileHash, queryClient, uploadZip} from '@cli/utils/index.js' // Takes the current command so we can get the project config // This could be made more composable @@ -21,6 +19,22 @@ interface ShipOptions { shipFlags?: ShipGameFlags // If provided, will override command flags } +function formatProgressLog( + label: string, + data: {progress: number; elapsedSeconds: number; speedMBps: number; [key: string]: any}, + bytesKey: 'writtenBytes' | 'loadedBytes', + totalKey: 'estimatedTotalBytes' | 'totalBytes', + isEstimated = false, +): string { + const elapsed = data.elapsedSeconds.toFixed(1) + const transferredMB = (data[bytesKey] / 1024 / 1024).toFixed(2) + const totalMB = (data[totalKey] / 1024 / 1024).toFixed(2) + const prgs = Math.round(data.progress * 100) + const speed = data.speedMBps.toFixed(2) + const totalPrefix = isEstimated ? '~' : '' + return `${label}: ${prgs}% (${transferredMB}MB / ${totalPrefix}${totalMB}MB) - ${elapsed}s - ${speed}MB/s` +} + export async function ship({command, log = () => {}, shipFlags}: ShipOptions): Promise { const commandFlags = command.getFlags() as ShipGameFlags const finalFlags = shipFlags || commandFlags @@ -51,37 +65,40 @@ export async function ship({command, log = () => {}, shipFlags}: ShipOptions): P const files = await fg(shippedFilesGlobs, {dot: true, ignore: ignoredFilesGlobs}) verbose && log(`Found ${files.length} files, adding to zip...`) - const zipFile = new ZipFile() - for (const file of files) { - zipFile.addFile(file, file) - } - - const outputZipToFile = (zip: ZipFile, fileName: string) => - new Promise((resolve) => { - const outputStream = fs.createWriteStream(fileName) - zip.outputStream.pipe(outputStream).on('close', () => resolve()) - zip.end() - }) const tmpZipFile = `${process.cwd()}/shipthis-${uuid()}.zip` log(`Creating zip file: ${tmpZipFile}`) - await outputZipToFile(zipFile, tmpZipFile) + await createZip({ + files, + outputPath: tmpZipFile, + onProgress: (data) => { + log(formatProgressLog('Zipping', data, 'writtenBytes', 'estimatedTotalBytes', true)) + }, + }) - verbose && log('Reading zip file buffer...') - const zipBuffer = fs.readFileSync(tmpZipFile) const {size} = fs.statSync(tmpZipFile) verbose && log('Requesting upload ticket...') const uploadTicket = await getNewUploadTicket(projectConfig.project.id) log('Uploading zip file...') - await axios.put(uploadTicket.url, zipBuffer, { - headers: { - 'Content-Type': 'application/zip', - 'Content-length': size, + const zipStream = fs.createReadStream(tmpZipFile) + + const response = await uploadZip({ + url: uploadTicket.url, + zipStream, + zipSize: size, + onProgress: (data) => { + log(formatProgressLog('Uploading', data, 'loadedBytes', 'totalBytes', false)) }, }) + if (!response.ok) { + throw new Error(`Upload failed: ${response.status} ${response.statusText}`) + } + + log(`Upload complete`) + verbose && log('Fetching Git info...') const gitInfo = await getCWDGitInfo() verbose && log('Computing file hash...') diff --git a/src/utils/upload.ts b/src/utils/upload.ts new file mode 100644 index 0000000..8e6b7a0 --- /dev/null +++ b/src/utils/upload.ts @@ -0,0 +1,72 @@ +import {Readable, Transform} from 'stream' + +export const ON_PROGRESS_THROTTLE_MS = 1000 + +export function createProgressStream( + totalSize: number, + onProgress: (sent: number, total: number) => void, + throttleMs?: number +): Transform { + let sent = 0 + let lastCallTime = 0 + + return new Transform({ + transform(chunk, encoding, callback) { + sent += chunk.length + + const now = Date.now() + if (!throttleMs || now - lastCallTime >= throttleMs) { + onProgress(sent, totalSize) + lastCallTime = now + } + + callback(null, chunk) + }, + }) +} + +interface ProgressData { + progress: number + loadedBytes: number + totalBytes: number + speedMBps: number + elapsedSeconds: number +} + +interface UploadProps { + url: string + zipStream: Readable + zipSize: number + onProgress: (data: ProgressData) => void +} + +// Uploads a zip file with progress tracking +export function uploadZip({url, zipStream, zipSize, onProgress}: UploadProps): Promise { + const startTime = Date.now() + + const progressStream = createProgressStream(zipSize, (sent, total) => { + const elapsedSeconds = (Date.now() - startTime) / 1000 + onProgress({ + progress: total ? sent / total : 0, + loadedBytes: sent, + totalBytes: total, + speedMBps: sent / elapsedSeconds / 1024 / 1024, + elapsedSeconds, + }) + }, ON_PROGRESS_THROTTLE_MS) + + const streamWithProgress = zipStream.pipe(progressStream) + const webStream = Readable.toWeb(streamWithProgress) as ReadableStream + + const response = fetch(url, { + method: 'PUT', + headers: { + 'Content-Type': 'application/zip', + 'Content-Length': zipSize.toString(), + }, + body: webStream, + duplex: 'half', + } as RequestInit & {duplex: 'half'}) + + return response +} diff --git a/src/utils/zip.ts b/src/utils/zip.ts new file mode 100644 index 0000000..407f097 --- /dev/null +++ b/src/utils/zip.ts @@ -0,0 +1,69 @@ +import fs from 'node:fs' +import {ZipFile} from 'yazl' + +import {createProgressStream, ON_PROGRESS_THROTTLE_MS} from './upload.js' + +// Used to estimate the final zip size +const COMPRESSION_RATIO = 0.65 + +export interface ZipProgressData { + progress: number + writtenBytes: number + estimatedTotalBytes: number + sourceTotalBytes: number + elapsedSeconds: number + speedMBps: number +} + +interface CreateZipProps { + files: string[] + outputPath: string + onProgress: (data: ZipProgressData) => void +} + +// Creates a zip file with progress tracking +export async function createZip({files, outputPath, onProgress}: CreateZipProps): Promise { + const startTime = Date.now() + + let totalSourceSize = 0 + for (const file of files) { + try { + const stats = fs.statSync(file) + totalSourceSize += stats.size + } catch { + // Skip inaccessible files + } + } + + const estimatedZipSize = Math.max(Math.round(totalSourceSize * COMPRESSION_RATIO), 1) + + const zipFile = new ZipFile() + for (const file of files) { + zipFile.addFile(file, file) + } + + return new Promise((resolve, reject) => { + const outputStream = fs.createWriteStream(outputPath) + + const progressStream = createProgressStream(estimatedZipSize, (written, total) => { + const elapsedSeconds = (Date.now() - startTime) / 1000 + onProgress({ + progress: total ? Math.min(1, written / total) : 0, + writtenBytes: written, + estimatedTotalBytes: total, + sourceTotalBytes: totalSourceSize, + elapsedSeconds, + speedMBps: written / elapsedSeconds / 1024 / 1024, + }) + }, ON_PROGRESS_THROTTLE_MS) + + zipFile.outputStream + .pipe(progressStream) + .pipe(outputStream) + .on('close', () => resolve()) + .on('error', reject) + + zipFile.end() + }) +} + From ed1a4bfea93c94b8315fac8fa8f2071bbf161b3e Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Thu, 22 Jan 2026 09:58:32 +0000 Subject: [PATCH 02/13] Less frequent updates --- src/utils/upload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/upload.ts b/src/utils/upload.ts index 8e6b7a0..9924e9d 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -1,6 +1,6 @@ import {Readable, Transform} from 'stream' -export const ON_PROGRESS_THROTTLE_MS = 1000 +export const ON_PROGRESS_THROTTLE_MS = 2000 export function createProgressStream( totalSize: number, From eca52f9b644714569e15f6ae8f4f27bcfcd6659d Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Thu, 22 Jan 2026 10:59:24 +0000 Subject: [PATCH 03/13] Simpler output --- src/utils/query/useShip.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/query/useShip.ts b/src/utils/query/useShip.ts index 808a6e7..b897d7a 100644 --- a/src/utils/query/useShip.ts +++ b/src/utils/query/useShip.ts @@ -66,8 +66,9 @@ export async function ship({command, log = () => {}, shipFlags}: ShipOptions): P verbose && log(`Found ${files.length} files, adding to zip...`) - const tmpZipFile = `${process.cwd()}/shipthis-${uuid()}.zip` - log(`Creating zip file: ${tmpZipFile}`) + const tmpZipFileName = `shipthis-${uuid()}.zip` + const tmpZipFile = `${process.cwd()}/${tmpZipFileName}` + log(`Creating zip file: ${tmpZipFileName}`) await createZip({ files, outputPath: tmpZipFile, From 180bc4df5f26e79bcb003792acf94ceb198febb7 Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Thu, 22 Jan 2026 11:48:57 +0000 Subject: [PATCH 04/13] Calc zip hash and cleanup earlier --- src/utils/query/useShip.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/utils/query/useShip.ts b/src/utils/query/useShip.ts index b897d7a..6ee8d6a 100644 --- a/src/utils/query/useShip.ts +++ b/src/utils/query/useShip.ts @@ -94,6 +94,12 @@ export async function ship({command, log = () => {}, shipFlags}: ShipOptions): P }, }) + verbose && log('Computing zip file hash...') + const zipFileMd5 = await getFileHash(tmpZipFile) + + verbose && log('Cleaning up temporary zip file...') + fs.unlinkSync(tmpZipFile) + if (!response.ok) { throw new Error(`Upload failed: ${response.status} ${response.statusText}`) } @@ -102,8 +108,6 @@ export async function ship({command, log = () => {}, shipFlags}: ShipOptions): P verbose && log('Fetching Git info...') const gitInfo = await getCWDGitInfo() - verbose && log('Computing file hash...') - const zipFileMd5 = await getFileHash(tmpZipFile) const uploadDetails: UploadDetails = { ...gitInfo, zipFileMd5, @@ -122,9 +126,6 @@ export async function ship({command, log = () => {}, shipFlags}: ShipOptions): P const jobs = await startJobsFromUpload(uploadTicket.id, startJobsOptions) - verbose && log('Cleaning up temporary zip file...') - fs.unlinkSync(tmpZipFile) - verbose && log('Job submission complete.') if (jobs.length === 0) { From 7f60e3d6284dfc4e3b9bd0c6d7bc50974a1ce2b4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Jan 2026 11:51:56 +0000 Subject: [PATCH 05/13] Initial plan From a8c71208da4261969f80becf8c3ac8c52a2f42d9 Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Thu, 22 Jan 2026 11:52:37 +0000 Subject: [PATCH 06/13] Update variable name Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/utils/query/useShip.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/query/useShip.ts b/src/utils/query/useShip.ts index 6ee8d6a..cfd93cd 100644 --- a/src/utils/query/useShip.ts +++ b/src/utils/query/useShip.ts @@ -29,10 +29,10 @@ function formatProgressLog( const elapsed = data.elapsedSeconds.toFixed(1) const transferredMB = (data[bytesKey] / 1024 / 1024).toFixed(2) const totalMB = (data[totalKey] / 1024 / 1024).toFixed(2) - const prgs = Math.round(data.progress * 100) + const progressPercent = Math.round(data.progress * 100) const speed = data.speedMBps.toFixed(2) const totalPrefix = isEstimated ? '~' : '' - return `${label}: ${prgs}% (${transferredMB}MB / ${totalPrefix}${totalMB}MB) - ${elapsed}s - ${speed}MB/s` + return `${label}: ${progressPercent}% (${transferredMB}MB / ${totalPrefix}${totalMB}MB) - ${elapsed}s - ${speed}MB/s` } export async function ship({command, log = () => {}, shipFlags}: ShipOptions): Promise { From f8cae3f84032c5176ff2de24a4696514f6eab470 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Jan 2026 11:54:41 +0000 Subject: [PATCH 07/13] Fix division by zero in speed calculation for zip and upload progress Co-authored-by: madebydavid <5401249+madebydavid@users.noreply.github.com> --- src/utils/upload.ts | 3 ++- src/utils/zip.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils/upload.ts b/src/utils/upload.ts index 9924e9d..9540463 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -46,11 +46,12 @@ export function uploadZip({url, zipStream, zipSize, onProgress}: UploadProps): P const progressStream = createProgressStream(zipSize, (sent, total) => { const elapsedSeconds = (Date.now() - startTime) / 1000 + const speedMBps = elapsedSeconds < 0.001 ? 0 : sent / elapsedSeconds / 1024 / 1024 onProgress({ progress: total ? sent / total : 0, loadedBytes: sent, totalBytes: total, - speedMBps: sent / elapsedSeconds / 1024 / 1024, + speedMBps, elapsedSeconds, }) }, ON_PROGRESS_THROTTLE_MS) diff --git a/src/utils/zip.ts b/src/utils/zip.ts index 407f097..572e400 100644 --- a/src/utils/zip.ts +++ b/src/utils/zip.ts @@ -47,13 +47,14 @@ export async function createZip({files, outputPath, onProgress}: CreateZipProps) const progressStream = createProgressStream(estimatedZipSize, (written, total) => { const elapsedSeconds = (Date.now() - startTime) / 1000 + const speedMBps = elapsedSeconds < 0.001 ? 0 : written / elapsedSeconds / 1024 / 1024 onProgress({ progress: total ? Math.min(1, written / total) : 0, writtenBytes: written, estimatedTotalBytes: total, sourceTotalBytes: totalSourceSize, elapsedSeconds, - speedMBps: written / elapsedSeconds / 1024 / 1024, + speedMBps, }) }, ON_PROGRESS_THROTTLE_MS) From 611d5ecbb27dda6087152ec026161234ccc057ee Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Thu, 22 Jan 2026 11:56:29 +0000 Subject: [PATCH 08/13] async size calculation Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/utils/zip.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/utils/zip.ts b/src/utils/zip.ts index 407f097..131152e 100644 --- a/src/utils/zip.ts +++ b/src/utils/zip.ts @@ -25,13 +25,21 @@ interface CreateZipProps { export async function createZip({files, outputPath, onProgress}: CreateZipProps): Promise { const startTime = Date.now() - let totalSourceSize = 0 - for (const file of files) { + const statPromises = files.map(async (file) => { try { - const stats = fs.statSync(file) - totalSourceSize += stats.size + return await fs.promises.stat(file) } catch { // Skip inaccessible files + return null + } + }) + + const statsResults = await Promise.all(statPromises) + + let totalSourceSize = 0 + for (const stats of statsResults) { + if (stats) { + totalSourceSize += stats.size } } From ac65e6ce39b7af31aec8fb6da440e6347e6d9d32 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:02:44 +0000 Subject: [PATCH 09/13] Initial plan From 4f06929e5e38a56f51bed9ec194e1abf7ab836c3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:05:08 +0000 Subject: [PATCH 10/13] Initial plan From 041bf5774a0ebbaf3b6226078c776973e48a3ec0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:05:22 +0000 Subject: [PATCH 11/13] Add explanatory comment for duplex property type assertion Co-authored-by: madebydavid <5401249+madebydavid@users.noreply.github.com> --- src/utils/upload.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/utils/upload.ts b/src/utils/upload.ts index 9540463..ce7fcec 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -59,6 +59,12 @@ export function uploadZip({url, zipStream, zipSize, onProgress}: UploadProps): P const streamWithProgress = zipStream.pipe(progressStream) const webStream = Readable.toWeb(streamWithProgress) as ReadableStream + // The 'duplex' property is required when using a ReadableStream as the request body. + // 'duplex: half' indicates half-duplex communication (one direction at a time), + // which is the mode needed for streaming request bodies with fetch(). + // Type assertion is necessary because 'duplex' is not yet part of the standard + // TypeScript RequestInit type definition, though it's required by the fetch spec + // for streaming uploads. const response = fetch(url, { method: 'PUT', headers: { From dfc642479ed08ec0eb81fd12c573cd0ce263e4ff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:07:57 +0000 Subject: [PATCH 12/13] Improve error handling in createZip function Co-authored-by: madebydavid <5401249+madebydavid@users.noreply.github.com> --- src/utils/zip.ts | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/utils/zip.ts b/src/utils/zip.ts index f868e80..88b582f 100644 --- a/src/utils/zip.ts +++ b/src/utils/zip.ts @@ -51,6 +51,25 @@ export async function createZip({files, outputPath, onProgress}: CreateZipProps) } return new Promise((resolve, reject) => { + let settled = false + + const handleError = (error: Error) => { + if (settled) return + settled = true + + // Clean up streams + outputStream.destroy() + progressStream.destroy() + + reject(error) + } + + const handleSuccess = () => { + if (settled) return + settled = true + resolve() + } + const outputStream = fs.createWriteStream(outputPath) const progressStream = createProgressStream(estimatedZipSize, (written, total) => { @@ -66,11 +85,15 @@ export async function createZip({files, outputPath, onProgress}: CreateZipProps) }) }, ON_PROGRESS_THROTTLE_MS) + // Add error handlers to all streams in the pipe chain + zipFile.outputStream.on('error', handleError) + progressStream.on('error', handleError) + outputStream.on('error', handleError) + outputStream.on('close', handleSuccess) + zipFile.outputStream .pipe(progressStream) .pipe(outputStream) - .on('close', () => resolve()) - .on('error', reject) zipFile.end() }) From 76277cb0c122872533e05d5387291eb0552b8fb8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:09:54 +0000 Subject: [PATCH 13/13] Add zipFile.outputStream cleanup to prevent resource leaks Co-authored-by: madebydavid <5401249+madebydavid@users.noreply.github.com> --- src/utils/zip.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/zip.ts b/src/utils/zip.ts index 88b582f..68a7e59 100644 --- a/src/utils/zip.ts +++ b/src/utils/zip.ts @@ -58,8 +58,9 @@ export async function createZip({files, outputPath, onProgress}: CreateZipProps) settled = true // Clean up streams - outputStream.destroy() + zipFile.outputStream.destroy() progressStream.destroy() + outputStream.destroy() reject(error) }