diff --git a/.github/badges/tests.svg b/.github/badges/tests.svg index c4fb836a1..78f1fc3ea 100644 --- a/.github/badges/tests.svg +++ b/.github/badges/tests.svg @@ -1 +1 @@ -teststests638/643 passed638/643 passed +teststests643 passed643 passed \ No newline at end of file diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFile.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFile.m index bb40730fc..5c6a9b511 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFile.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFile.m @@ -58,7 +58,15 @@ b = false; apiURL = this.downloadURL; % Return the URL as a string - command = sprintf('curl -L -o "%s" "%s"', this.downloadedFile, this.downloadURL); + % Our payloads are already compressed archives (.zip, .nbf.tgz). + % Asking the gateway to apply HTTP compression on top of that + % buys nothing and has produced corrupt archives on both Mac + % and Linux (stream decoders fail on already-compressed bytes). + % Request identity encoding so the raw file is delivered as-is. + % Use -f so HTTP errors surface as non-zero exit codes instead + % of writing a server error body into the destination file. + command = sprintf('curl -fsSL -H "Accept-Encoding: identity" -o "%s" "%s"', ... + this.downloadedFile, this.downloadURL); [status, result] = system(command); @@ -69,4 +77,4 @@ apiResponse = struct('StatusCode', 'N/A (cURL)', 'StatusLine', "Exit Status: " + status); end end -end \ No newline at end of file +end diff --git a/src/ndi/+ndi/+cloud/+download/downloadDatasetFiles.m b/src/ndi/+ndi/+cloud/+download/downloadDatasetFiles.m index 0dfde4fc1..9f6b28ab0 100644 --- a/src/ndi/+ndi/+cloud/+download/downloadDatasetFiles.m +++ b/src/ndi/+ndi/+cloud/+download/downloadDatasetFiles.m @@ -76,9 +76,14 @@ function downloadDatasetFiles(cloudDatasetId, targetFolder, fileUuids, options) end downloadURL = answer.downloadUrl; - % Save the file + % Save the file using curl so gateway-level HTTP compression + % does not corrupt the saved bytes (websave auto-decompresses). try - websave(targetFilepath, downloadURL); + [success_d, answer_d] = ndi.cloud.api.files.getFile(downloadURL, targetFilepath, 'useCurl', true); + if ~success_d + error('NDI:Cloud:FileDownloadFailed', ... + 'curl download failed: %s', char(string(answer_d))); + end catch ME if options.AbortOnError rethrow(ME) diff --git a/src/ndi/+ndi/+cloud/+download/downloadDocumentCollection.m b/src/ndi/+ndi/+cloud/+download/downloadDocumentCollection.m index a5cbf4501..4d6456cbe 100644 --- a/src/ndi/+ndi/+cloud/+download/downloadDocumentCollection.m +++ b/src/ndi/+ndi/+cloud/+download/downloadDocumentCollection.m @@ -24,7 +24,7 @@ % directly to avoid an extra API call to fetch the list again. % % options.Timeout - (1,1) double, optional -% The timeout in seconds for the websave download operation. +% The timeout in seconds for the download operation. % Default is 20. % % options.ChunkSize - (1,1) double, optional @@ -107,12 +107,20 @@ isFinished = false; t1 = tic; + lastErr = ''; % The download URL may not be immediately ready. Retry until timeout. + % Use curl (not websave) so the response body is written as-is; HTTP + % content-encoding applied at the gateway otherwise corrupts the zip. while ~isFinished && toc(t1) < options.Timeout try - websave(tempZipFilepath, downloadUrl); + [success_d, answer_d] = ndi.cloud.api.files.getFile(downloadUrl, tempZipFilepath, 'useCurl', true); + if ~success_d + lastErr = char(string(answer_d)); + error('NDI:Cloud:DocumentDownloadFailed', 'curl download failed: %s', lastErr); + end isFinished = true; catch ME + lastErr = ME.message; pause(1) % Wait a second before retrying end end @@ -120,7 +128,7 @@ if ~isFinished error('NDI:Cloud:DocumentDownloadFailed', ... ['Download failed for chunk %d with message:\n %s\n. If this persists, ', ... - 'consider increasing the Timeout value.'], c, ME.message); + 'consider increasing the Timeout value.'], c, lastErr); end % Unzip and process documents from the current chunk @@ -156,4 +164,4 @@ function deleteIfExists(filePath) if isfile(filePath) delete(filePath) end -end \ No newline at end of file +end diff --git a/src/ndi/+ndi/+cloud/+download/downloadGenericFiles.m b/src/ndi/+ndi/+cloud/+download/downloadGenericFiles.m index 46e2fa374..21f494e5a 100644 --- a/src/ndi/+ndi/+cloud/+download/downloadGenericFiles.m +++ b/src/ndi/+ndi/+cloud/+download/downloadGenericFiles.m @@ -167,8 +167,14 @@ continue; end + % Download using curl so gateway-level HTTP compression does not + % corrupt the saved bytes (websave auto-decompresses responses). try - websave(targetPath, answer.downloadUrl); + [success_d, answer_d] = ndi.cloud.api.files.getFile(answer.downloadUrl, targetPath, 'useCurl', true); + if ~success_d + error('NDI:downloadGenericFiles:DownloadError', ... + 'curl download failed: %s', char(string(answer_d))); + end downloadedFiles(end+1) = filename; %#ok catch ME warning('NDI:downloadGenericFiles:DownloadError', ...