Enhance dependency management by adding download, verification, and extraction functions in build process

Author: anchapin

WORKFLOW_TEST.md

@@ -1,6 +1,6 @@
 'use strict';
 
-const { build, clean, copyManifest, installDeps } = require('./tasks/build');
+const { build, clean, copyManifest, installDeps, downloadDeps, verifyDeps, extractDeps, cleanDeps } = require('./tasks/build');
 const { release } = require('./tasks/release');
 const { tmpTestFiles } = require('./tasks/tmpTestFiles');
 const { watch } = require('./tasks/watch');
@@ -9,6 +9,10 @@ exports.build = build;
 exports.clean = clean;
 exports.copyManifest = copyManifest;
 exports.installDeps = installDeps;
+exports.downloadDeps = downloadDeps;
+exports.verifyDeps = verifyDeps;
+exports.extractDeps = extractDeps;
+exports.cleanDeps = cleanDeps;
 exports.release = release;
 exports.tmpTestFiles = tmpTestFiles;
 exports.watch = watch;

gulpfile.js

@@ -197,14 +197,27 @@ function downloadDeps() {
       var destName = fileName.replace(/^.*[\\/]/, '');
     } else {
       // Need to concat endpoint (AWS) with the fileName
-      var uri = manifest.endpoint + fileName;
+      // Properly encode the filename to handle special characters like + 
+      var uri = manifest.endpoint + encodeURIComponent(fileName);
       var destName = fileName;
     }
 
-    return progress(request({uri: uri, timeout: 5000}))
+    console.log(`Downloading ${depend} from ${uri}...`);
+
+    return progress(request({
+      uri: uri, 
+      timeout: 30000, // Increased timeout for ARM64 dependencies which may be larger
+      followRedirect: true,
+      maxRedirects: 5
+    }))
       .on('progress', state => {
         console.log(`Downloading ${depend}, ${(state.percent * 100).toFixed(0)}%`);
       })
+      .on('error', (err) => {
+        console.error(`Download error for ${depend}: ${err.message}`);
+        console.error(`URI: ${uri}`);
+        throw new Error(`Failed to download ${depend}: ${err.message}`);
+      })
       .pipe(source(destName))
       .pipe(gulp.dest(destination));
   });
@@ -233,8 +246,31 @@ function extractDeps() {
     // directory level
     const properDestinationDir = path.join(destination, properName);
     jetpack.remove(properDestinationDir);
-    return jetpack.createReadStream(path.join(destination, destName))
+    
+    const filePath = path.join(destination, destName);
+    console.log(`Extracting ${depend} from ${filePath}...`);
+    
+    // Verify file exists before attempting extraction
+    if (!jetpack.exists(filePath)) {
+      throw new Error(`Dependency file not found: ${filePath}`);
+    }
+    
+    // Check file size to detect incomplete downloads
+    const fileStats = jetpack.inspect(filePath);
+    if (!fileStats || fileStats.size === 0) {
+      throw new Error(`Dependency file is empty or corrupted: ${filePath} (size: ${fileStats ? fileStats.size : 'unknown'})`);
+    }
+    
+    console.log(`File ${destName} size: ${fileStats.size} bytes`);
+    
+    return jetpack.createReadStream(filePath)
       .pipe(zlib.createGunzip())
+      .on('error', (err) => {
+        console.error(`Error extracting ${depend}: ${err.message}`);
+        console.error(`File path: ${filePath}`);
+        console.error(`File size: ${fileStats.size} bytes`);
+        throw new Error(`Extraction failed for ${depend}: ${err.message}. File may be corrupted.`);
+      })
       .pipe(tar.extract(properDestinationDir, {
         strip: 1,
         // There is a bug in tar-fs where, because stripped files & directories
@@ -246,13 +282,84 @@ function extractDeps() {
         ignore: (__, header) => {
           return header.name.length === 0;
         }
-      }));
+      }))
+      .on('error', (err) => {
+        console.error(`Error during tar extraction for ${depend}: ${err.message}`);
+        throw err;
+      });
   });
 
   const tasksAsPromises = tasks.map(task => new Promise((resolve, reject) => task.on('finish', resolve).on('error', reject)));
   return Promise.all(tasksAsPromises);
 }
 
+function verifyDeps() {
+  console.log('Verifying downloaded dependencies...');
+  var tasks = dependencies.map(depend => {
+    const fileInfo = getActualFileInfo(manifest, depend, platform, arch);
+    const fileName = fileInfo.name;
+
+    if( fileName.includes("http") ) {
+      var destName = fileName.replace(/^.*[\\/]/, '');
+    } else {
+      var destName = fileName;
+    }
+
+    const filePath = path.join(destination, destName);
+    
+    return new Promise((resolve, reject) => {
+      if (!jetpack.exists(filePath)) {
+        reject(new Error(`Dependency file not found: ${filePath}`));
+        return;
+      }
+      
+      const fileStats = jetpack.inspect(filePath);
+      if (!fileStats || fileStats.size === 0) {
+        reject(new Error(`Dependency file is empty: ${filePath}`));
+        return;
+      }
+      
+      console.log(`✓ ${depend}: ${destName} (${fileStats.size} bytes)`);
+      
+      // Test gzip header for compressed files
+      if (destName.endsWith('.tar.gz') || destName.endsWith('.tgz')) {
+        const stream = jetpack.createReadStream(filePath);
+        const chunks = [];
+        
+        stream.on('data', chunk => {
+          chunks.push(chunk);
+          if (chunks.length === 1) {
+            // Check gzip magic number (1f 8b)
+            const buffer = chunk;
+            if (buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b) {
+              console.log(`✓ ${depend}: Valid gzip header detected`);
+              stream.destroy();
+              resolve();
+            } else {
+              stream.destroy();
+              reject(new Error(`Invalid gzip header in ${destName}. Expected magic bytes 1f 8b, got ${buffer[0].toString(16)} ${buffer[1].toString(16)}`));
+            }
+          }
+        });
+        
+        stream.on('error', (err) => {
+          reject(new Error(`Error reading ${destName}: ${err.message}`));
+        });
+        
+        stream.on('end', () => {
+          if (chunks.length === 0) {
+            reject(new Error(`No data read from ${destName}`));
+          }
+        });
+      } else {
+        resolve();
+      }
+    });
+  });
+
+  return Promise.all(tasks);
+}
+
 function cleanDeps() {
   var tasks = dependencies.map(depend => {
     const fileInfo = getActualFileInfo(manifest, depend, platform, arch);
@@ -274,4 +381,8 @@ function cleanDeps() {
 exports.build = gulp.series(gulp.parallel(html, fonts, nodeModules, other, environment), finalizeBuild);
 exports.clean = clean;
 exports.copyManifest = copyManifest;
-exports.installDeps = gulp.series(downloadDeps, extractDeps, cleanDeps);
+exports.installDeps = gulp.series(downloadDeps, verifyDeps, extractDeps, cleanDeps);
+exports.downloadDeps = downloadDeps;
+exports.verifyDeps = verifyDeps;
+exports.extractDeps = extractDeps;
+exports.cleanDeps = cleanDeps;