apploader - install-app-from-files - fix android upload; handle data/edge cases

Author: kidneyhex

install_from_files.js

@@ -1,9 +1,16 @@
 /**
  * Apploader - Install App from selected files
+ * 
+ * This function allows users to install BangleJS apps by selecting files from their local filesystem.
+ * It reads metadata.json and uploads all referenced files to the watch.
  */
 function installFromFiles() {
   return new Promise(resolve => {
-    // Ask user to select all files at once (multi-select)
+    var MAX_WAIT_MS = 5000; // maximum time to wait for metadata.json
+    var RESCHEDULE_MS = 400; // retry interval while waiting
+    
+    // SOURCE: core/lib/espruinotools.js fileOpenDialog
+    // Request multi-file selection from user
     Espruino.Core.Utils.fileOpenDialog({
         id:"installappfiles",
         type:"arraybuffer",
@@ -14,7 +21,8 @@ function installFromFiles() {
       if (!installFromFiles.fileCollection) {
         installFromFiles.fileCollection = {
           files: [],
-          count: 0
+          count: 0,
+          firstTs: Date.now() // Track when first file arrived for timeout
         };
       }
 
@@ -28,20 +36,41 @@ function installFromFiles() {
 
       // Use setTimeout to batch-process after all callbacks complete
       clearTimeout(installFromFiles.processTimeout);
-      installFromFiles.processTimeout = setTimeout(function() {
-        var files = installFromFiles.fileCollection.files;
-        installFromFiles.fileCollection = null; // reset for next use
-
-        if (!files || files.length === 0) return resolve();
+      
+      // ANDROID FIX: Debounce and reschedule until metadata.json appears or timeout
+      // Standard desktop browsers deliver all files quickly; Android can have 100-500ms gaps
+      installFromFiles.processTimeout = setTimeout(function processSelection() {
+        var fc = installFromFiles.fileCollection;
+        var files = fc ? fc.files : null;
+        
+        if (!files || files.length === 0) {
+          // nothing yet; keep waiting until max wait then resolve silently
+          if (fc && (Date.now() - fc.firstTs) < MAX_WAIT_MS) {
+            installFromFiles.processTimeout = setTimeout(processSelection, RESCHEDULE_MS);
+            return;
+          }
+          installFromFiles.fileCollection = null; // reset
+          return resolve();
+        }
 
         // Find metadata.json
         var metadataFile = files.find(f => f.name === 'metadata.json' || f.name.endsWith('/metadata.json'));
 
         if (!metadataFile) {
+          if (fc && (Date.now() - fc.firstTs) < MAX_WAIT_MS) {
+            // Keep waiting for the rest of the files
+            installFromFiles.processTimeout = setTimeout(processSelection, RESCHEDULE_MS);
+            return;
+          }
+          // Timed out waiting for metadata.json
+          installFromFiles.fileCollection = null; // reset
           showToast('No metadata.json found in selected files', 'error');
           return resolve();
         }
 
+        // We have metadata.json; stop collecting and proceed
+        installFromFiles.fileCollection = null; // reset for next use
+
         // Parse metadata.json
         var metadata;
         try {
@@ -52,6 +81,7 @@ function installFromFiles() {
           return resolve();
         }
 
+        // Validate required fields per README.md
         if (!metadata.id) {
           showToast('metadata.json missing required "id" field', 'error');
           return resolve();
@@ -62,14 +92,17 @@ function installFromFiles() {
           return resolve();
         }
 
+        // SOURCE: core/js/appinfo.js getFiles() - build file map for lookup
         // Build file map by name (both simple filename and full path)
+        // This handles both "app.js" selections and "folder/app.js" selections
         var fileMap = {};
         files.forEach(f => {
           var simpleName = f.name.split('/').pop();
           fileMap[simpleName] = f;
           fileMap[f.name] = f;
         });
 
+        // SOURCE: core/js/appinfo.js createAppJSON() - build app object from metadata
         // Build app object from metadata
         var app = {
           id: metadata.id,
@@ -79,10 +112,25 @@ function installFromFiles() {
           tags: metadata.tags,
           sortorder: metadata.sortorder,
           storage: metadata.storage,
-          data: metadata.data || []
+          data: metadata.data || [] // NOTE: data[] files are NOT uploaded unless they have url/content
         };
 
+        // SOURCE: core/js/appinfo.js getFiles() - filter by device support
+        // Filter storage files by device compatibility (supports[] field)
+        if (app.storage.some(file => file.supports)) {
+          if (!device || !device.id) {
+            showToast('App requires device-specific files, but no device connected', 'error');
+            return resolve();
+          }
+          // Only keep files that either have no 'supports' field or that support this device
+          app.storage = app.storage.filter(file => {
+            if (!file.supports) return true;
+            return file.supports.includes(device.id);
+          });
+        }
+
         // Determine number of files that will actually be transferred
+        // This counts only files from storage[] that we found in the selected files
         var transferCount = app.storage.filter(storageEntry => {
           var url = storageEntry.url || storageEntry.name;
           return fileMap[url];
@@ -97,11 +145,14 @@ function installFromFiles() {
           var sourceContents = {}; // url -> content
           var missingFiles = [];
 
+          // SOURCE: core/js/appinfo.js parseJS() - detect text files by extension
           function isTextPath(p){
             return /\.(js|json|txt|md|html|css)$/i.test(p);
           }
 
+          // SOURCE: core/js/appinfo.js getFiles() - process all files referenced in storage
           // Process all files referenced in storage
+          // NOTE: We do NOT process data[] files here unless they have url/content specified
           app.storage.forEach(storageEntry => {
             var url = storageEntry.url || storageEntry.name;
             var file = fileMap[url];
@@ -113,13 +164,17 @@ function installFromFiles() {
             }
 
             try {
+              // EVALUATE FILES: If evaluate:true, file contains JS expression to evaluate on device
+              // Common use: app-icon.js with heatshrink-compressed image data
+              // Pattern from core/js/appinfo.js getFiles() and README.md
               var isText = storageEntry.evaluate || isTextPath(url);
 
               if (isText) {
                 // Convert to text
                 sourceContents[url] = new TextDecoder().decode(new Uint8Array(file.data));
               } else {
-                // Convert ArrayBuffer to binary string
+                // SOURCE: core/js/appinfo.js asJSExpr() - convert ArrayBuffer to binary string
+                // Convert ArrayBuffer to binary string (for images, etc.)
                 var a = new Uint8Array(file.data);
                 var s = "";
                 for (var i=0; i<a.length; i++) s += String.fromCharCode(a[i]);
@@ -131,13 +186,50 @@ function installFromFiles() {
             }
           });
 
+          // SOURCE: README.md metadata.json - handle data[] files with url/content
+          // Process data[] files that have url or content specified (initial data files to upload)
+          if (app.data && Array.isArray(app.data)) {
+            app.data.forEach(dataEntry => {
+              // Skip entries that are just tracking patterns (wildcard, or name-only without url/content)
+              if (dataEntry.wildcard) return;
+              if (!dataEntry.url && !dataEntry.content) return;
+              
+              var url = dataEntry.url || dataEntry.name;
+              var file = fileMap[url];
+              
+              if (!file && !dataEntry.content) {
+                console.warn(`Data file not found: ${url}`);
+                // Don't add to missingFiles - data files are optional
+                return;
+              }
+              
+              if (file) {
+                try {
+                  var isText = dataEntry.evaluate || isTextPath(url);
+                  if (isText) {
+                    sourceContents[url] = new TextDecoder().decode(new Uint8Array(file.data));
+                  } else {
+                    var a = new Uint8Array(file.data);
+                    var s = "";
+                    for (var i=0; i<a.length; i++) s += String.fromCharCode(a[i]);
+                    sourceContents[url] = s;
+                  }
+                } catch(err) {
+                  console.error(`Failed to read data file ${url}:`, err);
+                }
+              }
+            });
+          }
+
           if (missingFiles.length > 0) {
             Progress.hide({sticky:true});
             showToast('Missing or unreadable files: ' + missingFiles.join(', '), 'error');
             return resolve();
           }
 
-          // Build app object with inline contents
+          // SOURCE: core/js/appinfo.js createAppJSON() - build app object with inline contents
+          // Build app object with inline contents for upload
+          // This matches the structure expected by Comms.uploadApp
           var appForUpload = {
             id: app.id,
             name: app.name,
@@ -153,32 +245,111 @@ function installFromFiles() {
                 name: storageEntry.name,
                 url: storageEntry.url,
                 content: content,
-                evaluate: !!storageEntry.evaluate,
-                noOverwrite: !!storageEntry.noOverwrite,
-                dataFile: !!storageEntry.dataFile,
-                supports: storageEntry.supports
+                evaluate: !!storageEntry.evaluate,      // JS expression to eval on device
+                noOverwrite: !!storageEntry.noOverwrite, // Don't overwrite if exists (checked below)
+                dataFile: !!storageEntry.dataFile,      // File written by app (not uploaded)
+                supports: storageEntry.supports         // Device compatibility (already filtered above)
               };
             }).filter(Boolean),
-            data: app.data || []
+            data: app.data || [] // Files app writes - tracked for uninstall, optionally uploaded if url/content provided
           };
+          
+          // Add data[] files with content to storage for upload
+          if (app.data && Array.isArray(app.data)) {
+            app.data.forEach(dataEntry => {
+              // Only add if we have content and it's meant to be uploaded initially
+              if (!dataEntry.url && !dataEntry.content) return;
+              if (dataEntry.wildcard) return;
+              
+              var url = dataEntry.url || dataEntry.name;
+              var content = dataEntry.content || sourceContents[url];
+              if (content === undefined) return;
+              
+              appForUpload.storage.push({
+                name: dataEntry.name,
+                url: dataEntry.url,
+                content: content,
+                evaluate: !!dataEntry.evaluate,
+                noOverwrite: true, // Data files should not overwrite by default
+                dataFile: true,
+                storageFile: !!dataEntry.storageFile
+              });
+            });
+          }
 
-          return Promise.resolve(appForUpload)
-          .then(appForUpload => {
-            // Delete existing app if installed using the same pattern as updateApp
+          // SOURCE: core/js/index.js updateApp() lines 963-978
+          // Check for noOverwrite files that exist on device
+          var noOverwriteChecks = Promise.resolve();
+          var filesToCheck = appForUpload.storage.filter(f => f.noOverwrite);
+          
+          if (filesToCheck.length > 0) {
+            Progress.hide({sticky:true});
+            Progress.show({title:`Checking existing files...`});
+            
+            // Build a single command to check all noOverwrite files at once
+            var checkCmd = filesToCheck.map(f => 
+              `require('Storage').read(${JSON.stringify(f.name)})!==undefined`
+            ).join(',');
+            
+            noOverwriteChecks = new Promise((resolveCheck, rejectCheck) => {
+              Comms.eval(`[${checkCmd}]`, (result, err) => {
+                if (err) {
+                  console.warn('Error checking noOverwrite files:', err);
+                  resolveCheck(); // Continue anyway
+                  return;
+                }
+                try {
+                  var existsArray = result;
+                  // Remove files that already exist from the upload list
+                  filesToCheck.forEach((file, idx) => {
+                    if (existsArray[idx]) {
+                      console.log(`Skipping ${file.name} (noOverwrite and already exists)`);
+                      var fileIdx = appForUpload.storage.indexOf(file);
+                      if (fileIdx !== -1) {
+                        appForUpload.storage.splice(fileIdx, 1);
+                      }
+                    }
+                  });
+                  resolveCheck();
+                } catch(e) {
+                  console.warn('Error parsing noOverwrite check results:', e);
+                  resolveCheck(); // Continue anyway
+                }
+              });
+            });
+          }
+          
+          // SOURCE: core/js/index.js updateApp() lines 963-978
+          // Delete existing app if installed using the same pattern as updateApp
+          return noOverwriteChecks.then(appForUpload => {
+            // SOURCE: core/js/index.js updateApp() line 963
+            // Check if app is already installed
             Progress.hide({sticky:true});
             Progress.show({title:`Checking for existing version...`});
             return Comms.getAppInfo(appForUpload)
               .then(remove => {
                 if (!remove) return appForUpload; // not installed
                 Progress.hide({sticky:true});
                 Progress.show({title:`Removing old version...`});
-                // containsFileList:true so we trust the watch's file list
+                // SOURCE: core/js/index.js updateApp() line 978
+                // containsFileList:true tells removeApp to trust the watch's file list
+                // This matches the updateApp pattern exactly
                 return Comms.removeApp(remove, {containsFileList:true}).then(() => appForUpload);
               });
           }).then(appForUpload => {
+            // SOURCE: core/js/index.js uploadApp() line 840 and updateApp() line 983
             // Upload using the standard pipeline
             Progress.hide({sticky:true});
             Progress.show({title:`Installing ${appForUpload.name}...`, sticky:true});
+            // Pass device and language options like uploadApp/updateApp do
+            // NOTE: Comms.uploadApp handles:
+            //   - Creating .info file via AppInfo.createAppJSON
+            //   - Minification/pretokenisation via AppInfo.parseJS if settings.minify=true
+            //   - Module resolution
+            //   - Language translation
+            //   - File upload commands
+            //   - Progress updates
+            //   - Final success message via showUploadFinished()
             return Comms.uploadApp(appForUpload, {device: device, language: LANGUAGE});
           }).then(() => {
             Progress.hide({sticky:true});
@@ -196,17 +367,18 @@ function installFromFiles() {
           console.error(err);
           resolve();
         });
-      }, 100); // Small delay to ensure all file callbacks complete
+      }, 1200); // Debounce to gather all files (Android-friendly)
     });
   });
 }
 
-// Attach UI handler to the button
+// Attach UI handler to the button on window load
 window.addEventListener('load', (event) => {
   var btn = document.getElementById("installappfromfiles");
   if (!btn) return;
   btn.addEventListener("click", event => {
+    // SOURCE: core/js/index.js uploadApp/updateApp pattern
+    // Wrap in startOperation for consistent UI feedback
     startOperation({name:"Install App from Files"}, () => installFromFiles());
   });
-});
-
+});