Fix apk install

docker-compose-macos.yaml

@@ -134,7 +134,7 @@ services:
         container_name: devicehub-storage-plugin-apk
         env_file:
             - scripts/variables.env
-        command: stf storage-plugin-apk --port 3000 --storage-url http://devicehub-storage-temp/ --secret=${STF_SECRET}
+        command: stf storage-plugin-apk --port 3000 --storage-url http://devicehub-storage-temp:3000/ --secret=${STF_SECRET}
         depends_on:
             devicehub-migrate:
                 condition: service_completed_successfully
@@ -147,7 +147,7 @@ services:
         container_name: devicehub-storage-plugin-image
         env_file:
             - scripts/variables.env
-        command: stf storage-plugin-image --port 3000 --storage-url https://${STF_DOMAIN}:${STF_PORT}/ --secret=${STF_SECRET}
+        command: stf storage-plugin-image --port 3000 --storage-url http://devicehub-storage-temp:3000/ --secret=${STF_SECRET}
         depends_on:
             devicehub-migrate:
                 condition: service_completed_successfully
@@ -207,7 +207,7 @@ services:
         container_name: devicehub-websocket
         env_file:
             - scripts/variables.env
-        command: stf websocket --port 3000 --storage-url https://${STF_DOMAIN}:${STF_PORT}/ --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270 --secret=${STF_SECRET}
+        command: stf websocket --port 3000 --storage-url http://devicehub-storage-temp:3000/ --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270 --secret=${STF_SECRET}
         depends_on:
             devicehub-migrate:
                 condition: service_completed_successfully
@@ -252,7 +252,7 @@ services:
             devicehub-migrate:
                 condition: service_completed_successfully
         restart: unless-stopped
-        command: stf provider --name devicehub-provider --adb-host host.docker.internal --no-cleanup --connect-sub tcp://devicehub-triproxy-dev:7250 --connect-push tcp://devicehub-triproxy-dev:7270 --storage-url https://${STF_DOMAIN}:${STF_PORT}/ --public-ip ${STF_DOMAIN} --min-port=12010 --max-port=12100 --heartbeat-interval 10000 --screen-ws-url-pattern "wss://${STF_DOMAIN}:${STF_PORT}/d/devicehub-provider/<%= publicPort %>/" --secret=${STF_SECRET}
+        command: stf provider --name devicehub-provider --adb-host host.docker.internal --no-cleanup --connect-sub tcp://devicehub-triproxy-dev:7250 --connect-push tcp://devicehub-triproxy-dev:7270 --storage-url http://devicehub-storage-plugin-apk:3000/ --public-ip ${STF_DOMAIN} --min-port=12010 --max-port=12100 --heartbeat-interval 10000 --screen-ws-url-pattern "wss://${STF_DOMAIN}:${STF_PORT}/d/devicehub-provider/<%= publicPort %>/" --secret=${STF_SECRET}
         networks:
             devicehub:
     devicehub-ssl:

docker-compose-prod.yaml

@@ -150,7 +150,7 @@ services:
         container_name: devicehub-storage-plugin-apk
         env_file:
             - scripts/variables.env
-        command: stf storage-plugin-apk --port 3000 --storage-url http://devicehub-storage-temp/ --secret=${STF_SECRET}
+        command: stf storage-plugin-apk --port 3000 --storage-url http://devicehub-storage-temp:3000/ --secret=${STF_SECRET}
         depends_on:
             devicehub-migrate:
                 condition: service_completed_successfully
@@ -163,7 +163,7 @@ services:
         container_name: devicehub-storage-plugin-image
         env_file:
             - scripts/variables.env
-        command: stf storage-plugin-image --port 3000 --storage-url https://${STF_DOMAIN}:${STF_PORT}/ --secret=${STF_SECRET}
+        command: stf storage-plugin-image --port 3000 --storage-url http://devicehub-storage-temp:3000/ --secret=${STF_SECRET}
         depends_on:
             devicehub-migrate:
                 condition: service_completed_successfully
@@ -223,7 +223,7 @@ services:
         container_name: devicehub-websocket
         env_file:
             - scripts/variables.env
-        command: stf websocket --port 3000 --storage-url https://${STF_DOMAIN}:${STF_PORT}/ --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270 --secret=${STF_SECRET}
+        command: stf websocket --port 3000 --storage-url http://devicehub-storage-temp:3000/ --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270 --secret=${STF_SECRET}
         depends_on:
             devicehub-migrate:
                 condition: service_completed_successfully
@@ -270,7 +270,7 @@ services:
             adbd:
                 condition: service_healthy
         restart: unless-stopped
-        command: stf provider --name devicehub-provider --adb-host adbd --no-cleanup --connect-sub tcp://devicehub-triproxy-dev:7250 --connect-push tcp://devicehub-triproxy-dev:7270 --storage-url https://${STF_DOMAIN}:${STF_PORT}/ --public-ip ${STF_DOMAIN} --min-port=12010 --max-port=12100 --heartbeat-interval 10000 --screen-ws-url-pattern "wss://${STF_DOMAIN}:${STF_PORT}/d/devicehub-provider/<%= publicPort %>/" --secret=${STF_SECRET}
+        command: stf provider --name devicehub-provider --adb-host adbd --no-cleanup --connect-sub tcp://devicehub-triproxy-dev:7250 --connect-push tcp://devicehub-triproxy-dev:7270 --storage-url http://devicehub-storage-plugin-apk:3000/ --public-ip ${STF_DOMAIN} --min-port=12010 --max-port=12100 --heartbeat-interval 10000 --screen-ws-url-pattern "wss://${STF_DOMAIN}:${STF_PORT}/d/devicehub-provider/<%= publicPort %>/" --secret=${STF_SECRET}
         networks:
             devicehub:
     devicehub-ssl:

lib/cli/local/index.js

@@ -444,14 +444,14 @@ export const handler = function(argv) {
                 'storage-plugin-image',
                 '--port', argv.storagePluginImagePort,
                 '--storage-url',
-                util.format('http://localhost:%d/', argv.port),
+                util.format('http://localhost:%d/', argv.storagePort),
                 '--secret', argv.authSecret
             ],
             [ // apk processor
                 'storage-plugin-apk',
                 '--port', argv.storagePluginApkPort,
                 '--storage-url',
-                util.format('http://localhost:%d/', argv.port),
+                util.format('http://localhost:%d/', argv.storagePort),
                 '--secret', argv.authSecret
             ]
         ],

lib/units/base-device/support/storage.ts

@@ -0,0 +1,174 @@
+import util from 'util'
+import url from 'url'
+import syrup from '@devicefarmer/stf-syrup'
+import logger from '../../../util/logger.js'
+import { createReadStream, createWriteStream } from 'fs'
+import { unlink } from 'fs/promises'
+import temp from 'tmp-promise'
+import { finished } from 'stream/promises'
+import https from "https"
+import http from "http"
+import FormData from 'form-data'
+import { Readable } from 'stream'
+
+interface StorageOptions {
+    storageUrl: string
+    serial: string
+}
+
+interface FileMetadata {
+    jwt: string
+    filename?: string
+    contentType?: string
+    [key: string]: any
+}
+
+interface UploadedFile {
+    href: string
+    [key: string]: any
+}
+
+interface DownloadedFile {
+    path: string
+    cleanup: () => Promise<void>
+}
+
+interface UploadOptions {
+    url: string
+    headers?: Record<string, string>
+}
+
+function uploadFile(
+    options: UploadOptions,
+    stream: Readable,
+    meta: FileMetadata
+): Promise<{ statusCode: number; body: string }> {
+    return new Promise((resolve, reject) => {
+        const parsedUrl = new URL(options.url)
+        const protocol = parsedUrl.protocol === 'https:' ? https : http
+
+        const form = new FormData()
+        form.append('file', stream, meta)
+
+        const requestOptions: http.RequestOptions = {
+            hostname: parsedUrl.hostname,
+            port: parsedUrl.port,
+            path: parsedUrl.pathname + parsedUrl.search,
+            method: 'POST',
+            headers: {
+                ...options.headers,
+                ...form.getHeaders()
+            }
+        }
+
+        const req = protocol.request(requestOptions, (res) => {
+            const chunks: Buffer[] = []
+
+            res.on('data', (chunk) => {
+                chunks.push(chunk)
+            })
+
+            res.on('end', () => {
+                const body = Buffer.concat(chunks).toString()
+                resolve({
+                    statusCode: res.statusCode || 500,
+                    body
+                })
+            })
+        })
+
+        req.on('error', (err) => {
+            reject(err)
+        })
+
+        form.pipe(req)
+    })
+}
+
+export default syrup.serial()
+    .define((options: StorageOptions) => {
+        const log = logger.createLogger('base-device:support:storage')
+
+        const plugin = {
+            store: async (type: string, stream: Readable, meta: FileMetadata): Promise<UploadedFile> => {
+                const uploadUrl = url.resolve(options.storageUrl, util.format('s/upload/%s', type))
+                const headers = {
+                    internal: 'Internal ' + meta.jwt
+                }
+
+                try {
+                    const { statusCode, body } = await uploadFile(
+                        { url: uploadUrl, headers },
+                        stream,
+                        meta
+                    )
+
+                    if (statusCode !== 201) {
+                        log.error('Upload to %s failed: HTTP %s', uploadUrl, statusCode)
+                        log.debug(body)
+                        throw new Error(util.format('Upload to %s failed: HTTP %s', uploadUrl, statusCode))
+                    }
+
+                    const result = JSON.parse(body)
+                    log.info('Uploaded to %s', result.resources.file.href)
+                    return result.resources.file
+                } catch (err: any) {
+                    if (err instanceof SyntaxError) {
+                        log.error('Invalid JSON in response, err: %s', err)
+                        throw err
+                    }
+                    log.error('Upload to %s failed: %s', uploadUrl, err.stack)
+                    throw err
+                }
+            },
+
+            storeByPath: (path: string, type: string, meta: FileMetadata): Promise<UploadedFile> =>
+                plugin.store(type, createReadStream(path), meta),
+
+            get: async (href: string, channel: string, jwt: string): Promise<Readable> => {
+                const apkUrl = url.resolve(options.storageUrl, href)
+                const res = await fetch(apkUrl, {
+                    headers: {
+                        channel,
+                        Authorization: `Bearer ${jwt}`,
+                        device: options.serial
+                    }
+                })
+
+                log.info('Reading %s returned: %s', apkUrl, res.status)
+
+                if (res.status >= 300) {
+                    throw Error(`Could not download file. Server returned status = ${res.status}, ${await res.text()}`)
+                }
+                if (res.body === null) {
+                    throw Error(`Could not download file. Server returned no body and status = ${res.status}`)
+                }
+
+                return Readable.fromWeb(res.body as any)
+            },
+
+            download: async (
+                href: string,
+                channel: string,
+                jwt: string,
+                localPath?: string,
+                name?: string
+            ): Promise<DownloadedFile> => {
+                const fileStream = await plugin.get(href, channel, jwt)
+                const file = (localPath && { path: localPath }) || await temp.file(name ? { name } : {})
+                const writeStream = createWriteStream(file.path)
+
+                log.info('Downloading to %s', file.path)
+
+                await finished(fileStream.pipe(writeStream))
+
+                return {
+                    path: file.path,
+                    cleanup: () =>
+                        (file as any)?.cleanup ? (file as any).cleanup() : unlink(file.path)
+                }
+            }
+        }
+
+        return plugin
+    })