Fix/1184 file headers not sent

.github/workflows/ci.yml

@@ -122,9 +122,9 @@ jobs:
         if: ${{ env.DOCKERHUB_PASSWORD && env.DOCKERHUB_USERNAME }}
         run: |
           echo "Login to Docker Hub";echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
-          env:
-              DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-              DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Run Barge
         working-directory: ${{ github.workspace }}/barge
         run: |
@@ -145,6 +145,19 @@ jobs:
       - name: docker logs
         run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1
         if: ${{ failure() }}
+      - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets
+        if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD
+        run: |
+          DOCKER_REGISTRY_AUTHS=$(jq -n \
+            --arg username "$DOCKERHUB_USERNAME" \
+            --arg password "$DOCKERHUB_PASSWORD" \
+            '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }')
+          echo "DOCKER_REGISTRY_AUTHS<<EOF" >> $GITHUB_ENV
+          echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: integration tests
         run: npm run test:integration:cover
         env:
@@ -162,6 +175,7 @@ jobs:
           FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }'
           ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json'
           ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json'
+          DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }}
       - name: docker logs
         run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1
         if: ${{ failure() }}
@@ -237,7 +251,19 @@ jobs:
           repository: 'oceanprotocol/ocean-node'
           path: 'ocean-node'
           ref: ${{ github.event_name == 'pull_request' && github.head_ref || 'main' }}
-
+      - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets
+        if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD
+        run: |
+          DOCKER_REGISTRY_AUTHS=$(jq -n \
+            --arg username "$DOCKERHUB_USERNAME" \
+            --arg password "$DOCKERHUB_PASSWORD" \
+            '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }')
+          echo "DOCKER_REGISTRY_AUTHS<<EOF" >> $GITHUB_ENV
+          echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Start Ocean Node
         working-directory: ${{ github.workspace }}/ocean-node
         run: |
@@ -265,6 +291,7 @@ jobs:
           MAX_REQ_PER_MINUTE: 320
           MAX_CONNECTIONS_PER_MINUTE: 320
           DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":10}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration": 60,"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"minJobDuration": 10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]'
+          DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }}
       - name: Check Ocean Node is running
         run: |
           for i in $(seq 1 90); do

.github/workflows/docker.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }}
           password: ${{ secrets.DOCKER_PUSH_TOKEN }}
 
       - name: Set Docker metadata
@@ -111,7 +111,7 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }}
           password: ${{ secrets.DOCKER_PUSH_TOKEN }}
       - name: Create manifest list and push
         working-directory: /tmp/digests

src/@types/fileObject.ts

@@ -19,7 +19,7 @@ export interface BaseFileObject {
 export interface UrlFileObject extends BaseFileObject {
   url: string
   method: string
-  headers?: [HeadersObject]
+  headers?: HeadersObject
 }
 
 export interface IpfsFileObject extends BaseFileObject {

src/components/core/compute/utils.ts

@@ -54,12 +54,7 @@ export async function getAlgoChecksums(
               : null
       const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined
 
-      const { contentChecksum } = await fetchFileMetadata(
-        url,
-        'get',
-        false,
-        headers ? headers[0] : undefined
-      )
+      const { contentChecksum } = await fetchFileMetadata(url, 'get', false, headers)
       checksums.files = checksums.files.concat(contentChecksum)
     }
 

src/components/core/handler/fileInfoHandler.ts

@@ -39,7 +39,7 @@ async function formatMetadata(
     url,
     'get',
     false,
-    headers ? headers[0] : undefined
+    headers
   )
   CORE_LOGGER.logMessage(`Metadata for file: ${contentLength} ${contentType}`)
 

src/components/storage/index.ts

@@ -151,10 +151,11 @@ export class UrlStorage extends Storage {
   async getReadableStream(): Promise<StorageReadable> {
     const input = this.getDownloadUrl()
     const file = this.getFile()
+    const { headers } = file
     const response = await axios({
       method: 'get',
       url: input,
-      headers: file.headers ? file.headers[0] : undefined,
+      headers,
       responseType: 'stream',
       timeout: 30000
     })
@@ -215,9 +216,9 @@ export class UrlStorage extends Storage {
     const { url, method, headers } = fileObject
     const { contentLength, contentType, contentChecksum } = await fetchFileMetadata(
       url,
-      method,
+      method || 'get',
       forceChecksum,
-      headers ? headers[0] : undefined
+      headers
     )
     return {
       valid: true,

src/test/unit/storage.test.ts

@@ -29,12 +29,10 @@ describe('URL Storage tests', () => {
     type: 'url',
     url: 'http://someUrl.com/file.json',
     method: 'get',
-    headers: [
-      {
-        'Content-Type': 'application/json',
-        Authorization: 'Bearer auth_token_X'
-      }
-    ],
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: 'Bearer auth_token_X'
+    },
     encryptedBy: nodeId,
     encryptMethod: EncryptMethod.AES
   }
@@ -70,12 +68,10 @@ describe('URL Storage tests', () => {
     file = {
       type: 'url',
       method: 'get',
-      headers: [
-        {
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer auth_token_X'
-        }
-      ]
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: 'Bearer auth_token_X'
+      }
     }
     try {
       Storage.getStorageClass(file, config)
@@ -88,12 +84,10 @@ describe('URL Storage tests', () => {
     file = {
       type: 'url',
       url: 'http://someUrl.com/file.json',
-      headers: [
-        {
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer auth_token_X'
-        }
-      ]
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: 'Bearer auth_token_X'
+      }
     }
     try {
       Storage.getStorageClass(file, config)
@@ -109,12 +103,10 @@ describe('URL Storage tests', () => {
       type: 'url',
       url: 'http://someUrl.com/file.json',
       method: 'put',
-      headers: [
-        {
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer auth_token_X'
-        }
-      ]
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: 'Bearer auth_token_X'
+      }
     }
     try {
       Storage.getStorageClass(file, config)
@@ -129,12 +121,10 @@ describe('URL Storage tests', () => {
       type: 'url',
       url: './../dir/file.json',
       method: 'get',
-      headers: [
-        {
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer auth_token_X'
-        }
-      ]
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: 'Bearer auth_token_X'
+      }
     }
     try {
       Storage.getStorageClass(file, config)
@@ -150,12 +140,10 @@ describe('URL Storage tests', () => {
       type: 'url',
       url: 'http://someUrl.com/file.json',
       method: 'get',
-      headers: [
-        {
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer auth_token_X'
-        }
-      ]
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: 'Bearer auth_token_X'
+      }
     }
     storage = Storage.getStorageClass(file, config)
     expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json')
@@ -172,12 +160,12 @@ describe('URL Storage tests', () => {
     expect(stream).not.to.eql(null)
   })
 
-  it('Gets readable stream with headers', async () => {
+  it('Gets readable stream with headers as plain object', async () => {
     file = {
       type: 'url',
       url: 'https://stock-api.oceanprotocol.com/stock/stock.json',
       method: 'get',
-      headers: [{ 'X-Test-Header': 'test' }]
+      headers: { 'X-Test-Header': 'test' }
     }
     const storage = Storage.getStorageClass(file, config)
     const stream = await storage.getReadableStream()