diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb03b5790..353ead21a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,9 +122,9 @@ jobs: if: ${{ env.DOCKERHUB_PASSWORD && env.DOCKERHUB_USERNAME }} run: | echo "Login to Docker Hub";echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Run Barge working-directory: ${{ github.workspace }}/barge run: | @@ -145,6 +145,19 @@ jobs: - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} + - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD + run: | + DOCKER_REGISTRY_AUTHS=$(jq -n \ + --arg username "$DOCKERHUB_USERNAME" \ + --arg password "$DOCKERHUB_PASSWORD" \ + '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }') + echo "DOCKER_REGISTRY_AUTHS<> $GITHUB_ENV + echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: integration tests run: npm run test:integration:cover env: @@ -162,6 +175,7 @@ jobs: FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }' ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json' ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json' + DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }} - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} @@ -237,7 +251,19 @@ jobs: repository: 'oceanprotocol/ocean-node' path: 'ocean-node' ref: ${{ github.event_name == 'pull_request' && github.head_ref || 'main' }} - + - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD + run: | + DOCKER_REGISTRY_AUTHS=$(jq -n \ + --arg username "$DOCKERHUB_USERNAME" \ + --arg password "$DOCKERHUB_PASSWORD" \ + '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }') + echo "DOCKER_REGISTRY_AUTHS<> $GITHUB_ENV + echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Start Ocean Node working-directory: ${{ github.workspace }}/ocean-node run: | @@ -265,6 +291,7 @@ jobs: MAX_REQ_PER_MINUTE: 320 MAX_CONNECTIONS_PER_MINUTE: 320 DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":10}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration": 60,"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"minJobDuration": 10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]' + DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }} - name: Check Ocean Node is running run: | for i in $(seq 1 90); do diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ad32a67c2..941c28408 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} password: ${{ secrets.DOCKER_PUSH_TOKEN }} - name: Set Docker metadata @@ -111,7 +111,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} password: ${{ secrets.DOCKER_PUSH_TOKEN }} - name: Create manifest list and push working-directory: /tmp/digests diff --git a/src/@types/fileObject.ts b/src/@types/fileObject.ts index d29636215..3dbf42873 100644 --- a/src/@types/fileObject.ts +++ b/src/@types/fileObject.ts @@ -19,7 +19,7 @@ export interface BaseFileObject { export interface UrlFileObject extends BaseFileObject { url: string method: string - headers?: [HeadersObject] + headers?: HeadersObject } export interface IpfsFileObject extends BaseFileObject { diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index 0e3365763..dcbb4541d 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -54,12 +54,7 @@ export async function getAlgoChecksums( : null const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined - const { contentChecksum } = await fetchFileMetadata( - url, - 'get', - false, - headers ? headers[0] : undefined - ) + const { contentChecksum } = await fetchFileMetadata(url, 'get', false, headers) checksums.files = checksums.files.concat(contentChecksum) } diff --git a/src/components/core/handler/fileInfoHandler.ts b/src/components/core/handler/fileInfoHandler.ts index 5af575e73..d0677f8b3 100644 --- a/src/components/core/handler/fileInfoHandler.ts +++ b/src/components/core/handler/fileInfoHandler.ts @@ -39,7 +39,7 @@ async function formatMetadata( url, 'get', false, - headers ? headers[0] : undefined + headers ) CORE_LOGGER.logMessage(`Metadata for file: ${contentLength} ${contentType}`) diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index 0dd30da13..f10269f4e 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -151,10 +151,11 @@ export class UrlStorage extends Storage { async getReadableStream(): Promise { const input = this.getDownloadUrl() const file = this.getFile() + const { headers } = file const response = await axios({ method: 'get', url: input, - headers: file.headers ? file.headers[0] : undefined, + headers, responseType: 'stream', timeout: 30000 }) @@ -215,9 +216,9 @@ export class UrlStorage extends Storage { const { url, method, headers } = fileObject const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( url, - method, + method || 'get', forceChecksum, - headers ? headers[0] : undefined + headers ) return { valid: true, diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts index 98013d4c6..43ad6cbab 100644 --- a/src/test/unit/storage.test.ts +++ b/src/test/unit/storage.test.ts @@ -29,12 +29,10 @@ describe('URL Storage tests', () => { type: 'url', url: 'http://someUrl.com/file.json', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ], + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + }, encryptedBy: nodeId, encryptMethod: EncryptMethod.AES } @@ -70,12 +68,10 @@ describe('URL Storage tests', () => { file = { type: 'url', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -88,12 +84,10 @@ describe('URL Storage tests', () => { file = { type: 'url', url: 'http://someUrl.com/file.json', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -109,12 +103,10 @@ describe('URL Storage tests', () => { type: 'url', url: 'http://someUrl.com/file.json', method: 'put', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -129,12 +121,10 @@ describe('URL Storage tests', () => { type: 'url', url: './../dir/file.json', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -150,12 +140,10 @@ describe('URL Storage tests', () => { type: 'url', url: 'http://someUrl.com/file.json', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } storage = Storage.getStorageClass(file, config) expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') @@ -172,12 +160,12 @@ describe('URL Storage tests', () => { expect(stream).not.to.eql(null) }) - it('Gets readable stream with headers', async () => { + it('Gets readable stream with headers as plain object', async () => { file = { type: 'url', url: 'https://stock-api.oceanprotocol.com/stock/stock.json', method: 'get', - headers: [{ 'X-Test-Header': 'test' }] + headers: { 'X-Test-Header': 'test' } } const storage = Storage.getStorageClass(file, config) const stream = await storage.getReadableStream()