From 0d6a7e39500095bd65568ffd75c63163157e04f2 Mon Sep 17 00:00:00 2001 From: ndrpp Date: Wed, 4 Feb 2026 14:16:33 +0200 Subject: [PATCH 1/7] fix: use file object headers when accessing file --- src/components/core/compute/utils.ts | 3 ++- src/components/core/handler/fileInfoHandler.ts | 5 ++++- src/components/storage/index.ts | 9 +++++++-- src/test/unit/storage.test.ts | 12 ++++++++++++ src/utils/asset.ts | 7 +++++-- src/utils/logging/Logger.ts | 2 +- 6 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index 735af9351..3687e70c6 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -52,8 +52,9 @@ export async function getAlgoChecksums( : file.type === 'ipfs' ? urlJoin(config.ipfsGateway, (file as IpfsFileObject).hash) : null + const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined - const { contentChecksum } = await fetchFileMetadata(url, 'get', false) + const { contentChecksum } = await fetchFileMetadata(url, 'get', false, headers ? headers[0] : undefined) checksums.files = checksums.files.concat(contentChecksum) } diff --git a/src/components/core/handler/fileInfoHandler.ts b/src/components/core/handler/fileInfoHandler.ts index 342de9f4c..5af575e73 100644 --- a/src/components/core/handler/fileInfoHandler.ts +++ b/src/components/core/handler/fileInfoHandler.ts @@ -20,6 +20,7 @@ import { } from '../../httpRoutes/validateCommands.js' import { getFile } from '../../../utils/file.js' import { getConfiguration } from '../../../utils/index.js' + async function formatMetadata( file: ArweaveFileObject | IpfsFileObject | UrlFileObject, config: OceanNodeConfig @@ -32,11 +33,13 @@ async function formatMetadata( : file.type === 'ipfs' ? urlJoin(config.ipfsGateway, (file as IpfsFileObject).hash) : null + const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( url, 'get', - false + false, + headers ? headers[0] : undefined ) CORE_LOGGER.logMessage(`Metadata for file: ${contentLength} ${contentType}`) diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index 6842da24e..e3d6b31d2 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -43,9 +43,11 @@ export abstract class Storage { // similar to all subclasses async getReadableStream(): Promise { const input = this.getDownloadUrl() + const file = this.getFile() as UrlFileObject const response = await axios({ method: 'get', url: input, + headers: file.headers ? file.headers[0] : undefined, responseType: 'stream', timeout: 30000 }) @@ -194,11 +196,12 @@ export class UrlStorage extends Storage { fileObject: UrlFileObject, forceChecksum: boolean ): Promise { - const { url, method } = fileObject + const { url, method, headers } = fileObject const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( url, method, - forceChecksum + forceChecksum, + headers ? headers[0] : undefined ) return { valid: true, @@ -211,6 +214,8 @@ export class UrlStorage extends Storage { encryptMethod: fileObject.encryptMethod } } + + } export class ArweaveStorage extends Storage { diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts index 5a2f200f8..98013d4c6 100644 --- a/src/test/unit/storage.test.ts +++ b/src/test/unit/storage.test.ts @@ -171,6 +171,18 @@ describe('URL Storage tests', () => { const stream = await storage.getReadableStream() expect(stream).not.to.eql(null) }) + + it('Gets readable stream with headers', async () => { + file = { + type: 'url', + url: 'https://stock-api.oceanprotocol.com/stock/stock.json', + method: 'get', + headers: [{ 'X-Test-Header': 'test' }] + } + const storage = Storage.getStorageClass(file, config) + const stream = await storage.getReadableStream() + expect(stream).not.to.eql(null) + }) }) describe('Unsafe URL tests', () => { diff --git a/src/utils/asset.ts b/src/utils/asset.ts index cdab24390..897e8283e 100644 --- a/src/utils/asset.ts +++ b/src/utils/asset.ts @@ -9,6 +9,7 @@ import { KNOWN_CONFIDENTIAL_EVMS } from './address.js' import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/interfaces/IERC20Template.sol/IERC20Template.json' with { type: 'json' } import ERC20Template4 from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20Template4.sol/ERC20Template4.json' with { type: 'json' } import { getContractAddress, getNFTFactory } from '../components/Indexer/utils.js' +import { HeadersObject } from '../@types/fileObject.js' // Notes: // Asset as per asset.py on provider, is a class there, while on ocean.Js we only have a type @@ -44,7 +45,8 @@ export const AssetUtils = { export async function fetchFileMetadata( url: string, method: string, - forceChecksum: boolean + forceChecksum: boolean, + headers?: HeadersObject ): Promise<{ contentLength: string; contentType: string; contentChecksum: string }> { let contentType: string = '' let contentLength: number = 0 @@ -56,6 +58,7 @@ export async function fetchFileMetadata( const response = await axios({ url, method: method || 'get', + headers, responseType: 'stream', timeout: 30000 }) @@ -143,7 +146,7 @@ export async function isERC20Template4Active( ): Promise { try { const nftFactoryAddress = getContractAddress(network, 'ERC721Factory') - const factoryERC721 = await getNFTFactory(owner, nftFactoryAddress) + const factoryERC721 = getNFTFactory(owner, nftFactoryAddress) const currentTokenCount = await factoryERC721.getCurrentTemplateCount() for (let i = 1; i <= currentTokenCount; i++) { diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts index 7c47c89b5..25b010a4f 100644 --- a/src/utils/logging/Logger.ts +++ b/src/utils/logging/Logger.ts @@ -198,7 +198,7 @@ const alignedWithColorsAndTime: winston.Logform.Format = winston.format.combine( winston.format.timestamp(), winston.format.align(), winston.format.printf( - (info) => `${info.timestamp} ${info.level}: ${info.message.trim()}` + (info) => `${info.timestamp} ${info.level}: ${(info.message as string).trim()}` ) ) const consoleColorFormatting: winston.Logform.Format | Record = { From 28c4aaa5da64ebc731d4e5d47973be8b5b1eb4ca Mon Sep 17 00:00:00 2001 From: ndrpp Date: Wed, 4 Feb 2026 14:24:00 +0200 Subject: [PATCH 2/7] fix: review --- src/components/P2P/index.ts | 4 ++-- src/components/storage/index.ts | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/components/P2P/index.ts b/src/components/P2P/index.ts index cca8e3c74..986c345fe 100644 --- a/src/components/P2P/index.ts +++ b/src/components/P2P/index.ts @@ -913,8 +913,8 @@ export class OceanP2P extends EventEmitter { const cid = await cidFromRawString(input) const peersFound = [] try { - // @ts-ignore ignore the type mismatch - const f = await this._libp2p.contentRouting.findProviders(cid, { + const f = this._libp2p.contentRouting.findProviders(cid, { + // @ts-ignore ignore the type mismatch queryFuncTimeout: timeout || 20000 // 20 seconds // on timeout the query ends with an abort signal => CodeError: Query aborted }) diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index e3d6b31d2..5302b8d71 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -214,8 +214,6 @@ export class UrlStorage extends Storage { encryptMethod: fileObject.encryptMethod } } - - } export class ArweaveStorage extends Storage { From 4fecb805ffcd4b96c78caff7bf58c2554c73e47d Mon Sep 17 00:00:00 2001 From: ndrpp Date: Wed, 4 Feb 2026 14:26:57 +0200 Subject: [PATCH 3/7] fix: build issue & prettier --- src/components/P2P/index.ts | 2 +- src/components/core/compute/utils.ts | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/components/P2P/index.ts b/src/components/P2P/index.ts index 986c345fe..6e0bb4fee 100644 --- a/src/components/P2P/index.ts +++ b/src/components/P2P/index.ts @@ -913,8 +913,8 @@ export class OceanP2P extends EventEmitter { const cid = await cidFromRawString(input) const peersFound = [] try { + // @ts-ignore ignore the type mismatch const f = this._libp2p.contentRouting.findProviders(cid, { - // @ts-ignore ignore the type mismatch queryFuncTimeout: timeout || 20000 // 20 seconds // on timeout the query ends with an abort signal => CodeError: Query aborted }) diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index 3687e70c6..0e3365763 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -54,7 +54,12 @@ export async function getAlgoChecksums( : null const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined - const { contentChecksum } = await fetchFileMetadata(url, 'get', false, headers ? headers[0] : undefined) + const { contentChecksum } = await fetchFileMetadata( + url, + 'get', + false, + headers ? headers[0] : undefined + ) checksums.files = checksums.files.concat(contentChecksum) } From 435bb74bb1dd80321a838d25de9fb4032e24e078 Mon Sep 17 00:00:00 2001 From: ndrpp Date: Thu, 5 Feb 2026 10:38:29 +0200 Subject: [PATCH 4/7] fix: code review --- src/components/storage/index.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index 5302b8d71..0dd30da13 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -43,11 +43,9 @@ export abstract class Storage { // similar to all subclasses async getReadableStream(): Promise { const input = this.getDownloadUrl() - const file = this.getFile() as UrlFileObject const response = await axios({ method: 'get', url: input, - headers: file.headers ? file.headers[0] : undefined, responseType: 'stream', timeout: 30000 }) @@ -150,6 +148,24 @@ export class UrlStorage extends Storage { } } + async getReadableStream(): Promise { + const input = this.getDownloadUrl() + const file = this.getFile() + const response = await axios({ + method: 'get', + url: input, + headers: file.headers ? file.headers[0] : undefined, + responseType: 'stream', + timeout: 30000 + }) + + return { + httpStatus: response.status, + stream: response.data, + headers: response.headers as any + } + } + validate(): [boolean, string] { const file: UrlFileObject = this.getFile() as UrlFileObject if (!file.url || !file.method) { From 2427d206ae6786281c337691ab8dcf452405bf6f Mon Sep 17 00:00:00 2001 From: ndrpp Date: Thu, 5 Feb 2026 15:40:25 +0200 Subject: [PATCH 5/7] fix: simplify file headers usage --- src/@types/fileObject.ts | 2 +- src/components/core/compute/utils.ts | 2 +- .../core/handler/fileInfoHandler.ts | 2 +- src/components/storage/index.ts | 7 +- src/test/unit/storage.test.ts | 64 ++++++++----------- 5 files changed, 33 insertions(+), 44 deletions(-) diff --git a/src/@types/fileObject.ts b/src/@types/fileObject.ts index d29636215..3dbf42873 100644 --- a/src/@types/fileObject.ts +++ b/src/@types/fileObject.ts @@ -19,7 +19,7 @@ export interface BaseFileObject { export interface UrlFileObject extends BaseFileObject { url: string method: string - headers?: [HeadersObject] + headers?: HeadersObject } export interface IpfsFileObject extends BaseFileObject { diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index 0e3365763..6485ff645 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -58,7 +58,7 @@ export async function getAlgoChecksums( url, 'get', false, - headers ? headers[0] : undefined + headers ) checksums.files = checksums.files.concat(contentChecksum) } diff --git a/src/components/core/handler/fileInfoHandler.ts b/src/components/core/handler/fileInfoHandler.ts index 5af575e73..d0677f8b3 100644 --- a/src/components/core/handler/fileInfoHandler.ts +++ b/src/components/core/handler/fileInfoHandler.ts @@ -39,7 +39,7 @@ async function formatMetadata( url, 'get', false, - headers ? headers[0] : undefined + headers ) CORE_LOGGER.logMessage(`Metadata for file: ${contentLength} ${contentType}`) diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index 0dd30da13..f10269f4e 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -151,10 +151,11 @@ export class UrlStorage extends Storage { async getReadableStream(): Promise { const input = this.getDownloadUrl() const file = this.getFile() + const { headers } = file const response = await axios({ method: 'get', url: input, - headers: file.headers ? file.headers[0] : undefined, + headers, responseType: 'stream', timeout: 30000 }) @@ -215,9 +216,9 @@ export class UrlStorage extends Storage { const { url, method, headers } = fileObject const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( url, - method, + method || 'get', forceChecksum, - headers ? headers[0] : undefined + headers ) return { valid: true, diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts index 98013d4c6..43ad6cbab 100644 --- a/src/test/unit/storage.test.ts +++ b/src/test/unit/storage.test.ts @@ -29,12 +29,10 @@ describe('URL Storage tests', () => { type: 'url', url: 'http://someUrl.com/file.json', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ], + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + }, encryptedBy: nodeId, encryptMethod: EncryptMethod.AES } @@ -70,12 +68,10 @@ describe('URL Storage tests', () => { file = { type: 'url', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -88,12 +84,10 @@ describe('URL Storage tests', () => { file = { type: 'url', url: 'http://someUrl.com/file.json', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -109,12 +103,10 @@ describe('URL Storage tests', () => { type: 'url', url: 'http://someUrl.com/file.json', method: 'put', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -129,12 +121,10 @@ describe('URL Storage tests', () => { type: 'url', url: './../dir/file.json', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } try { Storage.getStorageClass(file, config) @@ -150,12 +140,10 @@ describe('URL Storage tests', () => { type: 'url', url: 'http://someUrl.com/file.json', method: 'get', - headers: [ - { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - ] + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } } storage = Storage.getStorageClass(file, config) expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') @@ -172,12 +160,12 @@ describe('URL Storage tests', () => { expect(stream).not.to.eql(null) }) - it('Gets readable stream with headers', async () => { + it('Gets readable stream with headers as plain object', async () => { file = { type: 'url', url: 'https://stock-api.oceanprotocol.com/stock/stock.json', method: 'get', - headers: [{ 'X-Test-Header': 'test' }] + headers: { 'X-Test-Header': 'test' } } const storage = Storage.getStorageClass(file, config) const stream = await storage.getReadableStream() From 32c528ea56b6bab1e82e04346522dcb884562c35 Mon Sep 17 00:00:00 2001 From: ndrpp Date: Thu, 5 Feb 2026 15:46:06 +0200 Subject: [PATCH 6/7] fix: formatting --- src/components/core/compute/utils.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index 6485ff645..dcbb4541d 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -54,12 +54,7 @@ export async function getAlgoChecksums( : null const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined - const { contentChecksum } = await fetchFileMetadata( - url, - 'get', - false, - headers - ) + const { contentChecksum } = await fetchFileMetadata(url, 'get', false, headers) checksums.files = checksums.files.concat(contentChecksum) } From e4aa6800913ef858ec4e5d0678d3f3051c6aa336 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 15:51:37 +0200 Subject: [PATCH 7/7] fix ci --- .github/workflows/ci.yml | 35 +++++++++++++++++++++++++++++++---- .github/workflows/docker.yml | 4 ++-- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb03b5790..353ead21a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,9 +122,9 @@ jobs: if: ${{ env.DOCKERHUB_PASSWORD && env.DOCKERHUB_USERNAME }} run: | echo "Login to Docker Hub";echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Run Barge working-directory: ${{ github.workspace }}/barge run: | @@ -145,6 +145,19 @@ jobs: - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} + - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD + run: | + DOCKER_REGISTRY_AUTHS=$(jq -n \ + --arg username "$DOCKERHUB_USERNAME" \ + --arg password "$DOCKERHUB_PASSWORD" \ + '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }') + echo "DOCKER_REGISTRY_AUTHS<> $GITHUB_ENV + echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: integration tests run: npm run test:integration:cover env: @@ -162,6 +175,7 @@ jobs: FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }' ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json' ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json' + DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }} - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} @@ -237,7 +251,19 @@ jobs: repository: 'oceanprotocol/ocean-node' path: 'ocean-node' ref: ${{ github.event_name == 'pull_request' && github.head_ref || 'main' }} - + - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD + run: | + DOCKER_REGISTRY_AUTHS=$(jq -n \ + --arg username "$DOCKERHUB_USERNAME" \ + --arg password "$DOCKERHUB_PASSWORD" \ + '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }') + echo "DOCKER_REGISTRY_AUTHS<> $GITHUB_ENV + echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Start Ocean Node working-directory: ${{ github.workspace }}/ocean-node run: | @@ -265,6 +291,7 @@ jobs: MAX_REQ_PER_MINUTE: 320 MAX_CONNECTIONS_PER_MINUTE: 320 DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":10}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration": 60,"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"minJobDuration": 10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]' + DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }} - name: Check Ocean Node is running run: | for i in $(seq 1 90); do diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ad32a67c2..941c28408 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} password: ${{ secrets.DOCKER_PUSH_TOKEN }} - name: Set Docker metadata @@ -111,7 +111,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} password: ${{ secrets.DOCKER_PUSH_TOKEN }} - name: Create manifest list and push working-directory: /tmp/digests