From 2e2e12f1af97bd00b1ccc9dc97f802fbed2df4e3 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 11:27:41 +0200 Subject: [PATCH 1/9] add dockerRegistrysAuth --- docs/env.md | 35 +++ src/@types/OceanNode.ts | 10 + src/components/c2d/compute_engine_base.ts | 20 +- src/components/c2d/compute_engine_docker.ts | 80 +++++- src/components/c2d/compute_engines.ts | 15 +- src/components/core/compute/initialize.ts | 5 +- .../integration/dockerRegistryAuth.test.ts | 243 ++++++++++++++++++ src/test/integration/imageCleanup.test.ts | 2 +- src/utils/config/constants.ts | 1 + src/utils/config/schemas.ts | 10 + src/utils/constants.ts | 5 + 11 files changed, 408 insertions(+), 18 deletions(-) create mode 100644 src/test/integration/dockerRegistryAuth.test.ts diff --git a/docs/env.md b/docs/env.md index 520502578..59e41d03b 100644 --- a/docs/env.md +++ b/docs/env.md @@ -226,3 +226,38 @@ The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable should be a JSON array of - **total**: Total number of the resource available. - **min**: Minimum number of the resource needed for a job. - **max**: Maximum number of the resource for a job. + +### Docker Registry Authentication + +- `DOCKER_REGISTRY_AUTHS`: JSON object mapping Docker registry URLs to authentication credentials. Used for accessing private Docker/OCI registries when validating and pulling Docker images. Each registry entry must provide either `username`+`password` or `auth`. Example: + +```json +{ + "https://registry-1.docker.io": { + "username": "myuser", + "password": "mypassword" + }, + "https://ghcr.io": { + "username": "myuser", + "password": "ghp_..." + }, + "https://registry.gitlab.com": { + "auth": "glpat-..." + } +} +``` + +**Configuration Options:** + +- **Registry URL** (key): The full registry URL including protocol (e.g., `https://registry-1.docker.io`, `https://ghcr.io`, `https://registry.gitlab.com`) +- **username** (optional): Username for registry authentication. Required if using password-based auth. +- **password** (optional): Password or personal access token for registry authentication. Required if using username-based auth. +- **auth** (optional): Authentication token (alternative to username+password). Required if not using username+password. + +**Notes:** + +- For Docker Hub (`registry-1.docker.io`), you can use your Docker Hub username and password, or a personal access token (PAT) as the password. +- For GitHub Container Registry (GHCR), use your GitHub username with a personal access token (PAT) as the password, or use a token directly. +- For GitLab Container Registry, use a personal access token (PAT) or deploy token. +- The registry URL must match exactly (including protocol) with the registry used in the Docker image reference. +- If no credentials are configured for a registry, the node will attempt unauthenticated access (works for public images only). diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index 9cda85acb..a6cbf9d63 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -93,8 +93,18 @@ export interface AccessListContract { [chainId: string]: string[] } +export interface dockerRegistryAuth { + username: string + password: string + auth: string +} +export interface dockerRegistrysAuth { + [registry: string]: dockerRegistryAuth +} + export interface OceanNodeConfig { dockerComputeEnvironments: C2DDockerConfig[] + dockerRegistrysAuth: dockerRegistrysAuth authorizedDecrypters: string[] authorizedDecryptersList: AccessListContract | null allowedValidators: string[] diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index 13e49a6a4..ec0c21e8e 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -21,22 +21,27 @@ import { C2DClusterType } from '../../@types/C2D/C2D.js' import { C2DDatabase } from '../database/C2DDatabase.js' import { Escrow } from '../core/utils/escrow.js' import { KeyManager } from '../KeyManager/index.js' - +import { dockerRegistryAuth, dockerRegistrysAuth } from '../../@types/OceanNode.js' +import { ValidateParams } from '../httpRoutes/validateCommands.js' export abstract class C2DEngine { private clusterConfig: C2DClusterInfo public db: C2DDatabase public escrow: Escrow public keyManager: KeyManager + public dockerRegistryAuths: dockerRegistrysAuth + public constructor( cluster: C2DClusterInfo, db: C2DDatabase, escrow: Escrow, - keyManager: KeyManager + keyManager: KeyManager, + dockerRegistryAuths: dockerRegistrysAuth ) { this.clusterConfig = cluster this.db = db this.escrow = escrow this.keyManager = keyManager + this.dockerRegistryAuths = dockerRegistryAuths } getKeyManager(): KeyManager { @@ -66,6 +71,9 @@ export abstract class C2DEngine { return null } + // eslint-disable-next-line require-await + public abstract checkDockerImage(image: string, platform?: any): Promise + public abstract startComputeJob( assets: ComputeAsset[], algorithm: ComputeAlgorithm, @@ -523,4 +531,12 @@ export abstract class C2DEngine { } return cost } + + public getDockerRegistryAuth(registry: string): dockerRegistryAuth | null { + if (!this.dockerRegistryAuths) return null + if (this.dockerRegistryAuths[registry]) { + return this.dockerRegistryAuths[registry] + } + return null + } } diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 7459e7ce2..8c3f2d9c0 100644 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -51,6 +51,7 @@ import { decryptFilesObject, omitDBComputeFieldsFromComputeJob } from './index.j import { ValidateParams } from '../httpRoutes/validateCommands.js' import { Service } from '@oceanprotocol/ddo-js' import { getOceanTokenAddressForChain } from '../../utils/address.js' +import { dockerRegistrysAuth } from '../../@types/OceanNode.js' export class C2DEngineDocker extends C2DEngine { private envs: ComputeEnvironment[] = [] @@ -69,9 +70,10 @@ export class C2DEngineDocker extends C2DEngine { clusterConfig: C2DClusterInfo, db: C2DDatabase, escrow: Escrow, - keyManager: KeyManager + keyManager: KeyManager, + dockerRegistryAuths: dockerRegistrysAuth ) { - super(clusterConfig, db, escrow, keyManager) + super(clusterConfig, db, escrow, keyManager, dockerRegistryAuths) this.docker = null if (clusterConfig.connection.socketPath) { @@ -438,7 +440,7 @@ export class C2DEngineDocker extends C2DEngine { return filteredEnvs } - private static parseImage(image: string) { + private parseImage(image: string) { let registry = C2DEngineDocker.DEFAULT_DOCKER_REGISTRY let name = image let ref = 'latest' @@ -472,13 +474,29 @@ export class C2DEngineDocker extends C2DEngine { return { registry, name, ref } } - public static async getDockerManifest(image: string): Promise { - const { registry, name, ref } = C2DEngineDocker.parseImage(image) + public async getDockerManifest(image: string): Promise { + const { registry, name, ref } = this.parseImage(image) const url = `${registry}/v2/${name}/manifests/${ref}` + + // Get registry auth from parent class + const dockerRegistryAuth = this.getDockerRegistryAuth(registry) + let headers: Record = { Accept: 'application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json' } + + // If we have auth credentials, add Basic auth header to initial request + if (dockerRegistryAuth) { + // Use auth string if available, otherwise encode username:password + const authString = dockerRegistryAuth.auth + ? dockerRegistryAuth.auth + : Buffer.from( + `${dockerRegistryAuth.username}:${dockerRegistryAuth.password}` + ).toString('base64') + headers.Authorization = `Basic ${authString}` + } + let response = await fetch(url, { headers }) if (response.status === 401) { @@ -489,7 +507,22 @@ export class C2DEngineDocker extends C2DEngine { const tokenUrl = new URL(match[1]) tokenUrl.searchParams.set('service', match[2]) tokenUrl.searchParams.set('scope', `repository:${name}:pull`) - const { token } = (await fetch(tokenUrl.toString()).then((r) => r.json())) as { + + // Add Basic auth to token request if we have credentials + const tokenHeaders: Record = {} + if (dockerRegistryAuth) { + // Use auth string if available, otherwise encode username:password + const authString = dockerRegistryAuth.auth + ? dockerRegistryAuth.auth + : Buffer.from( + `${dockerRegistryAuth.username}:${dockerRegistryAuth.password}` + ).toString('base64') + tokenHeaders.Authorization = `Basic ${authString}` + } + + const { token } = (await fetch(tokenUrl.toString(), { + headers: tokenHeaders + }).then((r) => r.json())) as { token: string } headers = { ...headers, Authorization: `Bearer ${token}` } @@ -511,12 +544,12 @@ export class C2DEngineDocker extends C2DEngine { * @param image name or tag * @returns boolean */ - public static async checkDockerImage( + public async checkDockerImage( image: string, platform?: RunningPlatform ): Promise { try { - const manifest = await C2DEngineDocker.getDockerManifest(image) + const manifest = await this.getDockerManifest(image) const platforms = Array.isArray(manifest.manifests) ? manifest.manifests.map((entry: any) => entry.platform) @@ -647,7 +680,7 @@ export class C2DEngineDocker extends C2DEngine { } } else { // already built, we need to validate it - const validation = await C2DEngineDocker.checkDockerImage(image, env.platform) + const validation = await this.checkDockerImage(image, env.platform) if (!validation.valid) throw new Error( `Cannot find image ${image} for ${env.platform.architecture}. Maybe it does not exist or it's build for other arhitectures.` @@ -1644,7 +1677,34 @@ export class C2DEngineDocker extends C2DEngine { const imageLogFile = this.getC2DConfig().tempFolder + '/' + job.jobId + '/data/logs/image.log' try { - const pullStream = await this.docker.pull(job.containerImage) + // Get registry auth for the image + const { registry } = this.parseImage(job.containerImage) + const dockerRegistryAuth = this.getDockerRegistryAuth(registry) + + // Prepare authconfig for Dockerode if credentials are available + const pullOptions: any = {} + if (dockerRegistryAuth) { + // Extract hostname from registry URL (remove protocol) + const registryUrl = new URL(registry) + const serveraddress = + registryUrl.hostname + (registryUrl.port ? `:${registryUrl.port}` : '') + + // Use auth string if available, otherwise encode username:password + const authString = dockerRegistryAuth.auth + ? dockerRegistryAuth.auth + : Buffer.from( + `${dockerRegistryAuth.username}:${dockerRegistryAuth.password}` + ).toString('base64') + + pullOptions.authconfig = { + username: dockerRegistryAuth.username, + password: dockerRegistryAuth.password, + auth: authString, + serveraddress + } + } + + const pullStream = await this.docker.pull(job.containerImage, pullOptions) await new Promise((resolve, reject) => { let wroteStatusBanner = false this.docker.modem.followProgress( diff --git a/src/components/c2d/compute_engines.ts b/src/components/c2d/compute_engines.ts index 24d7cd46b..62e367f0e 100644 --- a/src/components/c2d/compute_engines.ts +++ b/src/components/c2d/compute_engines.ts @@ -1,13 +1,14 @@ import { C2DClusterType, ComputeEnvironment } from '../../@types/C2D/C2D.js' import { C2DEngine } from './compute_engine_base.js' import { C2DEngineDocker } from './compute_engine_docker.js' -import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { OceanNodeConfig, dockerRegistrysAuth } from '../../@types/OceanNode.js' import { C2DDatabase } from '../database/C2DDatabase.js' import { Escrow } from '../core/utils/escrow.js' import { KeyManager } from '../KeyManager/index.js' + export class C2DEngines { public engines: C2DEngine[] - + private dockerRegistryAuths: dockerRegistrysAuth public constructor( config: OceanNodeConfig, db: C2DDatabase, @@ -23,7 +24,15 @@ export class C2DEngines { this.engines = [] for (const cluster of config.c2dClusters) { if (cluster.type === C2DClusterType.DOCKER) { - this.engines.push(new C2DEngineDocker(cluster, db, escrow, keyManager)) + this.engines.push( + new C2DEngineDocker( + cluster, + db, + escrow, + keyManager, + config.dockerRegistrysAuth + ) + ) } } } diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index 607dd59a8..7d4d74a76 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -28,7 +28,8 @@ import { sanitizeServiceFiles } from '../../../utils/util.js' import { FindDdoHandler } from '../handler/ddoHandler.js' import { isOrderingAllowedForAsset } from '../handler/downloadHandler.js' import { getNonceAsNumber } from '../utils/nonceHandler.js' -import { C2DEngineDocker, getAlgorithmImage } from '../../c2d/compute_engine_docker.js' +import { getAlgorithmImage } from '../../c2d/compute_engine_docker.js' + import { Credentials, DDOManager } from '@oceanprotocol/ddo-js' import { checkCredentials } from '../../../utils/credentials.js' import { PolicyServer } from '../../policyServer/index.js' @@ -387,7 +388,7 @@ export class ComputeInitializeHandler extends CommandHandler { if (hasDockerImages) { const algoImage = getAlgorithmImage(task.algorithm, generateUniqueID(task)) if (algoImage) { - const validation: ValidateParams = await C2DEngineDocker.checkDockerImage( + const validation: ValidateParams = await engine.checkDockerImage( algoImage, env.platform ) diff --git a/src/test/integration/dockerRegistryAuth.test.ts b/src/test/integration/dockerRegistryAuth.test.ts new file mode 100644 index 000000000..e49c062a9 --- /dev/null +++ b/src/test/integration/dockerRegistryAuth.test.ts @@ -0,0 +1,243 @@ +/* eslint-disable no-unused-expressions */ +/** + * Integration test for Docker registry authentication functionality. + * + * Tests the getDockerManifest method with: + * - Public images (no credentials) + * - Registry auth configuration (username/password and auth string) + * - Error handling + */ +import { expect, assert } from 'chai' +import { C2DEngineDocker } from '../../components/c2d/compute_engine_docker.js' +import { C2DClusterInfo, C2DClusterType } from '../../@types/C2D/C2D.js' +import { dockerRegistrysAuth } from '../../@types/OceanNode.js' + +describe('Docker Registry Authentication Integration Tests', () => { + describe('Public registry access (no credentials)', () => { + it('should successfully fetch manifest for public Docker Hub image', async () => { + // Create minimal engine instance for testing + const clusterConfig: C2DClusterInfo = { + type: C2DClusterType.DOCKER, + hash: 'test-cluster-hash', + connection: { + socketPath: '/var/run/docker.sock' + }, + tempFolder: '/tmp/test-docker' + } + + // Mock minimal dependencies - we only need getDockerManifest + const dockerEngine = new C2DEngineDocker( + clusterConfig, + null as any, + null as any, + null as any, + {} // No auth config + ) + + // Test with a well-known public image + const image = 'library/alpine:latest' + const manifest = await dockerEngine.getDockerManifest(image) + + expect(manifest).to.exist + expect(manifest).to.have.property('schemaVersion') + expect(manifest).to.have.property('mediaType') + }).timeout(10000) + + it('should successfully fetch manifest for public image with explicit tag', async () => { + const clusterConfig: C2DClusterInfo = { + type: C2DClusterType.DOCKER, + hash: 'test-cluster-hash-2', + connection: { + socketPath: '/var/run/docker.sock' + }, + tempFolder: '/tmp/test-docker-2' + } + + const dockerEngine = new C2DEngineDocker( + clusterConfig, + null as any, + null as any, + null as any, + {} + ) + + // Use a simple image reference that will default to Docker Hub + const image = 'hello-world:latest' + const manifest = await dockerEngine.getDockerManifest(image) + + expect(manifest).to.exist + expect(manifest).to.have.property('schemaVersion') + }).timeout(10000) + }) + + describe('Registry authentication configuration', () => { + it('should store and retrieve username/password credentials', () => { + const testAuth: dockerRegistrysAuth = { + 'https://registry-1.docker.io': { + username: 'testuser', + password: 'testpass', + auth: '' + } + } + + const clusterConfig: C2DClusterInfo = { + type: C2DClusterType.DOCKER, + hash: 'test-cluster-hash-auth', + connection: { + socketPath: '/var/run/docker.sock' + }, + tempFolder: '/tmp/test-docker-auth' + } + + const engineWithAuth = new C2DEngineDocker( + clusterConfig, + null as any, + null as any, + null as any, + testAuth + ) + + // Verify that getDockerRegistryAuth returns the credentials + const auth = (engineWithAuth as any).getDockerRegistryAuth( + 'https://registry-1.docker.io' + ) + expect(auth).to.exist + expect(auth?.username).to.equal('testuser') + expect(auth?.password).to.equal('testpass') + }) + + it('should use auth string when provided', () => { + const preEncodedAuth = Buffer.from('testuser:testpass').toString('base64') + const testAuth: dockerRegistrysAuth = { + 'https://registry-1.docker.io': { + username: 'testuser', + password: 'testpass', + auth: preEncodedAuth + } + } + + const clusterConfig: C2DClusterInfo = { + type: C2DClusterType.DOCKER, + hash: 'test-cluster-hash-auth2', + connection: { + socketPath: '/var/run/docker.sock' + }, + tempFolder: '/tmp/test-docker-auth2' + } + + const engineWithAuth = new C2DEngineDocker( + clusterConfig, + null as any, + null as any, + null as any, + testAuth + ) + + const auth = (engineWithAuth as any).getDockerRegistryAuth( + 'https://registry-1.docker.io' + ) + expect(auth).to.exist + expect(auth?.auth).to.equal(preEncodedAuth) + }) + + it('should return null for non-existent registry auth', () => { + const clusterConfig: C2DClusterInfo = { + type: C2DClusterType.DOCKER, + hash: 'test-cluster-hash-3', + connection: { + socketPath: '/var/run/docker.sock' + }, + tempFolder: '/tmp/test-docker-3' + } + + const dockerEngine = new C2DEngineDocker( + clusterConfig, + null as any, + null as any, + null as any, + {} + ) + + const auth = (dockerEngine as any).getDockerRegistryAuth( + 'https://nonexistent-registry.com' + ) + expect(auth).to.be.null + }) + + it('should handle multiple registry configurations', () => { + const testAuth: dockerRegistrysAuth = { + 'https://registry-1.docker.io': { + username: 'user1', + password: 'pass1', + auth: '' + }, + 'https://ghcr.io': { + username: 'user2', + password: 'pass2', + auth: '' + } + } + + const clusterConfig: C2DClusterInfo = { + type: C2DClusterType.DOCKER, + hash: 'test-cluster-hash-multi', + connection: { + socketPath: '/var/run/docker.sock' + }, + tempFolder: '/tmp/test-docker-multi' + } + + const engineWithAuth = new C2DEngineDocker( + clusterConfig, + null as any, + null as any, + null as any, + testAuth + ) + + const dockerHubAuth = (engineWithAuth as any).getDockerRegistryAuth( + 'https://registry-1.docker.io' + ) + expect(dockerHubAuth).to.exist + expect(dockerHubAuth?.username).to.equal('user1') + + const ghcrAuth = (engineWithAuth as any).getDockerRegistryAuth('https://ghcr.io') + expect(ghcrAuth).to.exist + expect(ghcrAuth?.username).to.equal('user2') + + const unknownAuth = (engineWithAuth as any).getDockerRegistryAuth( + 'https://unknown-registry.com' + ) + expect(unknownAuth).to.be.null + }) + }) + + describe('Error handling', () => { + it('should handle invalid image references gracefully', async () => { + const clusterConfig: C2DClusterInfo = { + type: C2DClusterType.DOCKER, + hash: 'test-cluster-hash-error', + connection: { + socketPath: '/var/run/docker.sock' + }, + tempFolder: '/tmp/test-docker-error' + } + + const dockerEngine = new C2DEngineDocker( + clusterConfig, + null as any, + null as any, + null as any, + {} + ) + + try { + await dockerEngine.getDockerManifest('invalid-image-reference') + assert.fail('Should have thrown an error for invalid image') + } catch (error: any) { + expect(error).to.exist + expect(error.message).to.include('Failed to get manifest') + } + }).timeout(10000) + }) +}) diff --git a/src/test/integration/imageCleanup.test.ts b/src/test/integration/imageCleanup.test.ts index 00245525b..c4b2f7040 100644 --- a/src/test/integration/imageCleanup.test.ts +++ b/src/test/integration/imageCleanup.test.ts @@ -188,7 +188,7 @@ describe('Docker Image Cleanup Integration Tests', () => { escrow = {} as Escrow keyManager = {} as KeyManager - dockerEngine = new C2DEngineDocker(clusterConfig, db, escrow, keyManager) + dockerEngine = new C2DEngineDocker(clusterConfig, db, escrow, keyManager, null) }) it('should track image usage when image is pulled', async () => { diff --git a/src/utils/config/constants.ts b/src/utils/config/constants.ts index 8f99b8bb4..13e75535c 100644 --- a/src/utils/config/constants.ts +++ b/src/utils/config/constants.ts @@ -31,6 +31,7 @@ export const ENV_TO_CONFIG_MAPPING = { ALLOWED_ADMINS: 'allowedAdmins', ALLOWED_ADMINS_LIST: 'allowedAdminsList', DOCKER_COMPUTE_ENVIRONMENTS: 'dockerComputeEnvironments', + DOCKER_REGISTRY_AUTHS: 'dockerRegistryAuth', P2P_BOOTSTRAP_NODES: 'p2pConfig.bootstrapNodes', P2P_BOOTSTRAP_TIMEOUT: 'p2pConfig.bootstrapTimeout', P2P_BOOTSTRAP_TAGNAME: 'p2pConfig.bootstrapTagName', diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index 1139b69c5..6d3055d88 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -84,6 +84,14 @@ export const OceanNodeDBConfigSchema = z.object({ dbType: z.string().nullable() }) +export const DockerRegistryAuthSchema = z.object({ + username: z.string(), + password: z.string(), + auth: z.string() +}) + +export const DockerRegistrysSchema = z.record(z.string(), DockerRegistryAuthSchema) + export const ComputeResourceSchema = z.object({ id: z.string(), total: z.number().optional(), @@ -256,6 +264,8 @@ export const OceanNodeConfigSchema = z .optional() .default([]), + dockerRegistrysAuth: jsonFromString(DockerRegistrysSchema).optional().default({}), + authorizedDecrypters: addressArrayFromString.optional().default([]), authorizedDecryptersList: jsonFromString(AccessListContractSchema).optional(), diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 632efb277..028a12d15 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -400,6 +400,11 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.DOCKER_COMPUTE_ENVIRONMENTS, required: false }, + DOCKER_REGISTRY_AUTHS: { + name: 'DOCKER_REGISTRY_AUTHS', + value: process.env.DOCKER_REGISTRY_AUTHS, + required: false + }, DOCKER_SOCKET_PATH: { name: 'DOCKER_SOCKET_PATH', value: process.env.DOCKER_SOCKET_PATH, From 543a02a5aed7d79a387aecb01351c41fde99b1f3 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 11:55:25 +0200 Subject: [PATCH 2/9] use either u&p or auth --- src/@types/OceanNode.ts | 6 +- src/components/c2d/compute_engine_docker.ts | 13 +- .../integration/dockerRegistryAuth.test.ts | 155 ++++++++++++++++++ src/utils/config/schemas.ts | 27 ++- 4 files changed, 189 insertions(+), 12 deletions(-) diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index a6cbf9d63..15b616dd4 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -94,9 +94,9 @@ export interface AccessListContract { } export interface dockerRegistryAuth { - username: string - password: string - auth: string + username?: string + password?: string + auth?: string } export interface dockerRegistrysAuth { [registry: string]: dockerRegistryAuth diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 8c3f2d9c0..4bf4b3058 100644 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -495,6 +495,7 @@ export class C2DEngineDocker extends C2DEngine { `${dockerRegistryAuth.username}:${dockerRegistryAuth.password}` ).toString('base64') headers.Authorization = `Basic ${authString}` + CORE_LOGGER.debug(`Using docker registry auth for ${registry}`) } let response = await fetch(url, { headers }) @@ -1697,11 +1698,15 @@ export class C2DEngineDocker extends C2DEngine { ).toString('base64') pullOptions.authconfig = { - username: dockerRegistryAuth.username, - password: dockerRegistryAuth.password, - auth: authString, - serveraddress + serveraddress, + ...(dockerRegistryAuth.auth + ? { auth: authString } + : { + username: dockerRegistryAuth.username, + password: dockerRegistryAuth.password + }) } + CORE_LOGGER.debug(`Using docker registry auth for ${registry}`) } const pullStream = await this.docker.pull(job.containerImage, pullOptions) diff --git a/src/test/integration/dockerRegistryAuth.test.ts b/src/test/integration/dockerRegistryAuth.test.ts index e49c062a9..f35660a3e 100644 --- a/src/test/integration/dockerRegistryAuth.test.ts +++ b/src/test/integration/dockerRegistryAuth.test.ts @@ -11,6 +11,7 @@ import { expect, assert } from 'chai' import { C2DEngineDocker } from '../../components/c2d/compute_engine_docker.js' import { C2DClusterInfo, C2DClusterType } from '../../@types/C2D/C2D.js' import { dockerRegistrysAuth } from '../../@types/OceanNode.js' +import { DockerRegistryAuthSchema } from '../../utils/config/schemas.js' describe('Docker Registry Authentication Integration Tests', () => { describe('Public registry access (no credentials)', () => { @@ -240,4 +241,158 @@ describe('Docker Registry Authentication Integration Tests', () => { } }).timeout(10000) }) + + describe('DockerRegistryAuthSchema validation', () => { + it('should validate schema with only auth string', () => { + const authData = { + auth: Buffer.from('testuser:testpass').toString('base64') + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.true + if (result.success) { + expect(result.data.auth).to.equal(authData.auth) + expect(result.data.username).to.be.undefined + expect(result.data.password).to.be.undefined + } + }) + + it('should validate schema with only username and password', () => { + const authData = { + username: 'testuser', + password: 'testpass' + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.true + if (result.success) { + expect(result.data.username).to.equal('testuser') + expect(result.data.password).to.equal('testpass') + expect(result.data.auth).to.be.undefined + } + }) + + it('should validate schema with all three fields (auth, username, password)', () => { + const authData = { + username: 'testuser', + password: 'testpass', + auth: Buffer.from('testuser:testpass').toString('base64') + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.true + if (result.success) { + expect(result.data.username).to.equal('testuser') + expect(result.data.password).to.equal('testpass') + expect(result.data.auth).to.equal(authData.auth) + } + }) + + it('should reject schema with only username (missing password)', () => { + const authData = { + username: 'testuser' + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.false + if (!result.success) { + expect(result.error.errors).to.exist + expect(result.error.errors[0].message).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + } + }) + + it('should reject schema with only password (missing username)', () => { + const authData = { + password: 'testpass' + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.false + if (!result.success) { + expect(result.error.errors).to.exist + expect(result.error.errors[0].message).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + } + }) + + it('should reject empty object', () => { + const authData = {} + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.false + if (!result.success) { + expect(result.error.errors).to.exist + expect(result.error.errors[0].message).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + } + }) + + it('should reject empty auth string with no username/password', () => { + const authData = { + auth: '' + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.false + if (!result.success) { + expect(result.error.errors).to.exist + expect(result.error.errors[0].message).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + } + }) + + it('should reject empty username with password provided', () => { + const authData = { + username: '', + password: 'testpass' + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.false + if (!result.success) { + expect(result.error.errors).to.exist + expect(result.error.errors[0].message).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + } + }) + + it('should reject empty password with username provided', () => { + const authData = { + username: 'testuser', + password: '' + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.false + if (!result.success) { + expect(result.error.errors).to.exist + expect(result.error.errors[0].message).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + } + }) + + it('should reject all empty strings', () => { + const authData = { + username: '', + password: '', + auth: '' + } + + const result = DockerRegistryAuthSchema.safeParse(authData) + expect(result.success).to.be.false + if (!result.success) { + expect(result.error.errors).to.exist + expect(result.error.errors[0].message).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + } + }) + }) }) diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index 6d3055d88..4e905871a 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -84,11 +84,28 @@ export const OceanNodeDBConfigSchema = z.object({ dbType: z.string().nullable() }) -export const DockerRegistryAuthSchema = z.object({ - username: z.string(), - password: z.string(), - auth: z.string() -}) +export const DockerRegistryAuthSchema = z + .object({ + username: z.string().optional(), + password: z.string().optional(), + auth: z.string().optional() + }) + .refine( + (data) => { + // Either 'auth' is provided, OR both 'username' and 'password' are provided + return ( + (data.auth !== undefined && data.auth !== '') || + (data.username !== undefined && + data.username !== '' && + data.password !== undefined && + data.password !== '') + ) + }, + { + message: + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + } + ) export const DockerRegistrysSchema = z.record(z.string(), DockerRegistryAuthSchema) From 760490698813ae7f9c7c7611d372a0684bd55c7e Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 11:57:09 +0200 Subject: [PATCH 3/9] pass empty dockerRegistrysAuth --- src/test/integration/imageCleanup.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/integration/imageCleanup.test.ts b/src/test/integration/imageCleanup.test.ts index c4b2f7040..ab3b5304c 100644 --- a/src/test/integration/imageCleanup.test.ts +++ b/src/test/integration/imageCleanup.test.ts @@ -188,7 +188,7 @@ describe('Docker Image Cleanup Integration Tests', () => { escrow = {} as Escrow keyManager = {} as KeyManager - dockerEngine = new C2DEngineDocker(clusterConfig, db, escrow, keyManager, null) + dockerEngine = new C2DEngineDocker(clusterConfig, db, escrow, keyManager, {}) }) it('should track image usage when image is pulled', async () => { From 1b1b2604e79a3af586f9d7c682cd98b73ae6a739 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 11:58:00 +0200 Subject: [PATCH 4/9] remove unneeded code --- src/components/c2d/compute_engines.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/components/c2d/compute_engines.ts b/src/components/c2d/compute_engines.ts index 62e367f0e..26ad035f9 100644 --- a/src/components/c2d/compute_engines.ts +++ b/src/components/c2d/compute_engines.ts @@ -1,14 +1,13 @@ import { C2DClusterType, ComputeEnvironment } from '../../@types/C2D/C2D.js' import { C2DEngine } from './compute_engine_base.js' import { C2DEngineDocker } from './compute_engine_docker.js' -import { OceanNodeConfig, dockerRegistrysAuth } from '../../@types/OceanNode.js' +import { OceanNodeConfig } from '../../@types/OceanNode.js' import { C2DDatabase } from '../database/C2DDatabase.js' import { Escrow } from '../core/utils/escrow.js' import { KeyManager } from '../KeyManager/index.js' export class C2DEngines { public engines: C2DEngine[] - private dockerRegistryAuths: dockerRegistrysAuth public constructor( config: OceanNodeConfig, db: C2DDatabase, From a4605d4ee12660efd0ef5be022366ab5c60b7435 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 12:18:42 +0200 Subject: [PATCH 5/9] add DOCKER_REGISTRY_AUTHS in ci --- .github/workflows/ci.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb03b5790..6725ac904 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,19 @@ jobs: - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} + - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD + run: | + DOCKER_REGISTRY_AUTHS=$(jq -n \ + --arg username "$DOCKERHUB_USERNAME" \ + --arg password "$DOCKERHUB_PASSWORD" \ + '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }') + echo "DOCKER_REGISTRY_AUTHS<> $GITHUB_ENV + echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: integration tests run: npm run test:integration:cover env: @@ -162,6 +175,7 @@ jobs: FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }' ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json' ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json' + DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }} - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} @@ -237,7 +251,19 @@ jobs: repository: 'oceanprotocol/ocean-node' path: 'ocean-node' ref: ${{ github.event_name == 'pull_request' && github.head_ref || 'main' }} - + - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD + run: | + DOCKER_REGISTRY_AUTHS=$(jq -n \ + --arg username "$DOCKERHUB_USERNAME" \ + --arg password "$DOCKERHUB_PASSWORD" \ + '{ "https://registry-1.docker.io": { "username": $username, "password": $password } }') + echo "DOCKER_REGISTRY_AUTHS<> $GITHUB_ENV + echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Start Ocean Node working-directory: ${{ github.workspace }}/ocean-node run: | @@ -265,6 +291,7 @@ jobs: MAX_REQ_PER_MINUTE: 320 MAX_CONNECTIONS_PER_MINUTE: 320 DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":10}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration": 60,"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"minJobDuration": 10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]' + DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }} - name: Check Ocean Node is running run: | for i in $(seq 1 90); do From 8e4b0b6b786720e7c25dc415b3e3295efb4fe976 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 12:20:54 +0200 Subject: [PATCH 6/9] use DOCKERHUB_PUSH_USERNAME for ci push --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ad32a67c2..941c28408 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} password: ${{ secrets.DOCKER_PUSH_TOKEN }} - name: Set Docker metadata @@ -111,7 +111,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} password: ${{ secrets.DOCKER_PUSH_TOKEN }} - name: Create manifest list and push working-directory: /tmp/digests From d185c565bf5ea1e6f38cfc3362e20b9aac2682e3 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 12:24:57 +0200 Subject: [PATCH 7/9] fix ci --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6725ac904..353ead21a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,9 +122,9 @@ jobs: if: ${{ env.DOCKERHUB_PASSWORD && env.DOCKERHUB_USERNAME }} run: | echo "Login to Docker Hub";echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Run Barge working-directory: ${{ github.workspace }}/barge run: | From 016a1397977180b4c652343c97bc63bf0b2ae071 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 5 Feb 2026 12:54:56 +0200 Subject: [PATCH 8/9] fix --- src/components/c2d/compute_engine_docker.ts | 8 ++++++-- src/utils/config/constants.ts | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 4bf4b3058..7f3bd8814 100644 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -495,7 +495,9 @@ export class C2DEngineDocker extends C2DEngine { `${dockerRegistryAuth.username}:${dockerRegistryAuth.password}` ).toString('base64') headers.Authorization = `Basic ${authString}` - CORE_LOGGER.debug(`Using docker registry auth for ${registry}`) + CORE_LOGGER.debug( + `Using docker registry auth for ${registry} to get manifest for image ${image}` + ) } let response = await fetch(url, { headers }) @@ -1706,7 +1708,9 @@ export class C2DEngineDocker extends C2DEngine { password: dockerRegistryAuth.password }) } - CORE_LOGGER.debug(`Using docker registry auth for ${registry}`) + CORE_LOGGER.debug( + `Using docker registry auth for ${registry} to pull image ${job.containerImage}` + ) } const pullStream = await this.docker.pull(job.containerImage, pullOptions) diff --git a/src/utils/config/constants.ts b/src/utils/config/constants.ts index 13e75535c..5d9c9cdad 100644 --- a/src/utils/config/constants.ts +++ b/src/utils/config/constants.ts @@ -31,7 +31,7 @@ export const ENV_TO_CONFIG_MAPPING = { ALLOWED_ADMINS: 'allowedAdmins', ALLOWED_ADMINS_LIST: 'allowedAdminsList', DOCKER_COMPUTE_ENVIRONMENTS: 'dockerComputeEnvironments', - DOCKER_REGISTRY_AUTHS: 'dockerRegistryAuth', + DOCKER_REGISTRY_AUTHS: 'dockerRegistrysAuth', P2P_BOOTSTRAP_NODES: 'p2pConfig.bootstrapNodes', P2P_BOOTSTRAP_TIMEOUT: 'p2pConfig.bootstrapTimeout', P2P_BOOTSTRAP_TAGNAME: 'p2pConfig.bootstrapTagName', From 4270cbe28f5e94ff407d973fbc3a0f42ee4ae17b Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Mon, 9 Feb 2026 13:14:07 +0200 Subject: [PATCH 9/9] use privateRegistry per job (#1197) * use privateRegistry per job --- docs/API.md | 29 +- docs/env.md | 167 +++++ src/@types/C2D/C2D.ts | 1 + src/@types/commands.ts | 2 + src/components/c2d/compute_engine_base.ts | 60 +- src/components/c2d/compute_engine_docker.ts | 117 +++- src/components/c2d/index.ts | 3 +- src/components/core/compute/initialize.ts | 19 +- src/components/core/compute/startCompute.ts | 37 +- src/components/httpRoutes/compute.ts | 8 +- src/test/integration/compute.test.ts | 622 ++++++++++++++++++ .../integration/dockerRegistryAuth.test.ts | 6 +- 12 files changed, 1028 insertions(+), 43 deletions(-) diff --git a/docs/API.md b/docs/API.md index f9093b62a..061a4473b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1402,20 +1402,21 @@ starts a free compute job and returns jobId if succesfull #### Parameters -| name | type | required | description | -| ----------------- | ------ | -------- | ----------------------------------------------------------------------------- | -| command | string | v | command name | -| node | string | | if not present it means current node | -| consumerAddress | string | v | consumer address | -| signature | string | v | signature (msg=String(nonce) ) | -| nonce | string | v | nonce for the request | -| datasets | object | | list of ComputeAsset to be used as inputs | -| algorithm | object | | ComputeAlgorithm definition | -| environment | string | v | compute environment to use | -| resources | object | | optional list of required resources | -| metadata | object | | optional metadata for the job, data provided by the user | -| additionalViewers | object | | optional array of addresses that are allowed to fetch the result | -| queueMaxWaitTime | number | | optional max time in seconds a job can wait in the queue before being started | +| name | type | required | description | +| --------------------------- | ------ | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| command | string | v | command name | +| node | string | | if not present it means current node | +| consumerAddress | string | v | consumer address | +| signature | string | v | signature (msg=String(nonce) ) | +| nonce | string | v | nonce for the request | +| datasets | object | | list of ComputeAsset to be used as inputs | +| algorithm | object | | ComputeAlgorithm definition | +| environment | string | v | compute environment to use | +| resources | object | | optional list of required resources | +| metadata | object | | optional metadata for the job, data provided by the user | +| additionalViewers | object | | optional array of addresses that are allowed to fetch the result | +| queueMaxWaitTime | number | | optional max time in seconds a job can wait in the queue before being started | +| encryptedDockerRegistryAuth | string | | Ecies encrypted docker auth schema for image (see [Private Docker Registries with Per-Job Authentication](../env.md#private-docker-registries-with-per-job-authentication)) | #### Request diff --git a/docs/env.md b/docs/env.md index 59e41d03b..36aba3222 100644 --- a/docs/env.md +++ b/docs/env.md @@ -261,3 +261,170 @@ The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable should be a JSON array of - For GitLab Container Registry, use a personal access token (PAT) or deploy token. - The registry URL must match exactly (including protocol) with the registry used in the Docker image reference. - If no credentials are configured for a registry, the node will attempt unauthenticated access (works for public images only). + +--- + +## Private Docker Registries with Per-Job Authentication + +In addition to node-level registry authentication via `DOCKER_REGISTRY_AUTHS`, you can provide encrypted Docker registry authentication credentials on a per-job basis. This allows different users to use different private registries or credentials for their compute jobs. + +### Overview + +The `encryptedDockerRegistryAuth` parameter allows you to securely provide Docker registry credentials that are: + +- Encrypted using ECIES (Elliptic Curve Integrated Encryption Scheme) with the node's public key +- Validated to ensure proper format (either `auth` string OR `username`+`password`) +- Used only for the specific compute job, overriding node-level configuration if provided + +### Encryption Format + +The `encryptedDockerRegistryAuth` must be: + +1. A JSON object matching the Docker registry auth schema (see below) +2. Encrypted using ECIES with the node's public key +3. Hex-encoded as a string + +**Auth Schema Format:** + +The decrypted JSON must follow this structure: + +```json +{ + "username": "myuser", + "password": "mypassword" +} +``` + +OR + +```json +{ + "auth": "base64-encoded-username:password" +} +``` + +OR (all fields present) + +```json +{ + "username": "myuser", + "password": "mypassword", + "auth": "base64-encoded-username:password" +} +``` + +**Validation Rules:** + +- Either `auth` string must be provided (non-empty), OR +- Both `username` AND `password` must be provided (both non-empty) +- Empty strings are not accepted + +### Usage Examples + +#### 1. Paid Compute Start (`POST /api/services/compute`) + +```json +{ + "command": "startCompute", + "consumerAddress": "0x...", + "signature": "...", + "nonce": "123", + "environment": "0x...", + "algorithm": { + "meta": { + "container": { + "image": "registry.example.com/myorg/myimage:latest" + } + } + }, + "datasets": [], + "payment": { ... }, + "encryptedDockerRegistryAuth": "0xdeadbeef..." // ECIES encrypted hex string +} +``` + +#### 2. Free Compute Start (`POST /api/services/freeCompute`) + +```json +{ + "command": "freeStartCompute", + "consumerAddress": "0x...", + "signature": "...", + "nonce": "123", + "environment": "0x...", + "algorithm": { + "meta": { + "container": { + "image": "ghcr.io/myorg/myimage:latest" + } + } + }, + "datasets": [], + "encryptedDockerRegistryAuth": "0xdeadbeef..." // ECIES encrypted hex string +} +``` + +#### 3. Initialize Compute + +The `initialize` command accepts `encryptedDockerRegistryAuth` as part of the command payload, as it validates the image + +```json +{ + "command": "initialize", + "datasets": [...], + "algorithm": { + "meta": { + "container": { + "image": "registry.gitlab.com/myorg/myimage:latest" + } + } + }, + "environment": "0x...", + "payment": { ... }, + "consumerAddress": "0x...", + "maxJobDuration": 3600, + "encryptedDockerRegistryAuth": "0xdeadbeef..." // ECIES encrypted hex string +} +``` + +### Encryption Process + +To create `encryptedDockerRegistryAuth`, you need to: + +1. **Prepare the auth JSON object:** + + ```json + { + "username": "myuser", + "password": "mypassword" + } + ``` + +2. **Get the node's public key** (available via the node's API or P2P interface) + +3. **Encrypt the JSON string** using ECIES with the node's public key + +4. **Hex-encode the encrypted result** + +### Behavior + +- **Priority**: If `encryptedDockerRegistryAuth` is provided, it takes precedence over node-level `DOCKER_REGISTRY_AUTHS` configuration for that specific job +- **Validation**: The encrypted auth is decrypted and validated before the job starts. Invalid formats will result in an error +- **Scope**: The credentials are used for: + - Validating the Docker image exists (during initialize) + - Pulling the Docker image (during job execution) +- **Security**: Credentials are encrypted and only decrypted by the node using its private key + +### Error Handling + +If `encryptedDockerRegistryAuth` is invalid, you'll receive an error: + +- **Decryption failure**: `Invalid encryptedDockerRegistryAuth: failed to parse JSON - [error message]` +- **Schema validation failure**: `Invalid encryptedDockerRegistryAuth: Either 'auth' must be provided, or both 'username' and 'password' must be provided` + +### Notes + +- The `encryptedDockerRegistryAuth` parameter is optional. If not provided, the node will use `DOCKER_REGISTRY_AUTHS` configuration or attempt unauthenticated access +- The registry URL in the Docker image reference must match the registry you're authenticating to +- For Docker Hub, use `registry-1.docker.io` as the registry URL +- Credentials are stored encrypted in the job record and decrypted only when needed for image operations diff --git a/src/@types/C2D/C2D.ts b/src/@types/C2D/C2D.ts index 6a6213a2e..530df3252 100644 --- a/src/@types/C2D/C2D.ts +++ b/src/@types/C2D/C2D.ts @@ -264,6 +264,7 @@ export interface DBComputeJob extends ComputeJob { metadata?: DBComputeJobMetadata additionalViewers?: string[] // addresses of additional addresses that can get results algoDuration: number // duration of the job in seconds + encryptedDockerRegistryAuth?: string } // make sure we keep them both in sync diff --git a/src/@types/commands.ts b/src/@types/commands.ts index 4ebb78db4..5642bb087 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -200,6 +200,7 @@ export interface ComputeInitializeCommand extends Command { maxJobDuration: number policyServer?: any // object to pass to policy server queueMaxWaitTime?: number // max time in seconds a job can wait in the queue before being started + encryptedDockerRegistryAuth?: string } export interface FreeComputeStartCommand extends Command { @@ -216,6 +217,7 @@ export interface FreeComputeStartCommand extends Command { metadata?: DBComputeJobMetadata additionalViewers?: string[] // addresses of additional addresses that can get results queueMaxWaitTime?: number // max time in seconds a job can wait in the queue before being started + encryptedDockerRegistryAuth?: string } export interface PaidComputeStartCommand extends FreeComputeStartCommand { payment: ComputePayment diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index ec0c21e8e..b6cb3e1e5 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -23,6 +23,9 @@ import { Escrow } from '../core/utils/escrow.js' import { KeyManager } from '../KeyManager/index.js' import { dockerRegistryAuth, dockerRegistrysAuth } from '../../@types/OceanNode.js' import { ValidateParams } from '../httpRoutes/validateCommands.js' +import { EncryptMethod } from '../../@types/fileObject.js' +import { CORE_LOGGER } from '../../utils/logging/common.js' +import { DockerRegistryAuthSchema } from '../../utils/config/schemas.js' export abstract class C2DEngine { private clusterConfig: C2DClusterInfo public db: C2DDatabase @@ -72,7 +75,11 @@ export abstract class C2DEngine { } // eslint-disable-next-line require-await - public abstract checkDockerImage(image: string, platform?: any): Promise + public abstract checkDockerImage( + image: string, + encryptedDockerRegistryAuth?: string, + platform?: any + ): Promise public abstract startComputeJob( assets: ComputeAsset[], @@ -86,7 +93,8 @@ export abstract class C2DEngine { jobId: string, metadata?: DBComputeJobMetadata, additionalViewers?: string[], - queueMaxWaitTime?: number + queueMaxWaitTime?: number, + encryptedDockerRegistryAuth?: string ): Promise public abstract stopComputeJob( @@ -539,4 +547,52 @@ export abstract class C2DEngine { } return null } + + public async checkEncryptedDockerRegistryAuth( + encryptedDockerRegistryAuth: string + ): Promise { + let decryptedDockerRegistryAuth: dockerRegistryAuth + try { + const decryptedDockerRegistryAuthBuffer = await this.keyManager.decrypt( + Uint8Array.from(Buffer.from(encryptedDockerRegistryAuth, 'hex')), + EncryptMethod.ECIES + ) + + // Convert decrypted buffer to string and parse as JSON + const decryptedDockerRegistryAuthString = + decryptedDockerRegistryAuthBuffer.toString() + + decryptedDockerRegistryAuth = JSON.parse(decryptedDockerRegistryAuthString) + } catch (error: any) { + const errorMessage = `Invalid encryptedDockerRegistryAuth: failed to parse JSON - ${error?.message || String(error)}` + CORE_LOGGER.error(errorMessage) + return { + valid: false, + reason: errorMessage, + status: 400 + } + } + + // Validate using schema - ensures either auth or username+password are provided + const validationResult = DockerRegistryAuthSchema.safeParse( + decryptedDockerRegistryAuth + ) + if (!validationResult.success) { + const errorMessageValidation = validationResult.error.errors + .map((err) => err.message) + .join('; ') + const errorMessage = `Invalid encryptedDockerRegistryAuth: ${errorMessageValidation}` + CORE_LOGGER.error(errorMessage) + return { + valid: false, + reason: errorMessage, + status: 400 + } + } + return { + valid: true, + reason: null, + status: 200 + } + } } diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 7f3bd8814..8b7444eed 100644 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -51,7 +51,8 @@ import { decryptFilesObject, omitDBComputeFieldsFromComputeJob } from './index.j import { ValidateParams } from '../httpRoutes/validateCommands.js' import { Service } from '@oceanprotocol/ddo-js' import { getOceanTokenAddressForChain } from '../../utils/address.js' -import { dockerRegistrysAuth } from '../../@types/OceanNode.js' +import { dockerRegistrysAuth, dockerRegistryAuth } from '../../@types/OceanNode.js' +import { EncryptMethod } from '../../@types/fileObject.js' export class C2DEngineDocker extends C2DEngine { private envs: ComputeEnvironment[] = [] @@ -474,12 +475,24 @@ export class C2DEngineDocker extends C2DEngine { return { registry, name, ref } } - public async getDockerManifest(image: string): Promise { + public async getDockerManifest( + image: string, + encryptedDockerRegistryAuth?: string + ): Promise { const { registry, name, ref } = this.parseImage(image) const url = `${registry}/v2/${name}/manifests/${ref}` - // Get registry auth from parent class - const dockerRegistryAuth = this.getDockerRegistryAuth(registry) + // Use user provided registry auth or get it from the config + let dockerRegistryAuth: dockerRegistryAuth | null = null + if (encryptedDockerRegistryAuth) { + const decryptedDockerRegistryAuth = await this.keyManager.decrypt( + Uint8Array.from(Buffer.from(encryptedDockerRegistryAuth, 'hex')), + EncryptMethod.ECIES + ) + dockerRegistryAuth = JSON.parse(decryptedDockerRegistryAuth.toString()) + } else { + dockerRegistryAuth = this.getDockerRegistryAuth(registry) + } let headers: Record = { Accept: @@ -543,16 +556,67 @@ export class C2DEngineDocker extends C2DEngine { } /** - * Checks the docker image by looking at the manifest + * Checks the docker image by looking at local images first, then remote manifest * @param image name or tag - * @returns boolean + * @param encryptedDockerRegistryAuth optional encrypted auth for remote registry + * @param platform optional platform to validate against + * @returns ValidateParams with valid flag and platform validation result */ public async checkDockerImage( image: string, + encryptedDockerRegistryAuth?: string, platform?: RunningPlatform ): Promise { + // Step 1: Try to check local image first + if (this.docker) { + try { + const dockerImage = this.docker.getImage(image) + const imageInfo = await dockerImage.inspect() + + // Extract platform information from local image + const localPlatform = { + architecture: imageInfo.Architecture || 'amd64', + os: imageInfo.Os || 'linux' + } + + // Normalize architecture (amd64 -> x86_64 for compatibility) + if (localPlatform.architecture === 'amd64') { + localPlatform.architecture = 'x86_64' + } + + // Validate platform if required + const isValidPlatform = platform + ? checkManifestPlatform(localPlatform, platform) + : true + + if (isValidPlatform) { + CORE_LOGGER.debug(`Image ${image} found locally and platform is valid`) + return { valid: true } + } else { + CORE_LOGGER.warn( + `Image ${image} found locally but platform mismatch: ` + + `local=${localPlatform.architecture}/${localPlatform.os}, ` + + `required=${platform.architecture}/${platform.os}` + ) + return { + valid: false, + status: 400, + reason: + `Platform mismatch: image is ${localPlatform.architecture}/${localPlatform.os}, ` + + `but environment requires ${platform.architecture}/${platform.os}` + } + } + } catch (localErr: any) { + // Image not found locally or error inspecting - fall through to remote check + CORE_LOGGER.debug( + `Image ${image} not found locally (${localErr.message}), checking remote registry` + ) + } + } + + // Step 2: Fall back to remote registry check (existing behavior) try { - const manifest = await this.getDockerManifest(image) + const manifest = await this.getDockerManifest(image, encryptedDockerRegistryAuth) const platforms = Array.isArray(manifest.manifests) ? manifest.manifests.map((entry: any) => entry.platform) @@ -588,7 +652,8 @@ export class C2DEngineDocker extends C2DEngine { jobId: string, metadata?: DBComputeJobMetadata, additionalViewers?: string[], - queueMaxWaitTime?: number + queueMaxWaitTime?: number, + encryptedDockerRegistryAuth?: string ): Promise { if (!this.docker) return [] // TO DO - iterate over resources and get default runtime @@ -636,6 +701,7 @@ export class C2DEngineDocker extends C2DEngine { throw new Error(`additionalDockerFiles cannot be used with queued jobs`) } } + const job: DBComputeJob = { clusterHash: this.getC2DConfig().hash, containerImage: image, @@ -672,7 +738,8 @@ export class C2DEngineDocker extends C2DEngine { additionalViewers, terminationDetails: { exitCode: null, OOMKilled: null }, algoDuration: 0, - queueMaxWaitTime: queueMaxWaitTime || 0 + queueMaxWaitTime: queueMaxWaitTime || 0, + encryptedDockerRegistryAuth // we store the encrypted docker registry auth in the job } if (algorithm.meta.container && algorithm.meta.container.dockerfile) { @@ -683,7 +750,11 @@ export class C2DEngineDocker extends C2DEngine { } } else { // already built, we need to validate it - const validation = await this.checkDockerImage(image, env.platform) + const validation = await this.checkDockerImage( + image, + job.encryptedDockerRegistryAuth, + env.platform + ) if (!validation.valid) throw new Error( `Cannot find image ${image} for ${env.platform.architecture}. Maybe it does not exist or it's build for other arhitectures.` @@ -1682,30 +1753,40 @@ export class C2DEngineDocker extends C2DEngine { try { // Get registry auth for the image const { registry } = this.parseImage(job.containerImage) - const dockerRegistryAuth = this.getDockerRegistryAuth(registry) + // Use user provided registry auth or get it from the config + let dockerRegistryAuthForPull: any + if (originaljob.encryptedDockerRegistryAuth) { + const decryptedDockerRegistryAuth = await this.keyManager.decrypt( + Uint8Array.from(Buffer.from(originaljob.encryptedDockerRegistryAuth, 'hex')), + EncryptMethod.ECIES + ) + dockerRegistryAuthForPull = JSON.parse(decryptedDockerRegistryAuth.toString()) + } else { + dockerRegistryAuthForPull = this.getDockerRegistryAuth(registry) + } // Prepare authconfig for Dockerode if credentials are available const pullOptions: any = {} - if (dockerRegistryAuth) { + if (dockerRegistryAuthForPull) { // Extract hostname from registry URL (remove protocol) const registryUrl = new URL(registry) const serveraddress = registryUrl.hostname + (registryUrl.port ? `:${registryUrl.port}` : '') // Use auth string if available, otherwise encode username:password - const authString = dockerRegistryAuth.auth - ? dockerRegistryAuth.auth + const authString = dockerRegistryAuthForPull.auth + ? dockerRegistryAuthForPull.auth : Buffer.from( - `${dockerRegistryAuth.username}:${dockerRegistryAuth.password}` + `${dockerRegistryAuthForPull.username}:${dockerRegistryAuthForPull.password}` ).toString('base64') pullOptions.authconfig = { serveraddress, - ...(dockerRegistryAuth.auth + ...(dockerRegistryAuthForPull.auth ? { auth: authString } : { - username: dockerRegistryAuth.username, - password: dockerRegistryAuth.password + username: dockerRegistryAuthForPull.username, + password: dockerRegistryAuthForPull.password }) } CORE_LOGGER.debug( diff --git a/src/components/c2d/index.ts b/src/components/c2d/index.ts index 87f4c1300..ccee0616d 100644 --- a/src/components/c2d/index.ts +++ b/src/components/c2d/index.ts @@ -42,7 +42,8 @@ export function omitDBComputeFieldsFromComputeJob(dbCompute: DBComputeJob): Comp 'assets', 'isRunning', 'isStarted', - 'containerImage' + 'containerImage', + 'encryptedDockerRegistryAuth' ]) as ComputeJob return job } diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index 7d4d74a76..24b2a5256 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -388,8 +388,25 @@ export class ComputeInitializeHandler extends CommandHandler { if (hasDockerImages) { const algoImage = getAlgorithmImage(task.algorithm, generateUniqueID(task)) if (algoImage) { - const validation: ValidateParams = await engine.checkDockerImage( + // validate encrypteddockerRegistryAuth + let validation: ValidateParams + if (task.encryptedDockerRegistryAuth) { + validation = await engine.checkEncryptedDockerRegistryAuth( + task.encryptedDockerRegistryAuth + ) + if (!validation.valid) { + return { + stream: null, + status: { + httpStatus: validation.status, + error: `Invalid encryptedDockerRegistryAuth :${validation.reason}` + } + } + } + } + validation = await engine.checkDockerImage( algoImage, + task.encryptedDockerRegistryAuth, env.platform ) if (!validation.valid) { diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 91fb2eaad..ac83856e5 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -151,6 +151,22 @@ export class PaidComputeStartHandler extends CommandHandler { } } } + // validate encrypteddockerRegistryAuth + if (task.encryptedDockerRegistryAuth) { + const validation = await engine.checkEncryptedDockerRegistryAuth( + task.encryptedDockerRegistryAuth + ) + if (!validation.valid) { + return { + stream: null, + status: { + httpStatus: validation.status, + error: `Invalid encryptedDockerRegistryAuth :${validation.reason}` + } + } + } + } + const { algorithm } = task const config = await getConfiguration() @@ -565,7 +581,8 @@ export class PaidComputeStartHandler extends CommandHandler { jobId, task.metadata, task.additionalViewers, - task.queueMaxWaitTime + task.queueMaxWaitTime, + task.encryptedDockerRegistryAuth ) CORE_LOGGER.logMessage( 'ComputeStartCommand Response: ' + JSON.stringify(response, null, 2), @@ -679,6 +696,21 @@ export class FreeComputeStartHandler extends CommandHandler { } } } + // validate encrypteddockerRegistryAuth + if (task.encryptedDockerRegistryAuth) { + const validation = await engine.checkEncryptedDockerRegistryAuth( + task.encryptedDockerRegistryAuth + ) + if (!validation.valid) { + return { + stream: null, + status: { + httpStatus: validation.status, + error: `Invalid encryptedDockerRegistryAuth :${validation.reason}` + } + } + } + } const policyServer = new PolicyServer() for (const elem of [...[task.algorithm], ...task.datasets]) { if (!('documentId' in elem)) { @@ -911,7 +943,8 @@ export class FreeComputeStartHandler extends CommandHandler { jobId, task.metadata, task.additionalViewers, - task.queueMaxWaitTime + task.queueMaxWaitTime, + task.encryptedDockerRegistryAuth ) CORE_LOGGER.logMessage( diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index c7137b89a..b664001b1 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -83,7 +83,9 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { authorization: req.headers?.authorization, additionalViewers: (req.body.additionalViewers as unknown as string[]) || null, queueMaxWaitTime: req.body.queueMaxWaitTime || 0, - caller: req.caller + caller: req.caller, + encryptedDockerRegistryAuth: + (req.body.encryptedDockerRegistryAuth as string) || null } if (req.body.output) { startComputeTask.output = req.body.output as ComputeOutput @@ -130,7 +132,9 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/freeCompute`, async (req, res) => authorization: req.headers?.authorization, additionalViewers: (req.body.additionalViewers as unknown as string[]) || null, queueMaxWaitTime: req.body.queueMaxWaitTime || 0, - caller: req.caller + caller: req.caller, + encryptedDockerRegistryAuth: + (req.body.encryptedDockerRegistryAuth as string) || null } if (req.body.output) { startComputeTask.output = req.body.output as ComputeOutput diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index ed5c8c0eb..2dc919b7d 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -75,6 +75,8 @@ import { import { freeComputeStartPayload } from '../data/commands.js' import { DDOManager } from '@oceanprotocol/ddo-js' +import Dockerode from 'dockerode' +import { C2DEngineDocker } from '../../components/c2d/compute_engine_docker.js' const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) @@ -1393,6 +1395,626 @@ describe('Compute', () => { }) }) + describe('encryptedDockerRegistryAuth integration tests', () => { + /** + * Helper function to encrypt docker registry auth using ECIES + */ + async function encryptDockerRegistryAuth(auth: { + username?: string + password?: string + auth?: string + }): Promise { + const authJson = JSON.stringify(auth) + const authData = Uint8Array.from(Buffer.from(authJson)) + const encrypted = await oceanNode + .getKeyManager() + .encrypt(authData, EncryptMethod.ECIES) + return Buffer.from(encrypted).toString('hex') + } + + it('should initialize compute with valid encryptedDockerRegistryAuth (username/password)', async () => { + const validAuth = { + username: 'testuser', + password: 'testpass' + } + const encryptedAuth = await encryptDockerRegistryAuth(validAuth) + + const dataset: ComputeAsset = { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: String(datasetOrderTxId) + } + const algorithm: ComputeAlgorithm = { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: String(algoOrderTxId), + meta: publishedAlgoDataset.ddo.metadata.algorithm + } + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: computeJobDuration, + consumerAddress: firstEnv.consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE, + encryptedDockerRegistryAuth: encryptedAuth + } + + const resp = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(resp, 'Failed to get response') + // Should succeed (200) or fail for other reasons, but not due to auth validation + // Check that error is not a validation error (format validation), even if Docker auth fails + if (resp.status.httpStatus !== 200) { + expect(resp.status.error).to.not.include('Invalid encryptedDockerRegistryAuth') + } + if (resp.status.httpStatus === 200) { + assert(resp.stream, 'Failed to get stream') + expect(resp.stream).to.be.instanceOf(Readable) + } + }) + + it('should initialize compute with valid encryptedDockerRegistryAuth (auth string)', async () => { + const validAuth = { + auth: Buffer.from('testuser:testpass').toString('base64') + } + const encryptedAuth = await encryptDockerRegistryAuth(validAuth) + + const dataset: ComputeAsset = { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: String(datasetOrderTxId) + } + const algorithm: ComputeAlgorithm = { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: String(algoOrderTxId), + meta: publishedAlgoDataset.ddo.metadata.algorithm + } + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: computeJobDuration, + consumerAddress: firstEnv.consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE, + encryptedDockerRegistryAuth: encryptedAuth + } + + const resp = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(resp, 'Failed to get response') + // Should succeed (200) or fail for other reasons, but not due to auth validation + // Check that error is not a validation error (format validation), even if Docker auth fails + if (resp.status.httpStatus !== 200) { + expect(resp.status.error).to.not.include('Invalid encryptedDockerRegistryAuth') + } + if (resp.status.httpStatus === 200) { + assert(resp.stream, 'Failed to get stream') + expect(resp.stream).to.be.instanceOf(Readable) + } + }) + + it('should fail initialize compute with invalid encryptedDockerRegistryAuth (missing password)', async () => { + const invalidAuth = { + username: 'testuser' + // missing password + } + const encryptedAuth = await encryptDockerRegistryAuth(invalidAuth) + + const dataset: ComputeAsset = { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: String(datasetOrderTxId) + } + const algorithm: ComputeAlgorithm = { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: String(algoOrderTxId), + meta: publishedAlgoDataset.ddo.metadata.algorithm + } + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: computeJobDuration, + consumerAddress: firstEnv.consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE, + encryptedDockerRegistryAuth: encryptedAuth + } + + const resp = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(resp, 'Failed to get response') + // Should fail with 400 due to validation error + assert( + resp.status.httpStatus === 400, + `Expected 400 but got ${resp.status.httpStatus}: ${resp.status.error}` + ) + expect(resp.status.error).to.include('Invalid encryptedDockerRegistryAuth') + expect(resp.status.error).to.include( + "Either 'auth' must be provided, or both 'username' and 'password' must be provided" + ) + }) + + it('should fail initialize compute with invalid encryptedDockerRegistryAuth (empty object)', async () => { + const invalidAuth = {} + const encryptedAuth = await encryptDockerRegistryAuth(invalidAuth) + + const dataset: ComputeAsset = { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: String(datasetOrderTxId) + } + const algorithm: ComputeAlgorithm = { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: String(algoOrderTxId), + meta: publishedAlgoDataset.ddo.metadata.algorithm + } + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: computeJobDuration, + consumerAddress: firstEnv.consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE, + encryptedDockerRegistryAuth: encryptedAuth + } + + const resp = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(resp, 'Failed to get response') + assert( + resp.status.httpStatus === 400, + `Expected 400 but got ${resp.status.httpStatus}: ${resp.status.error}` + ) + expect(resp.status.error).to.include('Invalid encryptedDockerRegistryAuth') + }) + + it('should start paid compute job with valid encryptedDockerRegistryAuth', async () => { + const validAuth = { + username: 'testuser', + password: 'testpass' + } + const encryptedAuth = await encryptDockerRegistryAuth(validAuth) + + const nonce = Date.now().toString() + const message = String( + (await consumerAccount.getAddress()) + publishedComputeDataset.ddo.id + nonce + ) + const consumerMessage = ethers.solidityPackedKeccak256( + ['bytes'], + [ethers.hexlify(ethers.toUtf8Bytes(message))] + ) + const messageHashBytes = ethers.toBeArray(consumerMessage) + const signature = await wallet.signMessage(messageHashBytes) + + const startComputeTask: PaidComputeStartCommand = { + command: PROTOCOL_COMMANDS.COMPUTE_START, + consumerAddress: await consumerAccount.getAddress(), + environment: firstEnv.id, + signature, + nonce, + datasets: [ + { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: String(datasetOrderTxId) + } + ], + algorithm: { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: String(algoOrderTxId), + meta: publishedAlgoDataset.ddo.metadata.algorithm + }, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: computeJobDuration, + encryptedDockerRegistryAuth: encryptedAuth + } + + const response = await new PaidComputeStartHandler(oceanNode).handle( + startComputeTask + ) + assert(response, 'Failed to get response') + // Should succeed (200) or fail for other reasons, but not due to auth validation + // Check that error is not a validation error (format validation), even if Docker auth fails + if (response.status.httpStatus !== 200) { + expect(response.status.error).to.not.include( + 'Invalid encryptedDockerRegistryAuth' + ) + } + if (response.status.httpStatus === 200) { + assert(response.stream, 'Failed to get stream') + expect(response.stream).to.be.instanceOf(Readable) + } + }) + + it('should fail paid compute start with invalid encryptedDockerRegistryAuth', async () => { + const invalidAuth = { + username: 'testuser' + // missing password + } + const encryptedAuth = await encryptDockerRegistryAuth(invalidAuth) + + const nonce = Date.now().toString() + const message = String( + (await consumerAccount.getAddress()) + publishedComputeDataset.ddo.id + nonce + ) + const consumerMessage = ethers.solidityPackedKeccak256( + ['bytes'], + [ethers.hexlify(ethers.toUtf8Bytes(message))] + ) + const messageHashBytes = ethers.toBeArray(consumerMessage) + const signature = await wallet.signMessage(messageHashBytes) + + const startComputeTask: PaidComputeStartCommand = { + command: PROTOCOL_COMMANDS.COMPUTE_START, + consumerAddress: await consumerAccount.getAddress(), + environment: firstEnv.id, + signature, + nonce, + datasets: [ + { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: String(datasetOrderTxId) + } + ], + algorithm: { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: String(algoOrderTxId), + meta: publishedAlgoDataset.ddo.metadata.algorithm + }, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: computeJobDuration, + encryptedDockerRegistryAuth: encryptedAuth + } + + const response = await new PaidComputeStartHandler(oceanNode).handle( + startComputeTask + ) + assert(response, 'Failed to get response') + assert( + response.status.httpStatus === 400, + `Expected 400 but got ${response.status.httpStatus}: ${response.status.error}` + ) + expect(response.status.error).to.include('Invalid encryptedDockerRegistryAuth') + }) + + it('should start free compute job with valid encryptedDockerRegistryAuth', async () => { + const validAuth = { + username: 'testuser', + password: 'testpass' + } + const encryptedAuth = await encryptDockerRegistryAuth(validAuth) + + const nonce = Date.now().toString() + const consumerMessage = ethers.solidityPackedKeccak256( + ['bytes'], + [ethers.hexlify(ethers.toUtf8Bytes(nonce))] + ) + const signature = await wallet.signMessage(ethers.toBeArray(consumerMessage)) + + const startComputeTask: FreeComputeStartCommand = { + command: PROTOCOL_COMMANDS.FREE_COMPUTE_START, + consumerAddress: await wallet.getAddress(), + signature, + nonce, + environment: firstEnv.id, + datasets: [ + { + fileObject: computeAsset.services[0].files.files[0], + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: datasetOrderTxId + } + ], + algorithm: { + fileObject: algoAsset.services[0].files.files[0], + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: algoOrderTxId, + meta: publishedAlgoDataset.ddo.metadata.algorithm + }, + output: {}, + encryptedDockerRegistryAuth: encryptedAuth + } + + const response = await new FreeComputeStartHandler(oceanNode).handle( + startComputeTask + ) + assert(response, 'Failed to get response') + // Should succeed (200) or fail for other reasons, but not due to auth validation + // Check that error is not a validation error (format validation), even if Docker auth fails + if (response.status.httpStatus !== 200) { + expect(response.status.error).to.not.include( + 'Invalid encryptedDockerRegistryAuth' + ) + } + if (response.status.httpStatus === 200) { + assert(response.stream, 'Failed to get stream') + expect(response.stream).to.be.instanceOf(Readable) + } + }) + + it('should fail free compute start with invalid encryptedDockerRegistryAuth', async () => { + const invalidAuth = { + password: 'testpass' + // missing username + } + const encryptedAuth = await encryptDockerRegistryAuth(invalidAuth) + + const nonce = Date.now().toString() + const consumerMessage = ethers.solidityPackedKeccak256( + ['bytes'], + [ethers.hexlify(ethers.toUtf8Bytes(nonce))] + ) + const signature = await wallet.signMessage(ethers.toBeArray(consumerMessage)) + + const startComputeTask: FreeComputeStartCommand = { + command: PROTOCOL_COMMANDS.FREE_COMPUTE_START, + consumerAddress: await wallet.getAddress(), + signature, + nonce, + environment: firstEnv.id, + datasets: [ + { + fileObject: computeAsset.services[0].files.files[0], + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: datasetOrderTxId + } + ], + algorithm: { + fileObject: algoAsset.services[0].files.files[0], + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: algoOrderTxId, + meta: publishedAlgoDataset.ddo.metadata.algorithm + }, + output: {}, + encryptedDockerRegistryAuth: encryptedAuth + } + + const response = await new FreeComputeStartHandler(oceanNode).handle( + startComputeTask + ) + assert(response, 'Failed to get response') + assert( + response.status.httpStatus === 400, + `Expected 400 but got ${response.status.httpStatus}: ${response.status.error}` + ) + expect(response.status.error).to.include('Invalid encryptedDockerRegistryAuth') + }) + + it('should handle invalid hex-encoded encryptedDockerRegistryAuth gracefully', async () => { + const invalidHex = 'not-a-valid-hex-string' + + const dataset: ComputeAsset = { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: String(datasetOrderTxId) + } + const algorithm: ComputeAlgorithm = { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: String(algoOrderTxId), + meta: publishedAlgoDataset.ddo.metadata.algorithm + } + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: computeJobDuration, + consumerAddress: firstEnv.consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE, + encryptedDockerRegistryAuth: invalidHex + } + + const resp = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(resp, 'Failed to get response') + // Should fail with 500 due to decryption/parsing error + assert( + resp.status.httpStatus === 400, + `Expected 400 but got ${resp.status.httpStatus}: ${resp.status.error}` + ) + expect(resp.status.error).to.include('Invalid encryptedDockerRegistryAuth') + }) + }) + + describe('Local Docker image checking', () => { + let docker: Dockerode + let dockerEngine: C2DEngineDocker + const testImage = 'alpine:3.18' + before(async function () { + // Skip if Docker not available + try { + docker = new Dockerode() + await docker.info() + const pullStream = await docker.pull(testImage) + await new Promise((resolve, reject) => { + let wroteStatusBanner = false + docker.modem.followProgress( + pullStream, + (err: any, res: any) => { + // onFinished + if (err) { + console.log(err) + return reject(err) + } + console.log(`Successfully pulled image: ${testImage}`) + resolve(res) + }, + (progress: any) => { + // onProgress + if (!wroteStatusBanner) { + wroteStatusBanner = true + console.log('############# Pull docker image status: ##############') + } + // only write the status banner once, its cleaner + let logText = '' + if (progress.id) logText += progress.id + ' : ' + progress.status + else logText = progress.status + console.log('Pulling image : ' + logText) + } + ) + }) + } catch (e) { + this.skip() + } + + // Get the Docker engine from oceanNode + const c2dEngines = oceanNode.getC2DEngines() + const engines = (c2dEngines as any).engines as C2DEngineDocker[] + dockerEngine = engines.find((e) => e instanceof C2DEngineDocker) + if (!dockerEngine) { + this.skip() + } + }) + + it('should check local image when it exists locally', async function () { + // Check the image - should find it locally + const result = await dockerEngine.checkDockerImage(testImage) + + assert(result, 'Result should exist') + assert(result.valid === true, 'Image should be valid') + }).timeout(30000) + + it('should validate platform for local images', async function () { + // Get the platform from the local image + const imageInfo = await docker.getImage(testImage).inspect() + const localArch = imageInfo.Architecture || 'amd64' + const localOs = imageInfo.Os || 'linux' + + // Check with matching platform + const matchingPlatform = { + architecture: localArch === 'amd64' ? 'x86_64' : localArch, + os: localOs + } + const resultMatching = await dockerEngine.checkDockerImage( + testImage, + undefined, + matchingPlatform + ) + + assert(resultMatching, 'Result should exist') + assert( + resultMatching.valid === true, + 'Image should be valid with matching platform' + ) + }).timeout(30000) + + it('should detect platform mismatch for local images', async function () { + // Check with mismatched platform (assuming local is linux/amd64 or linux/x86_64) + const mismatchedPlatform = { + architecture: 'arm64', // Different architecture + os: 'linux' + } + const resultMismatch = await dockerEngine.checkDockerImage( + testImage, + undefined, + mismatchedPlatform + ) + + assert(resultMismatch, 'Result should exist') + assert( + resultMismatch.valid === false, + 'Image should be invalid with mismatched platform' + ) + assert(resultMismatch.status === 400, 'Status should be 400 for platform mismatch') + assert( + resultMismatch.reason.includes('Platform mismatch'), + 'Reason should include platform mismatch message' + ) + }).timeout(30000) + + it('should fall back to remote registry when local image not found', async function () { + const nonExistentLocalImage = 'nonexistent-local-image:latest' + + // Ensure image doesn't exist locally + try { + const image = docker.getImage(nonExistentLocalImage) + await image.inspect() + // If we get here, image exists - remove it for test + await image.remove({ force: true }) + } catch (e) { + // Image doesn't exist locally, which is what we want + } + + // Check the image - should fall back to remote check + // This will likely fail with 404, but we're testing the fallback behavior + const result = await dockerEngine.checkDockerImage(nonExistentLocalImage) + + assert(result, 'Result should exist') + // Should have attempted remote check (will fail, but that's expected) + assert(result.valid === false, 'Image should be invalid (not found)') + assert(result.status === 404, 'Status should be 404 for not found') + }).timeout(30000) + + it('should work without platform validation when platform not specified', async function () { + // Check without platform - should succeed if image exists + const result = await dockerEngine.checkDockerImage(testImage) + + assert(result, 'Result should exist') + assert(result.valid === true, 'Image should be valid without platform check') + }).timeout(30000) + + after(async function () { + // Clean up test images if needed + try { + await docker.info() + } catch (e) { + // Docker not available, skip cleanup + } + + // Optionally remove test images to save space + // (commented out to avoid breaking other tests that might use these images) + /* + try { + const image = docker.getImage('alpine:3.18') + await image.remove({ force: true }) + } catch (e) { + // Ignore errors during cleanup + } + */ + }) + }) + after(async () => { await tearDownEnvironment(previousConfiguration) indexer.stopAllChainIndexers() diff --git a/src/test/integration/dockerRegistryAuth.test.ts b/src/test/integration/dockerRegistryAuth.test.ts index f35660a3e..394bbd1df 100644 --- a/src/test/integration/dockerRegistryAuth.test.ts +++ b/src/test/integration/dockerRegistryAuth.test.ts @@ -37,7 +37,7 @@ describe('Docker Registry Authentication Integration Tests', () => { // Test with a well-known public image const image = 'library/alpine:latest' - const manifest = await dockerEngine.getDockerManifest(image) + const manifest = await dockerEngine.getDockerManifest(image, null) expect(manifest).to.exist expect(manifest).to.have.property('schemaVersion') @@ -64,7 +64,7 @@ describe('Docker Registry Authentication Integration Tests', () => { // Use a simple image reference that will default to Docker Hub const image = 'hello-world:latest' - const manifest = await dockerEngine.getDockerManifest(image) + const manifest = await dockerEngine.getDockerManifest(image, null) expect(manifest).to.exist expect(manifest).to.have.property('schemaVersion') @@ -233,7 +233,7 @@ describe('Docker Registry Authentication Integration Tests', () => { ) try { - await dockerEngine.getDockerManifest('invalid-image-reference') + await dockerEngine.getDockerManifest('invalid-image-reference', null) assert.fail('Should have thrown an error for invalid image') } catch (error: any) { expect(error).to.exist