[Job Launcher] Create manifest module (#3306)

* feat: refactor job module to include manifest module and update related imports

* Add bsc testnet hmt token address

* refactor: remove unused oracle address configurations from environment files

* feat: update job creation process to include JobCaptchaDto and refactor manifest creation logic

* Refactor Manifest Module and Service

- Removed unused configuration services from ManifestModule.
- Updated ManifestService to use new job types (CvatJobType, AudinoJobType, FortuneJobType, HCaptchaJobType).
- Implemented new methods for CVAT elements count and image consistency checks.
- Refactored createManifest method to streamline manifest creation for different job types.
- Enhanced unit tests for ManifestService to reflect changes in job types and added mocks for storage service.
- Updated routing protocol and webhook service tests to use new job types.
- Cleaned up environment configuration files by removing deprecated oracle addresses.

* Merge branch 'develop' into feat/job-launcher/manifest-module

* feat: enhance DTO validation and update manifest service methods

Author: flopez7

packages/apps/job-launcher/server/.env.example

@@ -20,16 +20,9 @@ GAS_PRICE_MULTIPLIER=1
 PGP_PRIVATE_KEY=
 PGP_ENCRYPT=
 PGP_PASSPHRASE=
-FORTUNE_EXCHANGE_ORACLE_ADDRESS=
-FORTUNE_RECORDING_ORACLE_ADDRESS=
-CVAT_EXCHANGE_ORACLE_ADDRESS=
-CVAT_RECORDING_ORACLE_ADDRESS=
-AUDINO_EXCHANGE_ORACLE_ADDRESS=
-AUDINO_RECORDING_ORACLE_ADDRESS=
 REPUTATION_ORACLE_ADDRESS=
 HCAPTCHA_RECORDING_ORACLE_URI=
 HCAPTCHA_REPUTATION_ORACLE_URI=
-HCAPTCHA_ORACLE_ADDRESS=
 HCAPTCHA_SITE_KEY=
 RPC_URL_SEPOLIA=
 RPC_URL_POLYGON=

packages/apps/job-launcher/server/ENV.md

@@ -180,24 +180,9 @@ REPUTATION_ORACLE_ADDRESS=
 ### List of reputation oracle addresses, typically comma-separated. Required
 REPUTATION_ORACLES=
 
-### Address of the Fortune exchange oracle contract. Required
-FORTUNE_EXCHANGE_ORACLE_ADDRESS=
-
-### Address of the Fortune recording oracle contract. Required
-FORTUNE_RECORDING_ORACLE_ADDRESS=
-
-### Address of the CVAT exchange oracle contract. Required
-CVAT_EXCHANGE_ORACLE_ADDRESS=
-
-### Address of the CVAT recording oracle contract. Required
-CVAT_RECORDING_ORACLE_ADDRESS=
-
 ### URI for the hCaptcha recording oracle service. Required
 HCAPTCHA_RECORDING_ORACLE_URI=
 
 ### URI for the hCaptcha reputation oracle service. Required
 HCAPTCHA_REPUTATION_ORACLE_URI=
 
-### Address of the hCaptcha oracle contract. Required
-HCAPTCHA_ORACLE_ADDRESS=
-

packages/apps/job-launcher/server/jest.config.ts

@@ -1,13 +1,15 @@
+import { createDefaultPreset } from 'ts-jest';
+
+const jestTsPreset = createDefaultPreset({});
+
 module.exports = {
+  ...jestTsPreset,
   coverageDirectory: '../coverage',
   collectCoverageFrom: ['**/*.(t|j)s'],
   moduleFileExtensions: ['js', 'json', 'ts'],
   rootDir: 'src',
   testEnvironment: 'node',
   testRegex: '.*\\.spec\\.ts$',
-  transform: {
-    '^.+\\.(t|j)s$': 'ts-jest',
-  },
   moduleNameMapper: {
     '^uuid$': require.resolve('uuid'),
     '^typeorm$': require.resolve('typeorm'),

packages/apps/job-launcher/server/src/common/config/env-schema.ts

@@ -30,15 +30,8 @@ export const envValidator = Joi.object({
   GAS_PRICE_MULTIPLIER: Joi.number(),
   REPUTATION_ORACLE_ADDRESS: Joi.string().required(),
   REPUTATION_ORACLES: Joi.string().required(),
-  FORTUNE_EXCHANGE_ORACLE_ADDRESS: Joi.string().required(),
-  FORTUNE_RECORDING_ORACLE_ADDRESS: Joi.string().required(),
-  CVAT_EXCHANGE_ORACLE_ADDRESS: Joi.string().required(),
-  CVAT_RECORDING_ORACLE_ADDRESS: Joi.string().required(),
-  AUDINO_EXCHANGE_ORACLE_ADDRESS: Joi.string(),
-  AUDINO_RECORDING_ORACLE_ADDRESS: Joi.string(),
   HCAPTCHA_RECORDING_ORACLE_URI: Joi.string().required(),
   HCAPTCHA_REPUTATION_ORACLE_URI: Joi.string().required(),
-  HCAPTCHA_ORACLE_ADDRESS: Joi.string().required(),
   HCAPTCHA_SITE_KEY: Joi.string().required(),
   HCAPTCHA_SECRET: Joi.string().required(),
   HCAPTCHA_PROTECTION_URL: Joi.string().description(

packages/apps/job-launcher/server/src/common/config/web3-config.service.ts

@@ -45,46 +45,6 @@ export class Web3ConfigService {
     return this.configService.getOrThrow<string>('REPUTATION_ORACLES');
   }
 
-  /**
-   * Address of the Fortune exchange oracle contract.
-   * Required
-   */
-  get fortuneExchangeOracleAddress(): string {
-    return this.configService.getOrThrow<string>(
-      'FORTUNE_EXCHANGE_ORACLE_ADDRESS',
-    );
-  }
-
-  /**
-   * Address of the Fortune recording oracle contract.
-   * Required
-   */
-  get fortuneRecordingOracleAddress(): string {
-    return this.configService.getOrThrow<string>(
-      'FORTUNE_RECORDING_ORACLE_ADDRESS',
-    );
-  }
-
-  /**
-   * Address of the CVAT exchange oracle contract.
-   * Required
-   */
-  get cvatExchangeOracleAddress(): string {
-    return this.configService.getOrThrow<string>(
-      'CVAT_EXCHANGE_ORACLE_ADDRESS',
-    );
-  }
-
-  /**
-   * Address of the CVAT recording oracle contract.
-   * Required
-   */
-  get cvatRecordingOracleAddress(): string {
-    return this.configService.getOrThrow<string>(
-      'CVAT_RECORDING_ORACLE_ADDRESS',
-    );
-  }
-
   /**
    * URI for the hCaptcha recording oracle service.
    * Required
@@ -104,30 +64,4 @@ export class Web3ConfigService {
       'HCAPTCHA_REPUTATION_ORACLE_URI',
     );
   }
-
-  /**
-   * Address of the hCaptcha oracle contract.
-   * Required
-   */
-  get hCaptchaOracleAddress(): string {
-    return this.configService.getOrThrow<string>('HCAPTCHA_ORACLE_ADDRESS');
-  }
-
-  /**
-   * Address of the Audino exchange oracle.
-   */
-  get audinoExchangeOracleAddress(): string {
-    return this.configService.getOrThrow<string>(
-      'AUDINO_EXCHANGE_ORACLE_ADDRESS',
-    );
-  }
-
-  /**
-   * Address of the Audino recording oracle.
-   */
-  get audinoRecordingOracleAddress(): string {
-    return this.configService.getOrThrow<string>(
-      'AUDINO_RECORDING_ORACLE_ADDRESS',
-    );
-  }
 }

packages/apps/job-launcher/server/src/common/constants/index.ts

@@ -1,5 +1,5 @@
 import { ChainId } from '@human-protocol/sdk';
-import { JobRequestType, JobStatus } from '../enums/job';
+import { CvatJobType, JobStatus } from '../enums/job';
 
 export const SERVICE_NAME = 'Job Launcher';
 export const NS = 'hmt';
@@ -26,11 +26,11 @@ export const SENDGRID_API_KEY_DISABLED = 'sendgrid-disabled';
 export const HEADER_SIGNATURE_KEY = 'human-signature';
 
 export const CVAT_JOB_TYPES = [
-  JobRequestType.IMAGE_POLYGONS,
-  JobRequestType.IMAGE_BOXES,
-  JobRequestType.IMAGE_POINTS,
-  JobRequestType.IMAGE_BOXES_FROM_POINTS,
-  JobRequestType.IMAGE_SKELETONS_FROM_BOXES,
+  CvatJobType.IMAGE_POLYGONS,
+  CvatJobType.IMAGE_BOXES,
+  CvatJobType.IMAGE_POINTS,
+  CvatJobType.IMAGE_BOXES_FROM_POINTS,
+  CvatJobType.IMAGE_SKELETONS_FROM_BOXES,
 ];
 
 export const CANCEL_JOB_STATUSES = [

packages/apps/job-launcher/server/src/common/constants/tokens.ts

@@ -59,4 +59,10 @@ export const TOKEN_ADDRESSES: {
     //   decimals: 6,
     // },
   },
+  [ChainId.BSC_TESTNET]: {
+    [EscrowFundToken.HMT]: {
+      address: NETWORKS[ChainId.BSC_TESTNET]!.hmtAddress,
+      decimals: 18,
+    },
+  },
 };

packages/apps/job-launcher/server/src/common/enums/job.ts

@@ -27,17 +27,39 @@ export enum JobSortField {
   CREATED_AT = 'created_at',
 }
 
-export enum JobRequestType {
+export enum FortuneJobType {
+  FORTUNE = 'fortune',
+}
+
+export enum HCaptchaJobType {
+  HCAPTCHA = 'hcaptcha',
+}
+
+export enum CvatJobType {
   IMAGE_POINTS = 'image_points',
   IMAGE_POLYGONS = 'image_polygons',
   IMAGE_BOXES = 'image_boxes',
   IMAGE_BOXES_FROM_POINTS = 'image_boxes_from_points',
   IMAGE_SKELETONS_FROM_BOXES = 'image_skeletons_from_boxes',
-  HCAPTCHA = 'hcaptcha',
-  FORTUNE = 'fortune',
+}
+
+export enum AudinoJobType {
   AUDIO_TRANSCRIPTION = 'audio_transcription',
 }
 
+export const JobType = [
+  ...Object.values(CvatJobType),
+  ...Object.values(FortuneJobType),
+  ...Object.values(HCaptchaJobType),
+  ...Object.values(AudinoJobType),
+];
+
+export type JobRequestType =
+  | CvatJobType
+  | FortuneJobType
+  | AudinoJobType
+  | HCaptchaJobType;
+
 export enum JobCaptchaMode {
   BATCH = 'batch',
 }

packages/apps/job-launcher/server/src/common/utils/slack.spec.ts

@@ -1,8 +1,7 @@
 import axios from 'axios';
 import { sendSlackNotification } from './slack';
 
-export const MOCK_SLACK_ABUSE_NOTIFICATION_WEBHOOK_URL =
-  'https://slack.com/webhook';
+const MOCK_SLACK_ABUSE_NOTIFICATION_WEBHOOK_URL = 'https://slack.com/webhook';
 
 jest.mock('axios');
 describe('sendSlackNotification', () => {

packages/apps/job-launcher/server/src/common/utils/storage.spec.ts

@@ -1,5 +1,5 @@
 import { AWSRegions, StorageProviders } from '../enums/storage';
-import { JobRequestType } from '../enums/job';
+import { CvatJobType } from '../enums/job';
 import axios from 'axios';
 import { StorageDataDto } from '../../modules/job/job.dto';
 import { generateBucketUrl, listObjectsInBucket } from './storage';
@@ -15,7 +15,7 @@ describe('Storage utils', () => {
         bucketName: 'my-bucket',
         path: 'my-folder',
       };
-      const url = generateBucketUrl(storageData, JobRequestType.IMAGE_POINTS);
+      const url = generateBucketUrl(storageData, CvatJobType.IMAGE_POINTS);
       const objects = ['object1', 'object2'];
       const response = {
         status: 200,
@@ -44,7 +44,7 @@ describe('Storage utils', () => {
         bucketName: 'my-bucket',
         path: 'my-folder',
       };
-      const url = generateBucketUrl(storageData, JobRequestType.IMAGE_POINTS);
+      const url = generateBucketUrl(storageData, CvatJobType.IMAGE_POINTS);
       const objects = Array.from({ length: 4 }, (_, i) => `object${i + 1}`);
       const response1 = {
         status: 200,
@@ -90,7 +90,7 @@ describe('Storage utils', () => {
         bucketName: 'my-bucket',
         path: 'my-folder',
       };
-      const url = generateBucketUrl(storageData, JobRequestType.IMAGE_POINTS);
+      const url = generateBucketUrl(storageData, CvatJobType.IMAGE_POINTS);
       const response = {
         status: 200,
         data: `
@@ -112,7 +112,7 @@ describe('Storage utils', () => {
         bucketName: 'non-existent-bucket',
         path: 'my-folder',
       };
-      const url = generateBucketUrl(storageData, JobRequestType.IMAGE_POINTS);
+      const url = generateBucketUrl(storageData, CvatJobType.IMAGE_POINTS);
       const response = {
         status: 404,
         data: 'Bucket not found',
@@ -131,7 +131,7 @@ describe('Storage utils', () => {
         bucketName: 'private-bucket',
         path: 'my-folder',
       };
-      const url = generateBucketUrl(storageData, JobRequestType.IMAGE_POINTS);
+      const url = generateBucketUrl(storageData, CvatJobType.IMAGE_POINTS);
       const response = {
         status: 403,
         data: 'Access denied',