feat: added validation against the extractable service (MAPCO-9560, MAPCO-9561)

bundledApi.yaml

@@ -166,6 +166,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/error'
+        '409':
+          description: Conflict
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error'
         '500':
           description: Unexpected Error
           content:
@@ -198,6 +204,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/error'
+        '409':
+          description: Conflict
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error'
         '500':
           description: Unexpected Error
           content:
@@ -242,9 +254,7 @@ components:
       additionalProperties: false
     payloadForValidation:
       type: object
-      description: >-
-        3d model payload, if metadta is not provided will validate only the
-        sources
+      description: 3d model payload, if metadta is not provided will validate only the sources
       required:
         - modelPath
         - tilesetFilename
@@ -637,8 +647,7 @@ components:
           allOf:
             - $ref: '#/components/schemas/Geometry'
             - description: Geographic demarcation of the product
-            - example: >-
-                {"type":"Polygon","coordinates":[[[1,2],[3,4],[5,6],[7,8],[1,2]]]}
+            - example: '{"type":"Polygon","coordinates":[[[1,2],[3,4],[5,6],[7,8],[1,2]]]}'
         heightRangeFrom:
           type: number
           format: double
@@ -731,8 +740,7 @@ components:
           allOf:
             - $ref: '#/components/schemas/Geometry'
             - description: Geographic demarcation of the product
-            - example: >-
-                {"type":"Polygon","coordinates":[[[1,2],[3,4],[5,6],[7,8],[1,2]]]}
+            - example: '{"type":"Polygon","coordinates":[[[1,2],[3,4],[5,6],[7,8],[1,2]]]}'
         minResolutionMeter:
           type: number
           format: double
@@ -964,8 +972,7 @@ components:
           allOf:
             - $ref: '#/components/schemas/Geometry'
             - description: Geographic demarcation of the product
-            - example: >-
-                {"type":"Polygon","coordinates":[[[1,2],[3,4],[5,6],[7,8],[1,2]]]}
+            - example: '{"type":"Polygon","coordinates":[[[1,2],[3,4],[5,6],[7,8],[1,2]]]}'
         heightRangeFrom:
           type: number
           format: double

config/custom-environment-variables.json

@@ -45,6 +45,7 @@
   "externalServices": {
     "storeTrigger": "STORE_TRIGGER_URL",
     "catalog": "CATALOG_URL",
+    "extractable": "EXTRACTABLE_URL",
     "lookupTables": {
       "url": "LOOKUP_TABLES_URL",
       "subUrl": "LOOKUP_TABLES_SUB_URL"
@@ -76,5 +77,9 @@
       "__name": "S3_DEST_MAX_ATTEMPTS",
       "__format": "number"
     }
+  },
+  "isExtractableLogicEnabled": {
+    "__name": "ENABLE_EXTRACTABLE_MANAGEMENT",
+    "__format": "boolean"
   }
 }

config/default.json

@@ -36,6 +36,7 @@
   "externalServices": {
     "storeTrigger": "http://127.0.0.1:8080",
     "catalog": "http://127.0.0.1:8080",
+    "extractable": "http://127.0.0.1:8080",
     "lookupTables": {
       "url": "http://127.0.0.1:8080",
       "subUrl": "lookup-tables/lookupData"
@@ -58,5 +59,6 @@
     "forcePathStyle": true,
     "sslEnabled": false,
     "maxAttempts": 3
-  }
+  },
+  "isExtractableLogicEnabled": true
 }

helm/templates/configmap.yaml

@@ -29,6 +29,8 @@ data:
   SERVER_PORT: {{ .Values.env.targetPort | quote }}
   STORE_TRIGGER_URL: {{ .Values.env.storeTrigger.url | default (printf "http://%s-store-trigger" .Release.Name) }}
   CATALOG_URL: {{ .Values.env.catalog.url | default (printf "http://%s-catalog" .Release.Name) }}
+  EXTRACTABLE_URL: {{ .Values.env.extractable.url | default (printf "http://%s-extractable" .Release.Name) }}
+  ENABLE_EXTRACTABLE_MANAGEMENT: {{ .Values.global.isExtractableLogicEnabled | quote }}
   LOOKUP_TABLES_URL: {{ .Values.validations.lookupTables.url | quote }}
   LOOKUP_TABLES_SUB_URL: {{ .Values.validations.lookupTables.subUrl | quote }}
   {{ if eq $providers.source "NFS" }}

helm/values.yaml

@@ -28,7 +28,9 @@ global:
     name: ''
     pv_path: ''
     sub_path: ''
-
+
+  isExtractableLogicEnabled:
+
 cloudProvider:
   dockerRegistryUrl:
   flavor:
@@ -127,6 +129,8 @@ env:
     url:
   catalog:
     url:
+  extractable:
+    url:
 
 resources:
   enabled: true

openapi3.yaml

@@ -167,6 +167,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/error'
+        '409':
+          description: Conflict
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error'
         '500':
           description: Unexpected Error
           content:
@@ -200,6 +206,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/error'
+        '409':
+          description: Conflict
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error'
         '500':
           description: Unexpected Error
           content:

src/externalServices/catalog/catalogCall.ts

@@ -67,12 +67,20 @@ export class CatalogCall {
         });
         return response.data;
       } else {
-        this.logger.error({
-          msg: `Something went wrong in catalog when tring to find records, service returned ${response.status}`,
-          logContext,
-          response,
-        });
-        throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with the catalog during Finding Records', true);
+        /* istanbul ignore next */
+        {
+          this.logger.error({
+            msg: `Something went wrong in catalog when tring to find records, service returned ${response.status}`,
+            logContext,
+            response,
+          });
+          throw new AppError(
+            'catalog',
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            'Problem with the catalog during Finding Records',
+            true
+          );
+        }
       }
     } catch (err) {
       this.logger.error({

src/externalServices/extractable-management/extractableCall.ts

@@ -0,0 +1,61 @@
+import axios from 'axios';
+import { inject, injectable } from 'tsyringe';
+import { Logger } from '@map-colonies/js-logger';
+import { StatusCodes } from 'http-status-codes';
+import { Tracer } from '@opentelemetry/api';
+import { withSpanAsyncV4 } from '@map-colonies/telemetry';
+import { SERVICES } from '../../common/constants';
+import { AppError } from '../../common/appError';
+import { IConfig, LogContext } from '../../common/interfaces';
+
+@injectable()
+export class ExtractableCall {
+  private readonly logContext: LogContext;
+  private readonly extractable: string;
+
+  public constructor(
+    @inject(SERVICES.CONFIG) private readonly config: IConfig,
+    @inject(SERVICES.LOGGER) private readonly logger: Logger,
+    @inject(SERVICES.TRACER) public readonly tracer: Tracer
+  ) {
+    this.extractable = this.config.get<string>('externalServices.extractable');
+    this.logContext = {
+      fileName: __filename,
+      class: ExtractableCall.name,
+    };
+  }
+
+  @withSpanAsyncV4
+  public async isExtractableRecordExists(recordName: string): Promise<boolean> {
+    const logContext = { ...this.logContext, function: this.isExtractableRecordExists.name };
+    this.logger.debug({ msg: `Checking record '${recordName}' in extractable service`, logContext });
+
+    try {
+      const response = await axios.get(`${this.extractable}/records/${recordName}`, {
+        validateStatus: () => true,
+      });
+
+      if (response.status === StatusCodes.OK) {
+        this.logger.debug({ msg: `Record '${recordName}' exists in extractable`, logContext });
+        return true;
+      }
+
+      if (response.status === StatusCodes.NOT_FOUND) {
+        this.logger.debug({ msg: `Record '${recordName}' does not exist in extractable`, logContext });
+        return false;
+      }
+
+      this.logger.error({ msg: `Unexpected status from extractable: ${response.status}`, logContext, status: response.status });
+
+      throw new AppError('extractable', StatusCodes.INTERNAL_SERVER_ERROR, 'Unexpected response from extractable service', true);
+    } catch (error) {
+      this.logger.error({ msg: 'Error occurred during isExtractableRecordExists call', recordName, logContext, err: error });
+
+      if (error instanceof AppError) {
+        throw error;
+      }
+
+      throw new AppError('extractable', StatusCodes.INTERNAL_SERVER_ERROR, 'Failed to query extractable service', true);
+    }
+  }
+}

src/metadata/models/metadataManager.ts

@@ -51,10 +51,18 @@ export class MetadataManager {
 
     try {
       const refReason: FailedReason = { outFailedReason: '' };
-      const isValid: boolean = await this.validator.validateUpdate(identifier, payload, refReason);
+      const isValid = await this.validator.validateUpdate(identifier, payload, refReason);
+
       if (!isValid) {
         throw new AppError('badRequest', StatusCodes.BAD_REQUEST, refReason.outFailedReason, true);
       }
+
+      const record = (await this.catalog.getRecord(identifier)) as Record3D;
+      const doesNotExistInExtractable = await this.validator.isRecordAbsentFromExtractable(record, refReason);
+      if (!doesNotExistInExtractable) {
+        throw new AppError('conflict', StatusCodes.CONFLICT, refReason.outFailedReason, true);
+      }
+
       this.logger.info({
         msg: 'model validated successfully',
         logContext,
@@ -112,6 +120,13 @@ export class MetadataManager {
       } else if (record3D.productStatus == RecordStatus.BEING_DELETED) {
         throw new AppError('badRequest', StatusCodes.BAD_REQUEST, `Can't change status of record that is being deleted`, true);
       }
+
+      const refReason: FailedReason = { outFailedReason: '' };
+      const doesNotExistInExtractable = await this.validator.isRecordAbsentFromExtractable(record3D, refReason);
+      if (!doesNotExistInExtractable) {
+        throw new AppError('conflict', StatusCodes.CONFLICT, refReason.outFailedReason, true);
+      }
+
       this.logger.info({
         msg: 'model validated successfully',
         logContext,

src/validator/validationManager.ts

@@ -16,6 +16,8 @@ import { CatalogCall } from '../externalServices/catalog/catalogCall';
 import { convertSphereFromXYZToWGS84, convertRegionFromRadianToDegrees } from './calculatePolygonFromTileset';
 import { BoundingRegion, BoundingSphere, TileSetJson } from './interfaces';
 import { extractLink } from './extractPathFromLink';
+import { ExtractableCall } from '../externalServices/extractable-management/extractableCall';
+import { Record3D } from '../externalServices/catalog/interfaces';
 
 export const ERROR_METADATA_DATE = 'sourceStartDate should not be later than sourceEndDate';
 export const ERROR_METADATA_RESOLUTION = 'minResolutionMeter should not be bigger than maxResolutionMeter';
@@ -25,6 +27,7 @@ export const ERROR_METADATA_BOX_TILESET = `BoundingVolume of box is not supporte
 export const ERROR_METADATA_BAD_FORMAT_TILESET = 'Bad tileset format. Should be in 3DTiles format';
 export const ERROR_METADATA_ERRORED_TILESET = `File tileset validation failed`;
 export const ERROR_METADATA_FOOTPRINT_FAR_FROM_MODEL = `Wrong footprint! footprint's coordinates is not even close to the model!`;
+export const ERROR_METADATA_PRODUCT_NAME_CONFLICT = `An external service locks this record`;
 
 export interface FailedReason {
   outFailedReason: string;
@@ -34,20 +37,24 @@ export interface FailedReason {
 export class ValidationManager {
   private readonly limit: number;
   private readonly logContext: LogContext;
+  private readonly isExtractableManagementEnabled: boolean;
 
   public constructor(
     @inject(SERVICES.CONFIG) private readonly config: IConfig,
     @inject(SERVICES.LOGGER) private readonly logger: Logger,
     @inject(SERVICES.TRACER) public readonly tracer: Tracer,
     @inject(LookupTablesCall) private readonly lookupTables: LookupTablesCall,
     @inject(CatalogCall) private readonly catalog: CatalogCall,
+    @inject(ExtractableCall) private readonly extractable: ExtractableCall,
     @inject(SERVICES.PROVIDER) private readonly provider: Provider
   ) {
     this.logContext = {
       fileName: __filename,
       class: ValidationManager.name,
     };
     this.limit = this.config.get<number>('validation.percentageLimit');
+
+    this.isExtractableManagementEnabled = this.config.get<boolean>('isExtractableLogicEnabled');
   }
 
   @withSpanAsyncV4
@@ -344,6 +351,26 @@ export class ValidationManager {
     return { isValid: true };
   }
 
+  public async isRecordAbsentFromExtractable(record: Record3D, refReason: FailedReason): Promise<boolean> {
+    const logContext = { ...this.logContext, function: this.isRecordAbsentFromExtractable.name };
+
+    if (!this.isExtractableManagementEnabled) {
+      this.logger.debug({
+        msg: 'Extractable validation skipped - service disabled',
+        logContext,
+      });
+      return true;
+    }
+
+    const existsInExtractable = await this.extractable.isExtractableRecordExists(record.productName!);
+    if (existsInExtractable) {
+      refReason.outFailedReason = ERROR_METADATA_PRODUCT_NAME_CONFLICT;
+      return false;
+    }
+
+    return true;
+  }
+
   private validateCoordinates(footprint: Polygon): boolean {
     const length = footprint.coordinates[0].length;
     const first = footprint.coordinates[0][0];
@@ -429,18 +456,21 @@ export class ValidationManager {
         isValid: true,
       };
     } catch (err) {
-      const msg = `An error caused during the validation of the intersection`;
-      this.logger.error({
-        msg,
-        logContext,
-        err,
-        modelPolygon,
-        footprint,
-      });
-      return {
-        isValid: false,
-        message: msg,
-      };
+      /* istanbul ignore next */
+      {
+        const msg = `An error caused during the validation of the intersection`;
+        this.logger.error({
+          msg,
+          logContext,
+          err,
+          modelPolygon,
+          footprint,
+        });
+        return {
+          isValid: false,
+          message: msg,
+        };
+      }
     }
   }
 

tests/helpers/mockCreator.ts

@@ -25,6 +25,8 @@ export const validationManagerMock = {
   validateUpdate: jest.fn(),
   getTilesetModelPolygon: jest.fn(),
   isPolygonValid: jest.fn(),
+  validateUpdateStatus: jest.fn(),
+  isRecordAbsentFromExtractable: jest.fn(),
 };
 
 export const configMock = {
@@ -52,3 +54,7 @@ export const catalogMock = {
   patchMetadata: jest.fn(),
   changeStatus: jest.fn(),
 };
+
+export const extractableMock = {
+  isExtractableRecordExists: jest.fn(),
+};