Bulk import functionality for telemetry devices

api/src/paths/project/{projectId}/survey/{surveyId}/devices/import.test.ts

@@ -0,0 +1,89 @@
+import { expect } from 'chai';
+import sinon from 'sinon';
+import * as db from '../../../../../../database/db';
+import { HTTP422CSVValidationError } from '../../../../../../errors/http-error';
+import { ImportDeviceService } from '../../../../../../services/import-services/devices/import-device-service';
+import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db';
+import { importTelemetryDeviceCSV } from './import';
+
+describe('importTelemetryDeviceCSV', () => {
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('imports device CSV returns status 200', async () => {
+    const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() });
+    const getDBConnectionStub = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+
+    const importCSVWorksheetStub = sinon.stub(ImportDeviceService.prototype, 'importCSVWorksheet');
+
+    importCSVWorksheetStub.resolves([]);
+
+    const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importTelemetryDeviceCSV();
+
+    await requestHandler(mockReq, mockRes, mockNext);
+
+    expect(mockDBConnection.open).to.have.been.calledOnce;
+
+    expect(getDBConnectionStub).to.have.been.calledOnce;
+
+    expect(importCSVWorksheetStub).to.have.been.calledOnce;
+
+    expect(mockRes.status).to.have.been.calledOnceWithExactly(200);
+    expect(mockRes.send).to.have.been.calledOnceWithExactly();
+
+    expect(mockDBConnection.commit).to.have.been.calledOnce;
+    expect(mockDBConnection.release).to.have.been.calledOnce;
+  });
+
+  it('fails validation and returns status 422 with errors', async () => {
+    const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() });
+    const getDBConnectionStub = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+
+    const importCSVWorksheetStub = sinon.stub(ImportDeviceService.prototype, 'importCSVWorksheet');
+
+    importCSVWorksheetStub.resolves([
+      {
+        error: 'Test error',
+        solution: 'Test solution',
+        row: 1,
+        header: 'TEST'
+      }
+    ]);
+
+    const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importTelemetryDeviceCSV();
+
+    try {
+      await requestHandler(mockReq, mockRes, mockNext);
+    } catch (err: any) {
+      expect(err).to.be.instanceOf(HTTP422CSVValidationError);
+      expect(err.errors).to.deep.equal([
+        {
+          error: 'Test error',
+          solution: 'Test solution',
+          row: 1,
+          header: 'TEST'
+        }
+      ]);
+
+      expect(mockDBConnection.open).to.have.been.calledOnce;
+      expect(getDBConnectionStub).to.have.been.calledOnce;
+      expect(importCSVWorksheetStub).to.have.been.calledOnce;
+      expect(mockDBConnection.release).to.have.been.calledOnce;
+    }
+  });
+});

api/src/paths/project/{projectId}/survey/{surveyId}/devices/import.ts

@@ -0,0 +1,142 @@
+import { RequestHandler } from 'express';
+import { Operation } from 'express-openapi';
+import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
+import { getDBConnection } from '../../../../../../database/db';
+import { HTTP422CSVValidationError } from '../../../../../../errors/http-error';
+import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv';
+import { csvFileSchema } from '../../../../../../openapi/schemas/file';
+import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
+import { ImportDeviceService } from '../../../../../../services/import-services/devices/import-device-service';
+import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface';
+import { getLogger } from '../../../../../../utils/logger';
+import { parseMulterFile } from '../../../../../../utils/media/media-utils';
+import { getFileFromRequest } from '../../../../../../utils/request';
+import { constructXLSXWorkbook, getDefaultWorksheet } from '../../../../../../utils/xlsx-utils/worksheet-utils';
+
+const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/devices/import');
+
+export const POST: Operation = [
+  authorizeRequestHandler((req) => {
+    return {
+      or: [
+        {
+          validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR],
+          surveyId: Number(req.params.surveyId),
+          discriminator: 'ProjectPermission'
+        },
+        {
+          validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR],
+          discriminator: 'SystemRole'
+        }
+      ]
+    };
+  }),
+  importTelemetryDeviceCSV()
+];
+
+POST.apiDoc = {
+  description: 'Upload survey telemetry device submission file.',
+  tags: ['telemetry'],
+  security: [
+    {
+      Bearer: []
+    }
+  ],
+  parameters: [
+    {
+      in: 'path',
+      name: 'projectId',
+      required: true
+    },
+    {
+      in: 'path',
+      name: 'surveyId',
+      required: true
+    }
+  ],
+  requestBody: {
+    description: 'Survey telemetry device submission file to upload',
+    required: true,
+    content: {
+      'multipart/form-data': {
+        schema: {
+          type: 'object',
+          additionalProperties: false,
+          required: ['media'],
+          properties: {
+            media: {
+              description: 'A survey telemetry device submission file.',
+              type: 'array',
+              minItems: 1,
+              maxItems: 1,
+              items: csvFileSchema
+            }
+          }
+        }
+      }
+    }
+  },
+  responses: {
+    200: {
+      description: 'Import OK'
+    },
+    400: {
+      $ref: '#/components/responses/400'
+    },
+    401: {
+      $ref: '#/components/responses/401'
+    },
+    403: {
+      $ref: '#/components/responses/403'
+    },
+    422: CSVValidationErrorResponse,
+    500: {
+      $ref: '#/components/responses/500'
+    },
+    default: {
+      $ref: '#/components/responses/default'
+    }
+  }
+};
+
+/**
+ * Imports telemetry devices from a CSV file.
+ *
+ * @return {*}  {RequestHandler}
+ */
+export function importTelemetryDeviceCSV(): RequestHandler {
+  return async (req, res) => {
+    const connection = getDBConnection(req.keycloak_token);
+
+    try {
+      await connection.open();
+
+      const surveyId = Number(req.params.surveyId);
+      const rawFile = getFileFromRequest(req);
+
+      const mediaFile = parseMulterFile(rawFile);
+      const worksheet = getDefaultWorksheet(constructXLSXWorkbook(mediaFile));
+
+      const deviceService = new ImportDeviceService(connection, worksheet, surveyId);
+
+      const errors = await deviceService.importCSVWorksheet();
+
+      if (errors.length) {
+        throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors);
+      }
+
+      await connection.commit();
+
+      return res.status(200).send();
+    } catch (error) {
+      if (error instanceof HTTP422CSVValidationError === false) {
+        defaultLog.error({ label: 'importDevices', message: 'error', error });
+      }
+
+      await connection.rollback();
+      throw error;
+    } finally {
+      connection.release();
+    }
+  };
+}

api/src/services/import-services/devices/import-device-service.test.ts

@@ -0,0 +1,98 @@
+import chai, { expect } from 'chai';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import { WorkSheet } from 'xlsx';
+import * as csv from '../../../utils/csv-utils/csv-config-validation';
+import { CSVConfig, CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface';
+import { getMockDBConnection } from '../../../__mocks__/db';
+import { ImportDeviceService } from './import-device-service';
+
+chai.use(sinonChai);
+
+describe('ImportDeviceService', () => {
+  beforeEach(() => {
+    sinon.restore();
+  });
+
+  describe('getCSVConfig', () => {
+    it('should return the CSVConfig for Device', async () => {
+      const connection = getMockDBConnection();
+      const worksheet = {} as WorkSheet;
+      const service = new ImportDeviceService(connection, worksheet, 1);
+
+      const getVendorsStub = sinon.stub(service.codeRepository, 'getActiveTelemetryDeviceMakes');
+      getVendorsStub.resolves([{ name: 'Lotek', description: 'Lotek', id: 10 }]);
+
+      const config = await service.getCSVConfig();
+
+      expect(getVendorsStub).to.have.been.calledOnceWithExactly();
+      expect(config.staticHeadersConfig).to.have.keys('SERIAL', 'VENDOR', 'MODEL', 'COMMENT');
+    });
+  });
+
+  describe('importCSVWorksheet', () => {
+    it('should import the CSV worksheet', async () => {
+      const mockConnection = getMockDBConnection();
+      const worksheet = {} as WorkSheet;
+      const surveyId = 1;
+
+      const service = new ImportDeviceService(mockConnection, worksheet, surveyId);
+
+      const mockCSVConfig = {} as CSVConfig;
+      const mockGetConfig = sinon.stub(service, 'getCSVConfig').resolves(mockCSVConfig);
+
+      const validateStub = sinon.stub(csv, 'validateCSVWorksheet').returns({
+        errors: [],
+        rows: [
+          {
+            SERIAL: '1234',
+            VENDOR: 'lotek',
+            MODEL: 'ModelX',
+            COMMENT: 'Test device',
+            [CSVRowState]: {}
+          }
+        ]
+      });
+
+      service.vendorNameToId = new Map([['lotek', 10]]);
+      const createDeviceStub = sinon.stub(service.telemetryDeviceService, 'createDevice').resolves();
+
+      const result = await service.importCSVWorksheet();
+
+      expect(mockGetConfig).to.have.been.called;
+      expect(validateStub).to.have.been.calledOnceWithExactly(worksheet, mockCSVConfig);
+      expect(createDeviceStub).to.have.been.calledOnceWithExactly({
+        survey_id: 1,
+        serial: '1234',
+        device_make_id: 10,
+        model: 'ModelX',
+        comment: 'Test device'
+      });
+      expect(result).to.deep.equal([]);
+    });
+
+    it('should return CSV Validation error if rows fail validation', async () => {
+      const mockConnection = getMockDBConnection();
+      const worksheet = {} as WorkSheet;
+      const surveyId = 1;
+
+      const service = new ImportDeviceService(mockConnection, worksheet, surveyId);
+
+      const mockCSVConfig = {} as CSVConfig;
+      const mockGetConfig = sinon.stub(service, 'getCSVConfig').resolves(mockCSVConfig);
+
+      const mockValidate = sinon.stub(csv, 'validateCSVWorksheet').returns({
+        errors: [{ error: 'error', solution: 'solution', values: [], cell: 'A1', row: 1, header: 'SERIAL' }],
+        rows: []
+      });
+
+      const errors = await service.importCSVWorksheet();
+
+      expect(mockGetConfig).to.have.been.called;
+      expect(mockValidate).to.have.been.calledOnceWithExactly(worksheet, mockCSVConfig);
+      expect(errors).to.deep.equal([
+        { error: 'error', solution: 'solution', values: [], cell: 'A1', row: 1, header: 'SERIAL' }
+      ]);
+    });
+  });
+});