FCRM-5897 PoC for uploading a shapefile (Not secure)

package.json

@@ -23,7 +23,10 @@
     "postinstall": "npm run build",
     "kill": "sudo kill -9 $(sudo lsof -t -i:3000)",
     "increment-version": "npm version prerelease --preid=pre -m \"Set Version to %s\"",
-    "set-config": "run(){  cp ./config/server.$1.json ./config/server.json; }; run"
+    "set-config": "run(){  cp ./config/server.$1.json ./config/server.json; }; run",
+    "build:container": "docker build -t fmp-app .",
+    "start:container": "docker --context desktop-linux run --platform linux/x86_64 --env-file=.env -p 3001:3001 fmp-app",
+    "stop:container": "docker kill $(docker ps -q -f ancestor=fmp-app)"
   },
   "author": "defra",
   "license": "ISC",
@@ -62,9 +65,11 @@
     "html-webpack-plugin": "^5.6.3",
     "identity-obj-proxy": "^3.0.0",
     "joi": "^18.0.1",
+    "jszip": "^3.10.1",
     "magic-comments-loader": "^2.1.4",
     "mini-css-extract-plugin": "^2.9.2",
     "moment-timezone": "^0.6.0",
+    "multiparty": "^4.2.3",
     "ngr-to-bng": "0.0.1",
     "node-cache": "^5.1.2",
     "nunjucks": "^3.2.4",
@@ -74,6 +79,7 @@
     "sass": "^1.93.2",
     "sass-loader": "^16.0.3",
     "sass-mq": "^7.0.0",
+    "shpjs": "^6.2.0",
     "url-loader": "^4.1.1",
     "webpack": "^5.104.1",
     "webpack-bundle-analyzer": "^4.10.2",

server/constants.js

@@ -22,6 +22,7 @@ const FEEDBACK = 'feedback'
 const ORDER_NOT_SUBMITTED = 'order-not-submitted'
 const OS_TERMS = 'os-terms'
 const TERMS_AND_CONDITIONS = 'terms-and-conditions'
+const UPLOAD = 'upload'
 
 const views = {
   HOME,
@@ -42,7 +43,8 @@ const views = {
   FEEDBACK,
   ORDER_NOT_SUBMITTED,
   OS_TERMS,
-  TERMS_AND_CONDITIONS
+  TERMS_AND_CONDITIONS,
+  UPLOAD
 }
 
 const routes = {

server/plugins/router.js

@@ -27,7 +27,8 @@ const routes = [].concat(
   require('../routes/public'),
   require('../routes/results'),
   require('../routes/terms-and-conditions'),
-  require('../routes/triage')
+  require('../routes/triage'),
+  require('../routes/upload')
 )
 
 module.exports = {

server/routes/__tests__/upload.spec.js

@@ -0,0 +1,106 @@
+const constants = require('../../constants')
+const {
+  submitGetRequest,
+  submitPostRequest,
+  submitPostRequestExpectHandledError,
+  submitPostRequestExpectServiceError
+} = require('../../__test-helpers__/server')
+const mockPart = { filename: 'test.zip' }
+const shp = require('shpjs').default
+const { extractProjectionFiles } = require('../../services/zip-helper')
+const setupZipMocks = (geojson = validGeoJSON) => {
+  extractProjectionFiles.mockResolvedValue(new ArrayBuffer(8))
+  shp.mockResolvedValue(geojson)
+}
+
+jest.mock('../../services/zip-helper', () => ({
+  extractProjectionFiles: jest.fn()
+}))
+jest.mock('shpjs', () => ({
+  __esModule: true,
+  default: jest.fn()
+}), { virtual: true })
+jest.mock('../../services/validate-uploaded-shape-file', () => ({
+  validateShapeFile: jest.fn(),
+  validateGeoJSON: jest.fn()
+}))
+jest.mock('../../services/file-helper', () => ({
+  getFile: jest.fn(),
+  streamToBuffer: jest.fn()
+}))
+
+const { getFile, streamToBuffer } = require('../../services/file-helper')
+const { validateShapeFile, validateGeoJSON } = require('../../services/validate-uploaded-shape-file')
+
+const url = constants.routes.UPLOAD
+
+const validGeoJSON = {
+  features: [
+    {
+      geometry: {
+        type: 'Polygon',
+        coordinates: [[[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]]
+      }
+    }
+  ]
+}
+
+beforeEach(() => {
+  getFile.mockResolvedValue(mockPart)
+  streamToBuffer.mockResolvedValue(Buffer.from('fake zip data'))
+  validateShapeFile.mockReturnValue([])
+  validateGeoJSON.mockReturnValue([])
+  setupZipMocks()
+})
+
+afterEach(() => {
+  jest.clearAllMocks()
+})
+
+describe('Upload route', () => {
+  describe('GET', () => {
+    it('should return the upload view', async () => {
+      await submitGetRequest({ url }, 'Upload a boundary')
+    })
+  })
+
+  describe('POST', () => {
+    describe('file validation', () => {
+      it('should return an error for an invalid file extension', async () => {
+        validateShapeFile.mockReturnValue([{ text: 'Only upload a GeoJSON file (.geojson), Geopackage (.gpkg) or Shape files (.zip)', href: '#boundary' }])
+        await submitPostRequestExpectHandledError(
+          { url },
+          'Only upload a GeoJSON file (.geojson), Geopackage (.gpkg) or Shape files (.zip)'
+        )
+      })
+
+      describe('GeoJSON validation', () => {
+        it('should return an error if geojson is invalid', async () => {
+          validateGeoJSON.mockReturnValue([{ text: 'Only upload a GeoJSON with a single feature.', href: '#boundary' }])
+          await submitPostRequestExpectHandledError(
+            { url },
+            'Only upload a GeoJSON with a single feature.'
+          )
+        })
+      })
+    })
+
+    describe('successful upload', () => {
+      it('should redirect to the map route with the polygon coordinates', async () => {
+        setupZipMocks()
+        const response = await submitPostRequest({ url })
+        const expectedPolygon = JSON.stringify(
+          validGeoJSON.features[0].geometry.coordinates[0]
+        )
+        expect(response.headers.location).toBe(`${constants.routes.MAP}?polygon=${expectedPolygon}`)
+      })
+    })
+
+    describe('error handling', () => {
+      it('should return a service error if getFile fails', async () => {
+        getFile.mockRejectedValue(new Error('Form parse error'))
+        await submitPostRequestExpectServiceError({ url })
+      })
+    })
+  })
+})

server/routes/upload.js

@@ -0,0 +1,58 @@
+const { getFile, streamToBuffer } = require('../services/file-helper')
+const constants = require('../constants')
+const { validateShapeFile, validateGeoJSON } = require('../services/validate-uploaded-shape-file')
+const fiftyMbNumeric = 50
+const fiftyMbInBytes = fiftyMbNumeric * 1024 * 1024
+const { extractProjectionFiles } = require('../services/zip-helper')
+
+const handlers = {
+  get: async (_request, h) => h.view(constants.views.UPLOAD),
+  post: async (request, h) => {
+    const file = await getFile(request)
+    const errorSummary = validateShapeFile(file)
+    if (errorSummary.length > 0) {
+      return h.view(constants.views.UPLOAD, {
+        errorSummary
+      })
+    }
+    const { default: shp } = await import('shpjs') // needs to be imported here as only ESM can be used with shpjs
+    const buffer = await streamToBuffer(file)
+    const modifiedBuffer = await extractProjectionFiles(buffer)
+    const geojson = await shp(modifiedBuffer)
+    const boundaryErrorSummary = validateGeoJSON(geojson)
+
+    if (boundaryErrorSummary.length > 0) {
+      return h.view(constants.views.UPLOAD, {
+        errorSummary: boundaryErrorSummary
+      })
+    }
+
+    const polygon = geojson.features[0].geometry.coordinates[0]
+
+    return h.redirect(`${constants.routes.MAP}?polygon=${JSON.stringify(polygon)}`)
+  }
+}
+
+module.exports = [
+  {
+    method: 'GET',
+    path: constants.routes.UPLOAD,
+    options: {
+      description: 'Upload Page',
+      handler: handlers.get
+    }
+  },
+  {
+    method: 'POST',
+    path: constants.routes.UPLOAD,
+    handler: handlers.post,
+    options: {
+      payload: {
+        maxBytes: fiftyMbInBytes,
+        multipart: true,
+        output: 'stream',
+        parse: false
+      }
+    }
+  }
+]

server/services/__tests__/file-helper.spec.js

@@ -0,0 +1,94 @@
+const multiparty = require('multiparty')
+const { getFile, streamToBuffer } = require('../file-helper')
+
+jest.mock('multiparty')
+
+let mockForm
+let mockPart
+
+beforeEach(() => {
+  mockPart = {
+    filename: 'test.zip',
+    [Symbol.asyncIterator]: async function * () {
+      yield Buffer.from('fake zip data')
+    }
+  }
+
+  mockForm = {
+    on: jest.fn(),
+    parse: jest.fn()
+  }
+
+  multiparty.Form.mockImplementation(() => mockForm)
+
+  mockForm.on.mockImplementation((event, handler) => {
+    if (event === 'part') {
+      setImmediate(() => handler(mockPart))
+    }
+  })
+})
+
+afterEach(() => {
+  jest.clearAllMocks()
+})
+
+describe('getFile', () => {
+  it('should resolve with the file part when a file is received', async () => {
+    const result = await getFile({ raw: { req: {} } })
+    expect(result).toBe(mockPart)
+  })
+
+  it('should call form.parse with the raw request', async () => {
+    const mockRawReq = {}
+    await getFile({ raw: { req: mockRawReq } })
+    expect(mockForm.parse).toHaveBeenCalledWith(mockRawReq)
+  })
+
+  it('should reject if a non-file part is received', async () => {
+    mockForm.on.mockImplementation((event, handler) => {
+      if (event === 'part') setImmediate(() => handler({ filename: null }))
+    })
+    await expect(getFile({ raw: { req: {} } })).rejects.toThrow('Non file received')
+  })
+
+  it('should reject if multiparty emits an error', async () => {
+    mockForm.on.mockImplementation((event, handler) => {
+      if (event === 'error') setImmediate(() => handler(new Error('Form parse error')))
+    })
+    await expect(getFile({ raw: { req: {} } })).rejects.toThrow('Form parse error')
+  })
+
+  it('should preserve the original error when multiparty emits an error', async () => {
+    const originalError = new Error('Form parse error')
+    mockForm.on.mockImplementation((event, handler) => {
+      if (event === 'error') setImmediate(() => handler(originalError))
+    })
+    await expect(getFile({ raw: { req: {} } })).rejects.toBe(originalError)
+  })
+})
+
+describe('streamToBuffer', () => {
+  it('should convert a stream to a buffer', async () => {
+    const result = await streamToBuffer(mockPart)
+    expect(result).toEqual(Buffer.from('fake zip data'))
+  })
+
+  it('should handle a stream with multiple chunks', async () => {
+    const multiChunkStream = {
+      [Symbol.asyncIterator]: async function * () {
+        yield Buffer.from('chunk one ')
+        yield Buffer.from('chunk two')
+      }
+    }
+    const result = await streamToBuffer(multiChunkStream)
+    expect(result).toEqual(Buffer.from('chunk one chunk two'))
+  })
+
+  it('should return an empty buffer for an empty stream', async () => {
+    const emptyStream = {
+      [Symbol.asyncIterator]: async function * () { }
+    }
+    const result = await streamToBuffer(emptyStream)
+    expect(result).toEqual(Buffer.alloc(0))
+  })
+})