Use csv-parse's sync API throughout the scripts

package.json

@@ -29,7 +29,6 @@
   },
   "dependencies": {
     "axios": "^1.13.1",
-    "csv-reader": "^1.0.12",
     "csv-parse": "^5.5.6",
     "dotenv": "^16.3.1",
     "jwt-decode": "^3.1.2",

src/csv.ts

@@ -0,0 +1,13 @@
+import { AssertionError } from 'assert';
+
+export type CsvRow = Record<string, string>;
+
+export function assertIsCsvRow(row: unknown): asserts row is CsvRow {
+  if (row === null
+    || typeof row !== 'object'
+    || Object.keys(row).length === 0
+    || Object.keys(row).some((key) => typeof key !== 'string')
+    || Object.values(row).some((value) => typeof value !== 'string')) {
+    throw new AssertionError({ message: 'Given row is not a CsvRow!' });
+  }
+}

src/generateApplicationFormJson.ts

@@ -2,12 +2,13 @@
 // The CSV is usually derived from the following URL using xlsx export and `xslx2csv`:
 // https://docs.google.com/spreadsheets/d/1Ep3_MEIyIbhxJ5TpH5x4Q1fRZqr1CFHXZ_uv3fEOSEk
 import {
-  createReadStream,
   createWriteStream,
+  readFileSync,
 } from 'fs';
-import CsvReadableStream from 'csv-reader';
+import { parse as csvParse } from 'csv-parse/sync';
 import { parse } from 'ts-command-line-args';
 import axios, { AxiosError } from 'axios';
+import { assertIsCsvRow } from './csv';
 import { logger } from './logger';
 
 interface Args {
@@ -19,10 +20,9 @@ interface Args {
   apiUrl: string;
 }
 
-type CsvRow = Record<string, string>;
 interface ApplicationFormField {
   baseFieldId: number;
-  position: number | string | undefined;
+  position: number;
   label: string | undefined;
 }
 interface ApplicationForm {
@@ -35,7 +35,7 @@ interface BaseField {
   label: string;
   shortCode: string;
   dataType: string;
-  createdAt: string;
+  createdAt: Date;
 }
 
 const args = parse<Args>({
@@ -47,7 +47,10 @@ const args = parse<Args>({
   apiUrl: String,
 });
 
-const csvInput = createReadStream(args.inputFile, 'utf8');
+const csvParser = csvParse(readFileSync(args.inputFile, 'utf8'), {
+  columns: true,
+}) as unknown[];
+
 const jsonOutput = createWriteStream(args.outputFile, 'utf8');
 const {
   opportunityId, funder, bearerToken, apiUrl,
@@ -67,34 +70,39 @@ axios(`${apiUrl}/baseFields`, {
   },
 }).then((response) => {
   const fields: BaseField[] = response.data as BaseField[];
-  csvInput.pipe(
-    new CsvReadableStream({
-      parseNumbers: true,
-      parseBooleans: true,
-      trim: true,
-      allowQuotes: true,
-      asObject: true,
-    }),
-  ).on('data', (row: CsvRow) => {
-    const label = `${funder}: field label`;
-    const id = `${funder}: external ID`;
-    const pos = `${funder}: form position`;
-    let field: BaseField | undefined;
-    if (row[id] !== '') {
+  const label = `${funder}: field label`;
+  const pos = `${funder}: form position`;
+  Promise.all(csvParser.map((row) => {
+    assertIsCsvRow(row);
+    let field: BaseField;
+    if (row[label] !== '') {
       const shortCode = row['Internal field name'];
-      field = fields.find((e) => e.shortCode === shortCode);
-      if (field) {
-        const applicationFormField: ApplicationFormField = {
-          baseFieldId: field.id,
-          position: row[pos] ? '' : (counter += 1),
-          label: row[label],
-        };
-        applicationForm.fields.push(applicationFormField);
+      const fieldsFiltered = fields.filter((e) => e.shortCode === shortCode);
+      if (fieldsFiltered.length === 1 && fieldsFiltered[0] !== undefined) {
+        [field] = fieldsFiltered;
+      } else {
+        const code = shortCode !== undefined ? shortCode : 'undefined';
+        throw new Error(`Found ${fieldsFiltered.length} base fields (expected 1): shortCode=${code}`);
       }
+      const position = row[pos];
+      const applicationFormField: ApplicationFormField = {
+        baseFieldId: field.id,
+        position: typeof position === 'number' ? position : (counter += 1),
+        label: row[label],
+      };
+      return applicationFormField;
     }
-  }).on('end', () => {
+    return undefined;
+  })).then((applicationFormFields) => {
+    applicationFormFields.forEach((field) => {
+      if (field !== undefined) {
+        applicationForm.fields.push(field);
+      }
+    });
     jsonOutput.write(JSON.stringify(applicationForm));
     jsonOutput.close();
+  }).catch((error: unknown) => {
+    logger.error(`Error creating form fields: ${JSON.stringify(error)}`);
   });
 }).catch((error: AxiosError) => {
   logger.error({ error }, 'Error getting base fields');

src/generateBaseFieldsInserts.ts

@@ -2,12 +2,15 @@
 // The CSV is usually derived from the following URL using xlsx export and `xslx2csv`:
 // https://docs.google.com/spreadsheets/d/1Ep3_MEIyIbhxJ5TpH5x4Q1fRZqr1CFHXZ_uv3fEOSEk
 import {
-  createReadStream,
   createWriteStream,
+  readFileSync,
 } from 'fs';
 import { EOL } from 'os';
-import CsvReadableStream from 'csv-reader';
+import { AssertionError } from 'assert';
+import { parse as csvParse } from 'csv-parse/sync';
 import { parse } from 'ts-command-line-args';
+import { assertIsCsvRow } from './csv';
+import { logger } from './logger';
 
 interface Args {
   inputFile: string;
@@ -19,38 +22,40 @@ const args = parse<Args>({
   outputFile: String,
 });
 
-interface CsvRow {
-  Label: string,
-  'Internal field name' : string,
-  Type: string
-}
-
-const csvInput = createReadStream(args.inputFile, 'utf8');
+const csvParser = csvParse(readFileSync(args.inputFile, 'utf8'), {
+  columns: true,
+  skip_records_with_empty_values: true,
+}) as unknown[];
 const sqlOutput = createWriteStream(args.outputFile, 'utf8');
 
 sqlOutput.write(`INSERT INTO base_fields (label, short_code, data_type) VALUES${EOL}`);
 
 let firstRowArrived = false;
-csvInput.pipe(
-  new CsvReadableStream({
-    parseNumbers: true,
-    parseBooleans: true,
-    trim: true,
-    allowQuotes: true,
-    asObject: true,
-  }),
-).on('data', (row: CsvRow) => {
+
+Promise.all(csvParser.map((row) => {
+  assertIsCsvRow(row);
   const label = row.Label;
+  if (typeof label !== 'string') {
+    throw new AssertionError({ message: 'Expected label to be a string' });
+  }
   const shortCode = row['Internal field name'];
+  if (typeof shortCode !== 'string') {
+    throw new AssertionError({ message: 'Expected shortCode to be a string' });
+  }
   const dataType = row.Type;
+  if (typeof dataType !== 'string') {
+    throw new AssertionError({ message: 'Expected dataType to be a string' });
+  }
 
   if (firstRowArrived) {
-    sqlOutput.write(`,${EOL}('${label}', '${shortCode}', '${dataType}' )`);
-  } else {
-    sqlOutput.write(`('${label}', '${shortCode}', '${dataType}' )`);
+    return `,${EOL}('${label}', '${shortCode}', '${dataType}' )`;
   }
   firstRowArrived = true;
-}).on('end', () => {
+  return `('${label}', '${shortCode}', '${dataType}' )`;
+})).then((insertStatements) => {
+  insertStatements.forEach((statement) => sqlOutput.write(statement));
   sqlOutput.write(`;${EOL}`);
   sqlOutput.close();
+}).catch((error: unknown) => {
+  logger.error(error, 'Error while reading CSV or writing SQL.');
 });

src/postProposalVersions.ts

@@ -1,10 +1,11 @@
 // Takes a CSV file, creates JSON bodies for POST /proposalVersions, posts them.
 import { readFileSync } from 'fs';
-import { AssertionError } from 'assert';
 import { parse as csvParse } from 'csv-parse/sync';
 import { parse as argParse } from 'ts-command-line-args';
 import axios, { AxiosError } from 'axios';
+import { assertIsCsvRow } from './csv';
 import { logger } from './logger';
+import type { CsvRow } from './csv';
 
 interface Args {
   inputFile: string;
@@ -15,18 +16,6 @@ interface Args {
   apiUrl: string;
 }
 
-type CsvRow = Record<string, string>;
-
-function assertIsCsvRow(row: unknown): asserts row is CsvRow {
-  if (row === null
-    || typeof row !== 'object'
-    || Object.keys(row).length === 0
-    || Object.keys(row).some((key) => typeof key !== 'string')
-    || Object.values(row).some((value) => typeof value !== 'string')) {
-    throw new AssertionError({ message: 'Given row is not a CsvRow!' });
-  }
-}
-
 interface Applicant {
   readonly id: number;
   externalId: string;