Implement bulk import (#1457)

* Implement bulk import

* Name jobs based on dataset ID, instead of name

* Fix test against pipeline run working directory name

* Wrap text for dataset name and import path

* Remove unnecessary eslint rule ignore

* Display number of currently selected datasets in bulk import dialog

* Handle bulk import with no results

* Fix sporatic failures in bulk import

Run imports sequentially, to prevent ffprobe from failing. ffprobe seems
to fail if run with sufficiently many concurrent processes.

* Split up import dialogs

* Prevent v-data-table overflow on smaller screen sizes

Author: jjnesbitt

client/dive-common/apispec.ts

@@ -161,7 +161,7 @@ interface Api {
   saveAttributeTrackFilters(datasetId: string,
     args: SaveAttributeTrackFilterArgs): Promise<unknown>;
   // Non-Endpoint shared functions
-  openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean):
+  openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean):
     Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>;
   getTiles?(itemId: string, projection?: string): Promise<StringKeyObject>;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any

client/dive-common/components/ImportButton.vue

@@ -20,7 +20,7 @@ export default defineComponent({
       required: true,
     },
     openType: {
-      type: String as PropType<DatasetType | 'zip'>,
+      type: String as PropType<DatasetType | 'zip' | 'bulk'>,
       required: true,
     },
     multiCamImport: { //TODO: Temporarily used to hide the stereo settings from users

client/platform/desktop/backend/ipcService.ts

@@ -70,6 +70,11 @@ export default function register() {
     return defaults;
   });
 
+  ipcMain.handle('bulk-import-media', async (event, { path }: { path: string }) => {
+    const results = await common.bulkMediaImport(path);
+    return results;
+  });
+
   ipcMain.handle('import-media', async (event, { path }: { path: string }) => {
     const ret = await common.beginMediaImport(path);
     return ret;

client/platform/desktop/backend/native/common.spec.ts

@@ -448,7 +448,7 @@ describe('native.common', () => {
       name: 'myproject1_name',
       createdAt: (new Date()).toString(),
       originalBasePath: '/foo/bar/baz',
-      id: 'myproject1',
+      id: 'myproject1_name_tktfgyv2g9',
       originalImageFiles: [],
       transcodedImageFiles: [],
       originalVideoFile: '',
@@ -461,7 +461,7 @@ describe('native.common', () => {
     const contents = fs.readdirSync(result);
     expect(stat.isDirectory()).toBe(true);
     expect(contents).toEqual([]);
-    expect(result).toMatch(/DIVE_Jobs\/myproject1_name_mypipeline\.pipe_/);
+    expect(result).toMatch(/DIVE_Jobs\/myproject1_name_tktfgyv2g9_mypipeline\.pipe_/);
   });
 
   it('beginMediaImport image sequence success', async () => {

client/platform/desktop/backend/native/common.ts

@@ -801,6 +801,59 @@ async function findTrackandMetaFileinFolder(path: string) {
   return { trackFileAbsPath, metaFileAbsPath };
 }
 
+/**
+ * Attempt a media import on the provided path, which may or may not be a valid dataset.
+ */
+async function attemptMediaImport(path: string) {
+  try {
+    // Must await here, as otherwise the try/catch isn't correctly executed.
+    return await beginMediaImport(path);
+  } catch (e) {
+    console.warn(
+      `*** Failed to import at path "${path}", with message: "${(e as Error).message}".`
+      + ' This is expected if this file or directory does not contain a dataset.',
+    );
+  }
+
+  return undefined;
+}
+
+/**
+ * Recursively import all datasets in this directory, using a "breadth-first" approach.
+ * This function only recurses into a directory if the import of that directory fails.
+ */
+async function bulkMediaImport(path: string): Promise<DesktopMediaImportResponse[]> {
+  const children = await fs.readdir(path, { withFileTypes: true });
+  const results: {path: fs.Dirent, result: DesktopMediaImportResponse | undefined}[] = [];
+
+  // Use a for-of loop, to run imports sequentially. If run concurrently, they can fail behind the scenes.
+  // eslint-disable-next-line no-restricted-syntax
+  for (const dirent of children) {
+    // eslint-disable-next-line no-await-in-loop
+    const result = await attemptMediaImport(npath.resolve(path, dirent.name));
+    results.push({
+      path: dirent,
+      result,
+    });
+  }
+
+  // Filter successful imports
+  const importResults = results.filter((r) => r.result !== undefined).map((r) => r.result as DesktopMediaImportResponse);
+
+  // If the result was undefined and was a directory, recurse.
+  const toRecurse = results.filter((r) => r.result === undefined && r.path.isDirectory());
+
+  // Use a for-of loop, to run imports sequentially. If run concurrently, they can fail behind the scenes.
+  // eslint-disable-next-line no-restricted-syntax
+  for (const r of toRecurse) {
+    // eslint-disable-next-line no-await-in-loop
+    const results = await bulkMediaImport(npath.resolve(path, r.path.name));
+    importResults.push(...results);
+  }
+
+  return importResults;
+}
+
 /**
  * Begin a dataset import.
  */
@@ -1133,6 +1186,7 @@ export {
   ProjectsFolderName,
   JobsFolderName,
   autodiscoverData,
+  bulkMediaImport,
   beginMediaImport,
   dataFileImport,
   deleteDataset,

client/platform/desktop/backend/native/utils.ts

@@ -91,7 +91,7 @@ async function createWorkingDirectory(settings: Settings, jsonMetaList: JsonMeta
   const jobFolderPath = path.join(settings.dataPath, JobsFolderName);
   // eslint won't recognize \. as valid escape
   // eslint-disable-next-line no-useless-escape
-  const safeDatasetName = jsonMetaList[0].name.replace(/[\.\s/]+/g, '_');
+  const safeDatasetName = jsonMetaList[0].id.replace(/[\.\s/]+/g, '_');
   const runFolderName = moment().format(`[${safeDatasetName}_${pipeline}]_MM-DD-yy_hh-mm-ss.SSS`);
   const runFolderPath = path.join(jobFolderPath, runFolderName);
   if (!fs.existsSync(jobFolderPath)) {

client/platform/desktop/frontend/api.ts

@@ -26,7 +26,7 @@ import {
  * Native functions that run entirely in the renderer
  */
 
-async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text', directory = false) {
+async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text', directory = false) {
   let filters: FileFilter[] = [];
   const allFiles = { name: 'All Files', extensions: ['*'] };
   if (datasetType === 'video') {
@@ -53,7 +53,7 @@ async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotati
       allFiles,
     ];
   }
-  const props = (datasetType === 'image-sequence' || directory) ? 'openDirectory' : 'openFile';
+  const props = (['image-sequence', 'bulk'].includes(datasetType) || directory) ? 'openDirectory' : 'openFile';
   const results = await dialog.showOpenDialog({
     properties: [props],
     filters,
@@ -117,6 +117,10 @@ function importMedia(path: string): Promise<DesktopMediaImportResponse> {
   return ipcRenderer.invoke('import-media', { path });
 }
 
+function bulkImportMedia(path: string): Promise<DesktopMediaImportResponse[]> {
+  return ipcRenderer.invoke('bulk-import-media', { path });
+}
+
 function deleteDataset(datasetId: string): Promise<boolean> {
   return ipcRenderer.invoke('delete-dataset', { datasetId });
 }
@@ -228,6 +232,7 @@ export {
   exportConfiguration,
   finalizeImport,
   importMedia,
+  bulkImportMedia,
   deleteDataset,
   checkDataset,
   importAnnotationFile,

client/platform/desktop/frontend/components/BulkImportDialog.vue

@@ -0,0 +1,193 @@
+<script lang="ts">
+import {
+  defineComponent, ref, PropType,
+  computed,
+} from 'vue';
+import { DesktopMediaImportResponse } from 'platform/desktop/constants';
+import { clone, cloneDeep } from 'lodash';
+import ImportDialog from './ImportDialog.vue';
+
+const headers = [
+  {
+    text: 'Dataset Name',
+    align: 'start',
+    sortable: false,
+    value: 'name',
+  },
+  {
+    text: 'Path',
+    align: 'start',
+    sortable: false,
+    value: 'path',
+  },
+  {
+    text: 'Dataset Type',
+    align: 'start',
+    sortable: false,
+    value: 'jsonMeta.type',
+    width: '150',
+  },
+  {
+    text: 'Config',
+    align: 'end',
+    sortable: false,
+    value: 'config',
+  },
+];
+
+export default defineComponent({
+  name: 'BulkImportDialog',
+  components: {
+    ImportDialog,
+  },
+  props: {
+    importData: {
+      type: Array as PropType<DesktopMediaImportResponse[]>,
+      required: true,
+    },
+  },
+  setup(props, ctx) {
+    // Map imports to include generated "id" field, used in rendering.
+    const imports = ref(props.importData.map((im) => {
+      const cloned = cloneDeep(im);
+      cloned.jsonMeta.id = (Math.random() + 1).toString(36).substring(2);
+
+      return cloned;
+    }));
+
+    // The dataset import currently being configured
+    const currentImport = ref<DesktopMediaImportResponse>();
+
+    // Selected state and selected imports maintain the ordering of `imports`
+    const selectedState = ref(imports.value.map(() => true));
+    const selectedImports = computed(() => imports.value.filter((_, index) => selectedState.value[index]));
+    function updateSelected(val: DesktopMediaImportResponse[]) {
+      selectedState.value = imports.value.map((im) => val.includes(im));
+    }
+
+    function stripId(item: DesktopMediaImportResponse) {
+      const cloned = cloneDeep(item);
+      cloned.jsonMeta.id = '';
+      return item;
+    }
+
+    function finalizeImport() {
+      const finalImports = imports.value.filter((im) => selectedImports.value.includes(im)).map((im) => stripId(im));
+      ctx.emit('finalize-import', finalImports);
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    function updateImportConfig(oldItem: DesktopMediaImportResponse, newItem: DesktopMediaImportResponse) {
+      const itemIndex = imports.value.indexOf(oldItem);
+
+      // Need to modify the imports array ref to contain the new item
+      const newArray = clone(imports.value);
+      newArray.splice(itemIndex, 1, newItem);
+      imports.value = newArray;
+
+      currentImport.value = undefined;
+    }
+
+    function formatPath(item: DesktopMediaImportResponse) {
+      let path = item.jsonMeta.originalBasePath;
+
+      if (item.jsonMeta.originalVideoFile !== '') {
+        path = `${path}/${item.jsonMeta.originalVideoFile}`;
+      }
+
+      return path;
+    }
+
+    return {
+      updateSelected,
+      updateImportConfig,
+      formatPath,
+      imports,
+      headers,
+      selectedImports,
+      currentImport,
+      finalizeImport,
+    };
+  },
+});
+</script>
+
+<template>
+  <v-card outlined class="import-card" style="overflow-x: hidden;">
+    <v-card-title class="text-h5">
+      Bulk Import (Selecting {{ selectedImports.length }} of {{ imports.length }})
+    </v-card-title>
+
+    <v-dialog :value="currentImport !== undefined" width="800">
+      <ImportDialog
+        v-if="currentImport !== undefined"
+        :import-data="currentImport"
+        :embedded="true"
+        @abort="currentImport = undefined"
+        @finalize-import="updateImportConfig(currentImport, $event)"
+      />
+    </v-dialog>
+
+    <v-data-table
+      :value="selectedImports"
+      :items="imports"
+      item-key="jsonMeta.id"
+      :headers="headers"
+      show-select
+      disable-sort
+      @input="updateSelected"
+    >
+      <template #item.name="{ item }">
+        <div class="text-wrap" style="word-break: break-word;">
+          {{ item.jsonMeta.name }}
+        </div>
+      </template>
+
+      <template #item.path="{ item }">
+        <div class="text-wrap" style="word-break: break-word;">
+          {{ formatPath(item) }}
+        </div>
+      </template>
+
+      <template #item.config="{ item }">
+        <v-btn
+          icon
+          :disabled="!selectedImports.includes(item)"
+          @click="currentImport = item"
+        >
+          <v-icon>
+            mdi-cog
+          </v-icon>
+        </v-btn>
+      </template>
+    </v-data-table>
+
+    <v-card-text>
+      <div class="d-flex flex-row mt-4">
+        <v-spacer />
+        <v-btn
+          text
+          outlined
+          class="mr-5"
+          @click="$emit('abort')"
+        >
+          Cancel
+        </v-btn>
+        <v-btn
+          color="primary"
+          @click="finalizeImport"
+        >
+          Import Selected
+        </v-btn>
+      </div>
+    </v-card-text>
+  </v-card>
+</template>
+
+<style lang="scss">
+@import 'dive-common/components/styles/KeyValueTable.scss';
+
+.v-data-table__selected {
+  background: unset !important;
+}
+</style>

client/platform/desktop/frontend/components/ImportDialog.vue

@@ -22,6 +22,11 @@ export default defineComponent({
       type: Boolean,
       default: false,
     },
+    // If being embedded into the bulk import dialog
+    embedded: {
+      type: Boolean,
+      default: false,
+    },
   },
   setup(props) {
     const argCopy = ref(cloneDeep(props.importData));
@@ -321,7 +326,7 @@ export default defineComponent({
           :disabled="!ready || disabled"
           @click="$emit('finalize-import', argCopy)"
         >
-          Finish Import
+          {{ embedded ? "Save" : "Finish Import" }}
         </v-btn>
       </div>
     </v-card-text>