fix exporting, download overflow, poetry wheels locking (#1433)

Author: BryonLewis

client/platform/web-girder/views/Export.vue

@@ -1,3 +1,4 @@
+<!-- eslint-disable vue/no-ref-as-operand -->
 <script lang="ts">
 import {
   computed, defineComponent, ref, shallowRef, toRef, watch, PropType, Ref,
@@ -43,14 +44,14 @@ export default defineComponent({
 
     /** State populated from provides if the dialog exists inside a viewer context */
     let save = () => Promise.resolve();
-    const pendingSaveCount = ref(0);
-    const checkedTypes = ref([] as readonly string[]);
-    const revisionId = ref(null as null | number);
+    let pendingSaveCount = ref(0);
+    let checkedTypes = ref([] as readonly string[]);
+    let revisionId = ref(null as null | number);
     if (props.blockOnUnsaved) {
       save = useHandler().save;
-      pendingSaveCount.value = usePendingSaveCount().value;
-      checkedTypes.value = useTrackFilters().checkedTypes.value;
-      revisionId.value = useRevisionId().value;
+      pendingSaveCount = usePendingSaveCount();
+      checkedTypes = useTrackFilters().checkedTypes;
+      revisionId = useRevisionId();
     }
 
     async function doExport({ forceSave = false, url }: { url?: string; forceSave?: boolean }) {
@@ -225,6 +226,7 @@ export default defineComponent({
       />
       <v-card
         outlined
+        class="downloadMenu"
       >
         <v-card-title>
           Download options
@@ -266,7 +268,7 @@ export default defineComponent({
 
           <v-card-text class="pb-2">
             <div>Get latest annotation csv only</div>
-            <template v-if="dataset.confidenceFilters">
+            <template v-if="dataset.confidenceFilters || true">
               <v-checkbox
                 v-model="excludeBelowThreshold"
                 label="exclude tracks below confidence threshold"
@@ -391,3 +393,11 @@ export default defineComponent({
     </template>
   </v-menu>
 </template>
+
+<style scoped>
+.downloadMenu {
+  max-height: calc(100vh - 30px);
+  overflow-y: auto;
+  overflow-x: hidden;
+}
+</style>

docker/girder.Dockerfile

@@ -24,7 +24,7 @@ WORKDIR /opt/dive/src
 RUN apt-get update
 RUN apt-get install -y build-essential libssl-dev libffi-dev python3-libtiff libgdal-dev python3-dev cargo npm
 # Recommended poetry install https://python-poetry.org/docs/master/#installation
-RUN curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 POETRY_HOME=/opt/dive/poetry python -
+RUN curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.3 POETRY_HOME=/opt/dive/poetry python -
 ENV PATH="/opt/dive/poetry/bin:$PATH"
 # Create a virtual environment for the installation
 RUN python -m venv /opt/dive/local/venv

docker/girder_worker.Dockerfile

@@ -33,7 +33,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
 RUN ln -s /usr/bin/python3.11 /usr/bin/python
 WORKDIR /opt/dive/src
 
-RUN curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 POETRY_HOME=/opt/dive/poetry python -
+RUN curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.3 POETRY_HOME=/opt/dive/poetry python -
 ENV PATH="/opt/dive/poetry/bin:$PATH"
 # Create a virtual environment for the installation
 RUN python -m venv --copies /opt/dive/local/venv

server/dive_server/crud_dataset.py

@@ -306,12 +306,16 @@ def makeDiveJson():
                     if (not nestedExcludeBelowThreshold) or track.exceeds_thresholds(thresholds):
                         # filter by types if applicable
                         if nestedTypeFilter:
-                            confidence_pairs = [item for item in track.confidencePairs if item[0] in nestedTypeFilter]
+                            confidence_pairs = [
+                                item
+                                for item in track.confidencePairs
+                                if item[0] in nestedTypeFilter
+                            ]
                             # skip line if no confidence pairs
                             if not confidence_pairs:
                                 continue
                         updated_tracks[t] = tracks[t]
-                annotations['tracks'] = updated_tracks         
+                annotations['tracks'] = updated_tracks
                 yield json.dumps(annotations)
 
             for data in z.addFile(makeMetajson, Path(f'{zip_path}meta.json')):

server/dive_server/crud_rpc.py

@@ -331,28 +331,53 @@ def _get_data_by_type(
         )
         meta = None
         if fps is not None:
-            meta = { "fps" : fps }
-        return {'annotations': converted, 'meta': meta, 'attributes': attributes, 'type': as_type}, warnings
+            meta = {"fps": fps}
+        return {
+            'annotations': converted,
+            'meta': meta,
+            'attributes': attributes,
+            'type': as_type,
+        }, warnings
     if as_type == crud.FileType.MEVA_KPF:
         converted, attributes = kpf.convert(kpf.load(file_string))
-        return {'annotations': converted, 'meta': None, 'attributes': attributes, 'type': as_type}, warnings
+        return {
+            'annotations': converted,
+            'meta': None,
+            'attributes': attributes,
+            'type': as_type,
+        }, warnings
 
     # All filetypes below are JSON, so if as_type was specified, it needs to be loaded.
     if data_dict is None:
         data_dict = json.loads(file_string)
     if as_type == crud.FileType.COCO_JSON:
         converted, attributes = kwcoco.load_coco_as_tracks_and_attributes(data_dict)
-        return {'annotations': converted, 'meta': None, 'attributes': attributes, 'type': as_type}, warnings
+        return {
+            'annotations': converted,
+            'meta': None,
+            'attributes': attributes,
+            'type': as_type,
+        }, warnings
     if as_type == crud.FileType.DIVE_CONF:
-        return {'annotations': None, 'meta': data_dict, 'attributes': None, 'type': as_type}, warnings
+        return {
+            'annotations': None,
+            'meta': data_dict,
+            'attributes': None,
+            'type': as_type,
+        }, warnings
     if as_type == crud.FileType.DIVE_JSON:
         migrated = dive.migrate(data_dict)
         annotations, attributes = viame.load_json_as_track_and_attributes(data_dict)
-        return {'annotations': migrated, 'meta': None, 'attributes': attributes, 'type': as_type}, warnings
+        return {
+            'annotations': migrated,
+            'meta': None,
+            'attributes': attributes,
+            'type': as_type,
+        }, warnings
     return None, None
 
 
-def process_items( 
+def process_items(
     folder: types.GirderModel,
     user: types.GirderUserModel,
     additive=False,
@@ -423,6 +448,7 @@ def process_items(
             crud_dataset.update_metadata(folder, results['meta'], False)
     return aggregate_warnings
 
+
 def postprocess(
     user: types.GirderUserModel,
     dsFolder: types.GirderModel,

server/dive_server/views_annotation.py

@@ -9,7 +9,7 @@
 from girder.exceptions import RestException
 from girder.models.folder import Folder
 
-from dive_utils import constants, setContentDisposition, models, fromMeta
+from dive_utils import constants, fromMeta, models, setContentDisposition
 
 from . import crud, crud_annotation
 
@@ -170,7 +170,9 @@ def export(
                 if (not excludeBelowThreshold) or track.exceeds_thresholds(thresholds):
                     # filter by types if applicable
                     if typeFilter:
-                        confidence_pairs = [item for item in track.confidencePairs if item[0] in typeFilter]
+                        confidence_pairs = [
+                            item for item in track.confidencePairs if item[0] in typeFilter
+                        ]
                         # skip line if no confidence pairs
                         if not confidence_pairs:
                             continue

server/dive_utils/models.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 from bson.objectid import ObjectId
 from pydantic import BaseModel, Field, validator
@@ -60,14 +60,31 @@ class BaseAnnotation(BaseModel):
     attributes: Dict[str, Any] = Field(default_factory=lambda: {})
     meta: Optional[Dict[str, Any]]
 
-    def exceeds_thresholds(self, thresholds: Dict[str, float]) -> bool:
+    def exceeds_thresholds(self, thresholds: Dict[str, float], typeFilter: Set[str]) -> bool:
         defaultThresh = thresholds.get('default', 0)
-        return any(
-            [
-                confidence >= thresholds.get(field, defaultThresh)
-                for field, confidence in self.confidencePairs
-            ]
-        )
+
+        # Check if there is any confidence value that exceeds the threshold
+        exceeds_default_threshold = False
+        for field, confidence in self.confidencePairs:
+            if confidence > thresholds.get(field, defaultThresh):
+                if (
+                    typeFilter and field in typeFilter
+                ):  # field is in the set and confidence > threshold
+                    exceeds_default_threshold = True
+                elif (
+                    typeFilter
+                ):  # if typeFilter has a set and the field isn't in it we return false
+                    exceeds_default_threshold = False
+                elif (
+                    not typeFilter
+                ):  # if no typeFilter we return true based on confidence > threshold
+                    exceeds_default_threshold = True
+            else:
+                exceeds_default_threshold = False
+            if exceeds_default_threshold:
+                return True
+
+        return False
 
     def __hash__(self):
         return self.id

server/dive_utils/serializers/viame.py

@@ -68,7 +68,7 @@ def _deduceType(value: Any) -> Union[bool, float, str, None]:
         return None
     if value is None:
         return None
-    
+
     if value == "true":
         return True
     if value == "false":
@@ -279,7 +279,8 @@ def custom_sort(row):
 
 
 def load_csv_as_tracks_and_attributes(
-    rows: List[str], imageMap: Optional[Dict[str, int]] = None,
+    rows: List[str],
+    imageMap: Optional[Dict[str, int]] = None,
 ) -> Tuple[types.DIVEAnnotationSchema, dict, List[str], Optional[str]]:
     """
     Convert VIAME CSV to json tracks
@@ -300,8 +301,8 @@ def load_csv_as_tracks_and_attributes(
     for row in sortedlist:
         if len(row) == 0 or row[0].startswith('#'):
             # This is not a data row
-            if (len(row) > 0 and row[0] == '# metadata'):
-                if (row[1].startswith('Fps: ')):
+            if len(row) > 0 and row[0] == '# metadata':
+                if row[1].startswith('Fps: '):
                     fps_splits = row[1].split(':')
                     if len(fps_splits) > 1:
                         fps = fps_splits[1]
@@ -500,7 +501,7 @@ def export_tracks_as_csv(
 
     for t in track_iterator:
         track = Track(**t)
-        if (not excludeBelowThreshold) or track.exceeds_thresholds(thresholds):
+        if (not excludeBelowThreshold) or track.exceeds_thresholds(thresholds, typeFilter):
             # filter by types if applicable
             if typeFilter:
                 confidence_pairs = [item for item in track.confidencePairs if item[0] in typeFilter]