Allow overlapping annos in Trainer (#245)

* CU-869bacykn: enable overlapping span rendering on annotions in prep for medcatv2 models that support this

* CU-869bacykn: frontend tests for clinical text component

---------

Co-authored-by: Tom Searle <tom@cogstack.org>

Author: tomolopolis

medcat-trainer/webapp/api/api/utils.py

@@ -155,14 +155,6 @@ def get_create_cdb_infos(cdb, concept, cui, cui_info_prop, code_prop, desc_prop,
     return model_clazz.objects.filter(code__in=codes)
 
 
-def _remove_overlap(project, document, start, end):
-    anns = AnnotatedEntity.objects.filter(project=project, document=document)
-
-    for ann in anns:
-        if (start <= ann.start_ind <= end) or (start <= ann.end_ind <= end):
-            logger.debug("Removed %s ", str(ann))
-            ann.delete()
-
 
 def create_annotation(source_val: str, selection_occurrence_index: int, cui: str, user: User,
                       project: ProjectAnnotateEntities, document, cat: CAT):
@@ -180,9 +172,8 @@ def create_annotation(source_val: str, selection_occurrence_index: int, cui: str
     start = all_occurrences_start_idxs[selection_occurrence_index]
 
     if start is not None and len(source_val) > 0 and len(cui) > 0:
-        # Remove overlaps
+        # Allow overlapping annotations - removed overlap constraint
         end = start + len(source_val)
-        _remove_overlap(project, document, start, end)
 
         cnt = Entity.objects.filter(label=cui).count()
         if cnt == 0:

medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue

@@ -78,48 +78,152 @@ export default {
         return this.addAnnos ? `<div @contextmenu.prevent.stop="showCtxMenu($event)">${text}</div>` : `<div>${text}</div>`
       }
 
+      // Sort entities by start_ind, then by end_ind (longer first for same start)
+      // Preserve original index for click handlers
+      const sortedEnts = this.ents.map((ent, origIdx) => ({ ent, origIdx })).sort((a, b) => {
+        if (a.ent.start_ind !== b.ent.start_ind) {
+          return a.ent.start_ind - b.ent.start_ind
+        }
+        return b.ent.end_ind - a.ent.end_ind // Longer spans first when same start
+      })
+
       const taskHighlightDefault = 'highlight-task-default'
-      let formattedText = ''
-      let start = 0
       let timeout = 0
-      for (let i = 0; i < this.ents.length; i++) {
-        // highlight the span with default
-        let highlightText = this.text.slice(this.ents[i].start_ind, this.ents[i].end_ind)
 
+      // Create events for start and end of each annotation
+      const events = []
+      sortedEnts.forEach((entData, i) => {
+        const ent = entData.ent
+        const origIdx = entData.origIdx
+        events.push({ pos: ent.start_ind, type: 'start', entIndex: i, origIndex: origIdx, ent: ent })
+        events.push({ pos: ent.end_ind, type: 'end', entIndex: i, origIndex: origIdx, ent: ent })
+      })
+      events.sort((a, b) => {
+        if (a.pos !== b.pos) {
+          return a.pos - b.pos
+        }
+        // At same position, process starts before ends
+        if (a.type !== b.type) {
+          return a.type === 'start' ? -1 : 1
+        }
+        return 0
+      })
+
+      let formattedText = ''
+      let currentPos = 0
+      const activeEnts = [] // Stack of active entities (ordered by when they were opened)
+
+      // Helper function to get style class for an entity
+      const getStyleClass = (ent, origIndex) => {
         let styleClass = taskHighlightDefault
-        if (this.ents[i].assignedValues[this.taskName] !== null) {
-          let btnIndex = this.taskValues.indexOf(this.ents[i].assignedValues[this.taskName])
+        if (ent.assignedValues[this.taskName] !== null) {
+          let btnIndex = this.taskValues.indexOf(ent.assignedValues[this.taskName])
           styleClass = `highlight-task-${btnIndex}`
         }
 
-        if (this.ents[i].id === this.currentRelStartEnt.id) {
+        if (ent.id === this.currentRelStartEnt.id) {
           styleClass += ' current-rel-start'
-        } else if (this.ents[i].id === this.currentRelEndEnt.id) {
+        } else if (ent.id === this.currentRelEndEnt.id) {
           styleClass += ' current-rel-end'
         }
 
-        styleClass = this.ents[i] === this.currentEnt ? `${styleClass} highlight-task-selected` : styleClass
-        timeout = this.ents[i] === this.currentEnt && i === 0 ? 500 : timeout
+        if (ent === this.currentEnt) {
+          styleClass += ' highlight-task-selected'
+          timeout = origIndex === 0 ? 500 : timeout
+        }
+        return styleClass
+      }
+
+      // Helper function to build opening span tag
+      const buildOpenSpan = (ent, origIndex) => {
+        const styleClass = getStyleClass(ent, origIndex)
+        return `<span @click.stop="selectEnt(${origIndex})" class="${styleClass}">`
+      }
 
+      // Helper function to build closing span tag with optional remove button
+      const buildCloseSpan = (ent, origIndex, isInnermost) => {
         let removeButtonEl = ''
-        if (this.ents[i].manually_created) {
-          removeButtonEl = `<font-awesome-icon icon="times" class="remove-new-anno" @click="removeNewAnno(${i})"></font-awesome-icon>`
+        if (isInnermost && ent.manually_created) {
+          removeButtonEl = `<font-awesome-icon icon="times" class="remove-new-anno" @click.stop="removeNewAnno(${origIndex})"></font-awesome-icon>`
         }
-        let spanText = `<span @click="selectEnt(${i})" class="${styleClass}">${_.escape(highlightText)}${removeButtonEl}</span>`
-
-        let precedingText = _.escape(this.text.slice(start, this.ents[i].start_ind))
-        precedingText = precedingText.length !== 0 ? precedingText : ' '
-        start = this.ents[i].end_ind
-        formattedText += precedingText + spanText
-        if (i === this.ents.length - 1) {
-          formattedText += this.text.slice(start, this.text.length)
+        return `${removeButtonEl}</span>`
+      }
+
+      for (const event of events) {
+        // Handle start events first (before adding text)
+        if (event.type === 'start') {
+          // Add any text up to this point
+          if (event.pos > currentPos) {
+            const textSegment = this.text.slice(currentPos, event.pos)
+            if (textSegment.length > 0) {
+              formattedText += _.escape(textSegment)
+            }
+            currentPos = event.pos
+          }
+          // Open the span for this annotation
+          formattedText += buildOpenSpan(event.ent, event.origIndex)
+          activeEnts.push({ entIndex: event.entIndex, origIndex: event.origIndex, ent: event.ent })
+        } else if (event.type === 'end') {
+          // Close the span (in reverse order to maintain nesting)
+          const index = activeEnts.findIndex(ae => ae.entIndex === event.entIndex)
+          if (index !== -1) {
+            // If this is not the innermost span, we need to handle overlapping text
+            if (index < activeEnts.length - 1) {
+              // Add text up to the end position while all spans are still active
+              // This text is inside all active spans including this one
+              if (event.pos > currentPos) {
+                const textSegment = this.text.slice(currentPos, event.pos)
+                if (textSegment.length > 0) {
+                  formattedText += _.escape(textSegment)
+                }
+                currentPos = event.pos
+              }
+              // Close all inner spans (from innermost to the one after this)
+              // Don't add remove buttons here - these are temporary closes for nesting
+              // We'll add remove buttons only when we reach the actual end position
+              for (let j = activeEnts.length - 1; j > index; j--) {
+                const innerData = activeEnts[j]
+                // Always pass false here - these are temporary closes, not final ends
+                formattedText += buildCloseSpan(innerData.ent, innerData.origIndex, false)
+              }
+              // Close this span (temporary close, no remove button)
+              formattedText += buildCloseSpan(event.ent, event.origIndex, false)
+              // Reopen inner spans (in the same order) so text after this position is inside them
+              for (let j = index + 1; j < activeEnts.length; j++) {
+                const innerData = activeEnts[j]
+                formattedText += buildOpenSpan(innerData.ent, innerData.origIndex)
+              }
+            } else {
+              // This is the innermost span at its final end position
+              // Add text then close it with remove button if needed
+              if (event.pos > currentPos) {
+                const textSegment = this.text.slice(currentPos, event.pos)
+                if (textSegment.length > 0) {
+                  formattedText += _.escape(textSegment)
+                }
+                currentPos = event.pos
+              }
+              // Only add remove button when closing at the actual end position
+              formattedText += buildCloseSpan(event.ent, event.origIndex, true)
+            }
+            activeEnts.splice(index, 1)
+          }
         }
       }
 
-      // escape '<' '>' that may be interpreted as start/end tags, escape inserted span tags.
-      // formattedText = formattedText
-      //   .replace(/<(?!\/?span|font-awesome-icon)/g, '&lt')
-      //   .replace(/(?<!<span @click="selectEnt\(\d\d?\d?\d?\)".*"|\/span)>/g, '&gt')
+      // Add remaining text after all events
+      if (currentPos < this.text.length) {
+        const textSegment = this.text.slice(currentPos)
+        if (textSegment.length > 0) {
+          formattedText += _.escape(textSegment)
+        }
+        // Close any remaining active spans (in reverse order)
+        for (let j = activeEnts.length - 1; j >= 0; j--) {
+          const activeData = activeEnts[j]
+          const isInnermost = j === activeEnts.length - 1
+          formattedText += buildCloseSpan(activeData.ent, activeData.origIndex, isInnermost)
+        }
+      }
 
       formattedText = this.addAnnos ? `<div @contextmenu.prevent.stop="showCtxMenu($event)">${formattedText}</div>` : `<div>${formattedText}</div>`
       this.scrollIntoView(timeout)
@@ -238,18 +342,64 @@ export default {
   box-shadow: 0px -2px 3px 2px rgba(0, 0, 0, 0.2);
   padding: 25px;
   white-space: pre-wrap;
+  line-height: 1.6; // Base line height for normal text
+
+  // Increase line height when there are 3 or more nested underlines
+  // to prevent underlines from overlapping with next line
+  [class^="highlight-task-"] [class^="highlight-task-"] [class^="highlight-task-"] {
+    line-height: 2.2; // Increased line height for 3+ levels of nesting
+    padding-bottom: 4px; // Extra padding to push next line down
+    display: inline-block; // Ensure padding applies
+  }
+
+  // Also handle when default is deeply nested
+  .highlight-task-default [class^="highlight-task-"] [class^="highlight-task-"] {
+    line-height: 2.2;
+    padding-bottom: 4px;
+    display: inline-block;
+  }
 }
 
 .highlight-task-default {
-  background: lightgrey;
-  border: 1px solid lightgrey;
-  border-radius: 3px;
+  text-decoration: underline;
+  text-decoration-color: lightgrey;
+  text-decoration-thickness: 3px;
+  text-underline-offset: 3px; // Moved down 1px to avoid descender breaks
   cursor: pointer;
+
+  // Stack underlines when nested - each nested level gets a larger offset with clear spacing
+  [class^="highlight-task-"] {
+    text-underline-offset: 7px; // Second level underline (4px spacing from first, moved down 1px)
+  }
+
+  [class^="highlight-task-"] [class^="highlight-task-"] {
+    text-underline-offset: 11px; // Third level underline (4px spacing from second, moved down 1px)
+    // Increase line height for 3+ levels to prevent overlap with next line
+    line-height: 2.2;
+    padding-bottom: 4px;
+    display: inline-block;
+  }
+
+  [class^="highlight-task-"] [class^="highlight-task-"] [class^="highlight-task-"] {
+    text-underline-offset: 15px; // Fourth level underline (4px spacing from third, moved down 1px)
+    // Further increase line height for 4+ levels
+    line-height: 2.4;
+    padding-bottom: 6px;
+    display: inline-block;
+  }
 }
 
 .highlight-task-selected {
-  font-weight: bold;
-  font-size: 1.15rem;
+  // Background highlight is applied via the specific highlight-task-{i} class
+  // This ensures the background color matches the state color
+  text-decoration-thickness: 4px;
+}
+
+// Selected state for default (unvalidated) annotations
+.highlight-task-default.highlight-task-selected {
+  background-color: rgba(211, 211, 211, 0.3); // Light grey background for selected default
+  padding: 1px 2px;
+  border-radius: 2px;
 }
 
 .current-rel-start {

medcat-trainer/webapp/frontend/src/styles/_common.scss

@@ -46,31 +46,62 @@ $blur-radius: 10px;
 .title {
   padding: 5px 15px;
   font-size: 14pt;
-  box-shadow: 0 5px 5px -5px rgba(0,0,0,0.2);
+  box-shadow: 0 5px 5px -5px rgba(0, 0, 0, 0.2);
   color: black;
   height: $title-height;
 }
 
 @each $i, $col in $task-colors {
   .highlight-task-#{$i} {
-    background-color: $col;
-    border-radius: 3px;
+    text-decoration: underline;
+    text-decoration-color: $col;
+    text-decoration-thickness: 3px;
+    text-underline-offset: 3px; // Moved down 1px to avoid descender breaks
     cursor: pointer;
-    border: 1px solid $col;
-    color: white;
+    color: inherit; // Keep original text color
+
+    // When selected, add background highlight with state color
+    &.highlight-task-selected {
+      background-color: $col;
+      padding: 1px 2px;
+      border-radius: 2px;
+      color: white;
+    }
+
+    // Stack underlines when nested - each nested level gets a larger offset with clear spacing
+    [class^="highlight-task-"] {
+      text-underline-offset: 7px; // Second level underline (4px spacing from first, moved down 1px)
+    }
+
+    [class^="highlight-task-"] [class^="highlight-task-"] {
+      text-underline-offset: 11px; // Third level underline (4px spacing from second, moved down 1px)
+      // Increase line height for 3+ levels to prevent overlap with next line
+      line-height: 2.2;
+      padding-bottom: 4px;
+      display: inline-block;
+    }
+
+    [class^="highlight-task-"] [class^="highlight-task-"] [class^="highlight-task-"] {
+      text-underline-offset: 15px; // Fourth level underline (4px spacing from third, moved down 1px)
+      // Further increase line height for 4+ levels
+      line-height: 2.4;
+      padding-bottom: 6px;
+      display: inline-block;
+    }
   }
 }
 
-.alert-enter-active, .alert-leave-active {
+.alert-enter-active,
+.alert-leave-active {
   transition: opacity .5s;
 }
 
-.alert-enter, .alert-leave-to {
+.alert-enter,
+.alert-leave-to {
   opacity: 0;
 }
 
 .overlay-message {
   padding-left: 10px;
   opacity: 0.5;
-}
-
+}