All Adjudications
Total Complete
- {{ all_anns_frac }}
-
+ {{ progress }} +
- + {{ progress }} +
-
+ {% endif %} +
+ {% endif %} + + {% if conflicts.count > 0 %} +
Search for record
- {% if messages %} - + + {% if bookmarked.count > 0 %} +
+
+ {% if page_info.0 %}
+ Bookmarked Conflicts
+
+ {% if page_info.0.has_previous %}
+ Previous
+ Page
+ {% endif %}
+ {% if page_info.0.has_next %}
+ Next
+ Page
+ {% endif %}
+ {% if request.GET.bookmarked_page == 'all' %}
+ Show pages
{% endif %}
-
- {% if search %}
- {% for k,info in search.items %}
- {% if k == "com" %}
- {% for batch in info %}
-
- {% endfor %} - {% elif k == "inc" %} - {% for batch in info %} -
- {% endfor %} - {% endif %} + {% if page_info.0.has_next or page_info.0.has_previous %} + Show all + {% endif %} +
+ {% endif %}
+
-
-
-
-
- | - {{ cat }} - | - {% endfor %} - -- Edit adjudication - | -- Delete adjudication - | -
|---|---|---|
| - {{ v }} - | - {% endfor %} - {% else %} - {% for v in val|slice:"2:" %} -- {{ v }} - | - {% endfor %} - {% endif %} -
- {% endfor %} - {% elif k == "inc" %} - {% for batch in info %} -
-
-
-
-
- | - {{ cat }} - | - {% endfor %} -- Edit adjudication - | -
|---|---|
| - {{ v }} - | - {% endfor %} -
- {% endfor %} - {% endif %} + {% if page_info.0.has_next or page_info.0.has_previous %} + Show all + {% endif %} +
+ {% for waveform in bookmarked%}
+ {% ifchanged waveform.project waveform.record %}
+ {% if not forloop.first %}
+
+ + {% endif %} +
+
+ {% endifchanged %} + +
+
+
+ {% endfor %} +
+ {{ waveform.project}}: {{ waveform.record }}
+ {% endifchanged %} + +
{{ waveform.event }} (View + waveform | Delete Bookmark + )
+ +| + {{ cat }} + | + {% endfor %} +|||
|---|---|---|---|
| + {{ annotation.user.username }} + | ++ {{ annotation.decision}} + | ++ {{ annotation.decision_date }} + | ++ {{ annotation.comments }} + | +
+ {% endfor %} +
+ {% endif %} + + {% if conflicts.count > 0 %} +
+
+ {% if page_info.1 %}
+ Conflicting Annotations
+
+ {% if page_info.1.has_previous %}
+ Previous
+ Page
+ {% endif %}
+ {% if page_info.1.has_next %}
+ Next
+ Page
+ {% endif %}
+ {% if request.GET.conflicts_page == 'all' %}
+ Show pages
+ {% endif %}
+ {% if page_info.1.has_next or page_info.1.has_previous %}
+ Show all
{% endif %}
-
-
{% endif %} - {% for batch in complete_adjudications %} -
+ {% endif %} + {% if adjudicated.count > 0 %} +
{% endif %} - {% for batch in incomplete_adjudications %} -
+
+{% endblock %}
\ No newline at end of file
diff --git a/waveform-django/waveforms/templates/waveforms/adjudicator_console.html b/waveform-django/waveforms/templates/waveforms/adjudicator_console.html
deleted file mode 100644
index e9c0e15..0000000
--- a/waveform-django/waveforms/templates/waveforms/adjudicator_console.html
+++ /dev/null
@@ -1,66 +0,0 @@
-{% extends "base.html" %}
-{% load static %}
-{% load plotly_dash %}
-{% block content %}
-
-
-
-Complete Adjudications
+ {% endif %} +
+ {% for waveform in conflicts%}
+ {% ifchanged waveform.project waveform.record %}
+ {% if not forloop.first %}
- {% if complete_page %}
-
- {% if complete_page.has_previous %}
- Previous Page
- {% endif %}
- {% if complete_page.has_next %}
- Next Page
- {% endif %}
- {% if request.GET %}
- {% if request.GET.complete_page == 'all' %}
- Show pages
- {% else %}
- {% if n_complete > 5 %}
- Show all
- {% endif %}
- {% endif %}
- {% else %}
- {% if n_complete > 5 %}
- Show all
- {% endif %}
- {% endif %}
-
+ {% endif %} - {% for batch in complete_adjudications %} -
-
-
-
- {% endfor %} -
-
+
+ {% endfor %} +
+ | - {{ cat }} - | - {% endfor %} - -- Edit adjudication - | -- Delete adjudication - | -|
|---|---|---|---|
| - {{ v }} - | - {% endfor %} - {% else %} - {% for v in val|slice:"2:" %} -- {{ v }} - | - {% endfor %} - {% endif %} -
| + {{ cat }} + | {% endfor %} -
|---|
- {% endfor %} -
-
-
+
+ {{ annotation.user.username }}
+
+
+ {{ annotation.decision}}
+
+
+ {{ annotation.decision_date }}
+
+
+ {{ annotation.comments }}
+
+
+ {% endfor %}
+
Incomplete Adjudications
+ + + {% for annotation in waveform.annotation_set.all %} ++ {% endfor %} +
+ {% endif %} + {% if adjudicated.count > 0 %} +
+
+ {% if page_info.2 %}
+ Adjudicated Waveforms
+
+ {% if page_info.2.has_previous %}
+ Previous
+ Page
+ {% endif %}
+ {% if page_info.2.has_next %}
+ Next
+ Page
+ {% endif %}
+ {% if request.GET.adjudicated_page == 'all' %}
+ Show pages
+ {% endif %}
+ {% if page_info.2.has_next or page_info.2.has_previous %}
+ Show all
+ {% endif %}
+
+ {% endif %}
+
+ {% for waveform in adjudicated%}
+ {% ifchanged waveform.project waveform.record %}
+ {% if not forloop.first %}
- {% if incomplete_page %}
-
- {% if incomplete_page.has_previous %}
- Previous Page
- {% endif %}
- {% if incomplete_page.has_next %}
- Next Page
- {% endif %}
- {% if request.GET %}
- {% if request.GET.incomplete_page == 'all' %}
- Show pages
- {% else %}
- {% if n_incomplete > 5 %}
- Show all
- {% endif %}
- {% endif %}
- {% else %}
- {% if n_incomplete > 5 %}
- Show all
- {% endif %}
- {% endif %}
-
+ {% endif %} - {% for batch in incomplete_adjudications %} -
-
-
-
- {% endfor %} - -
+ -{% endblock %} + {% for annotation in waveform.annotation_set.all %} +
+
+ {{ annotation.user.username }}
+
+
+ {{ annotation.decision}}
+
+
+ {{ annotation.decision_date }}
+
+
+ {{ annotation.comments }}
+
+
+ {% endfor %}
+
+
+ {% endfor %} +
+ {% endif %}
+ | - {{ cat }} - | - {% endfor %} -- Edit adjudication - | -||
|---|---|---|---|
| - {{ v }} - | - {% endfor %} -
| + {{ cat }} + | {% endfor %} -
|---|
- {% endfor %} - -
+ -{% endblock %} + {% for annotation in waveform.annotation_set.all %} +
+ {% endfor %} +
+
-
-
-
-{% endblock %}
diff --git a/waveform-django/waveforms/templates/waveforms/admin_console.html b/waveform-django/waveforms/templates/waveforms/admin_console.html
index 7b52a09..b253c5e 100644
--- a/waveform-django/waveforms/templates/waveforms/admin_console.html
+++ b/waveform-django/waveforms/templates/waveforms/admin_console.html
@@ -14,79 +14,79 @@ Adjudicator Console
- - - - -
- {% plotly_app_bootstrap name='waveform_graph_adjudicate' initial_arguments=dash_context %}
-
-Admin Console
All Users
{% if messages %}
- {% for message in messages %}
- {% if message.level == DEFAULT_MESSAGE_LEVELS.ERROR %}
-
- {% elif message.level == DEFAULT_MESSAGE_LEVELS.INFO %}
-
+ {% for message in messages %}
+ {% if message.level == DEFAULT_MESSAGE_LEVELS.ERROR %}
+
+ {% elif message.level == DEFAULT_MESSAGE_LEVELS.INFO %}
+
+{% endif %} +{% endif %} + -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/waveform-django/waveforms/templates/waveforms/assessment_info.html b/waveform-django/waveforms/templates/waveforms/assessment_info.html new file mode 100644 index 0000000..62e0734 --- /dev/null +++ b/waveform-django/waveforms/templates/waveforms/assessment_info.html @@ -0,0 +1,66 @@ +{% extends "base.html" %} +{% load static %} + +{% block content %} +
{% else %}
-
- {% endif %}
+
+
+ {% endif %}
{{ message|safe }}
- {% endfor %}
- {% endif %}
- Invite a new user by email address
-+{% endif %} +{% endif %} + -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/waveform-django/waveforms/templates/waveforms/assessment_info.html b/waveform-django/waveforms/templates/waveforms/assessment_info.html new file mode 100644 index 0000000..62e0734 --- /dev/null +++ b/waveform-django/waveforms/templates/waveforms/assessment_info.html @@ -0,0 +1,66 @@ +{% extends "base.html" %} +{% load static %} + +{% block content %} +
+
+
+{% endblock %}
diff --git a/waveform-django/waveforms/templates/waveforms/assessment_result.html b/waveform-django/waveforms/templates/waveforms/assessment_result.html
new file mode 100755
index 0000000..b9234a5
--- /dev/null
+++ b/waveform-django/waveforms/templates/waveforms/assessment_result.html
@@ -0,0 +1,49 @@
+{% extends "base.html" %}
+{% load static %}
+{% block content %}
+Entry Assessment
+ +
+
+
+ Instructions
+ +-
+
- You have been assigned a set of {{ total }} waveform events to annotate, which can be accessed on the View All Assessment Samples page. +
- This assessment will not be timed, but the staff will see the results of this assessment. +
- You may return to this page at any time. +
- When you are ready to submit your exam, simply press the "Submit Assessment" button below. Your responses will be sent to the administrators. +
- You cannot change your answers once you submit. +
+
+
+ {% if user.practice_status == "CO" %}
+ {% if user.is_annotator == True %}
+ Results
+ {% else %} +Responses
+ {% endif %} + {% for project,events in results.items %} + {% for event,responses in events.items %} ++ {{ project }} {{ event }} Your answer: {{ responses.1 }} + {% if user.is_annotator %} + Correct answer: {{ responses.0 }} + {% if responses.0 == responses.1 %} ✔️ {% else %} ❌ {% endif %} + {% endif %} +
+ {% endfor %} + {% endfor %} + {% if user.is_annotator %} +Score: {{ correct }}/{{ total }}
+ {% else %} + Score is hidden until the staff review your responses + {% endif %} + {% endif %} +
+
+
+
+
+
+
+
+
+
+
+ {% if annotator.is_annotator %} +
+
+{% endblock %}
diff --git a/waveform-django/waveforms/templates/waveforms/overview.html b/waveform-django/waveforms/templates/waveforms/overview.html
new file mode 100755
index 0000000..16d3c96
--- /dev/null
+++ b/waveform-django/waveforms/templates/waveforms/overview.html
@@ -0,0 +1,79 @@
+{% extends "base.html" %}
+{% load static %}
+
+{% block content %}
+Assessment Results
++
Username: {{annotator.username}}
++
Responses:
+| Project | +Event | +Correct Label | +Response | +
|---|---|---|---|
| {{ project }} | +{{ event }} | +{{ responses.1 }} | +{{ responses.2 }} {% if responses.1 == responses.2 %} ✔️ {% else %} ❌ {% endif %} | +
+
Total Score: {{ correct }} / {{ total }}
++ {% if annotator.is_annotator %} +
+
+
+ {% else %}
+
+
+
+ {% endif %}
+
+
+
+
+
+
+
+{% endblock %}
diff --git a/waveform-django/waveforms/templates/waveforms/practice.html b/waveform-django/waveforms/templates/waveforms/practice.html
index 67228b3..c5eb45a 100755
--- a/waveform-django/waveforms/templates/waveforms/practice.html
+++ b/waveform-django/waveforms/templates/waveforms/practice.html
@@ -3,41 +3,77 @@
{% block content %}
Project Overview
++
Thank you for offering to volunteer your time and expertise to help us build our database!
++
Objective
+
+
+ + Our goal is to build a large dataset of patient-monitoring data, where ECG samples that show signs of Ventricular Tachycardia (VT) have been classified by experts. +
-
+
- Several commercial heart-monitor companies have provided us with data captured from patient monitors +
- This data represents instances where the vendor's VT detection algorithm was triggered +
- Commercial VT detection algorithms are often unreliable, as they are easily triggered without the occurance of a VT event +
- Hospital staff often have to deal with false-positive alarms, which can desensitize them to the alarms +
- Desensitization may lead to delayed responses when a patient is suffering from a true VT event, which can significantly impact their chances of receiving appropriate care in a timely fashion +
- The overabundance of alarms has also shown to increase the stress experienced by patients and staff in the ICU +
- There is a great need to improve the VT detection abilities of bedside monitors +
- We aim to employ a Machine-Learning approach to developing a VT detection algorithm +
- To begin this process, we first need a database to train and test our ML model on +
- Each data sample must be labeled based on the presence of a VT event by experts +
- This platform allows experts to view and label data samples in an effort to build our database +
-
+
-
+
+
Getting Started
+
+
+ Initial Assessment
++ Before you can begin annotating events, we require all participants to take an evaluation on their ability to accurately identify VT. +
-
+
- We have created an assessment that consists of a set of events that our experts have already labeled +
- New members are asked to view these events and label them to the best of their abilities +
- These labels will be compared to the correct labels, and the assessment will be scored for accuracy +
- After submitting the assessment, your results will be sent to our team +
- If your score is approved by our team, you will be given access to our unlabeled data, and you may proceed to assign yourself events to annotate + {% if user.is_annotator %} +
- Our records indicate that you have completed this assessment, and have been approved to annotate new events + {% else %} + {% if user.practice_status == "CO" %} +
- Our records indicate that you have completed this assessment and are awaiting evaluation by our team. Please check back at a later time! + {% else %} +
- Our records indicate that you have not completed this assessment. Please go to the Assessment Info page to get started. + {% endif %} + {% endif %} +
-
+
+
+
+Annotating Events
++
-
+
- To begin annotating events, you must first assign yourself a set of events you plan to annotate +
- You can do this on the Current Assignment page +
- You may begin annotating events by going to the Create Annotation page +
- The Current Assignment page will show you all of your assigned events +
- This includes annotated events, unannotated events, and events that you have saved for later +
-
+
-
+
-
-
Answer practice questions
+ + {% if user.is_annotator == False%} +Entry Assessment
+ {% else %} +Answer practice questions
+ {% endif %}Instructions
--
-
- You will be assigned a set of {{ total }} waveform events to complete. -
- You can come return to this page whenever you are ready to view the answers. -
- This assessment will not be timed or graded, and may be exited at any time by pressing the "End Practice" button below. -
- When you wish to continue working on the main dataset, simply press the "End Practice" button below. You will resume working on your active assignment. -
- You may take the practice test as many times as you wish, but your score will be lost after you end the practice session. + + {% if user.is_annotator == False%} +
- You have been assigned a set of {{ total }} waveform events to annotate, which can be accessed on the "View All Exam Samples" Page. +
- This assessment will not be timed, but the staff will see the results of this assessment. +
- You may return to this page at any time. +
- When you are ready to submit your exam, simply press the "Submit Assessment" button below. Your responses will be sent to the administrators. +
- You cannot change your answers once you submit. +
- You will be assigned a set of {{ total }} waveform events to complete. +
- This assessment will not be timed or graded, and may be exited at any time by pressing the "End Practice" button below. +
- You may return to this page whenever you are ready to view the answers. +
- When you wish to continue working on the main dataset, simply press the "End Practice" button below. You will resume working on your active assignment. +
- You may take the practice test as many times as you wish, but your score will be lost after you end the practice session.
- After logging in, you should see your username with the option to logout in the top left corner. -
- Below that, you should see links to view your current annotations, view a brief tutorial explaining the layout of the annotator (this page), and a way to change the settings / preferences of your annotator (including grid color, time before and after an event, the level of downsampling, etc.). +
- Below that, you should see links to view your current annotations, view a brief tutorial explaining the layout of the annotator (this page), and a way to change the settings / preferences of your annotator (including grid color, time before and after an event, the level of downsampling, etc.).
- To begin annotating, go to the annotator home and scroll down to view the entire annotator.
- On the top left side, you should see the name of the current annotation record, event, and label (It has been filtered so only VTach and VFib/VFlutter events are displayed. Please let us know if you see otherwise!).
- Directly below this is where you can enter your decision for the label (True, False, Uncertain, Reject, or Save for Later) as well as your additional comments explaining your decision (this field is not necessary but may prove useful later on for either self-reference or research purposes).
- You can submit the annotation with the "Submit" button, or go to the next or previous annotation without saving using the arrows. Submitting the annotation will automatically send you to the next event.
- If you hover over the signals, you will be able to see its value and how it aligns with the other signals.
- Also, you can click and drag the signal in any direction to view more of it. -
- At any point, you can view all of your previous annotations with the "View current annotations" link and either edit or delete them if needed. You will also be able to see your progress on the annotations using the fractions on the far right side. Also note that if you reload the annotator home page at any point, you will be forwarded to your first non-annotated event. +
- At any point, you can view all of your previous annotations with the "View current annotations" link and either edit or delete them if needed. You will also be able to see your progress on the annotations using the fractions on the far right side. Also note that if you reload the annotator home page at any point, you will be forwarded to your first non-annotated event.
- You can also adjust the settings for the annotator using the "Change annotator settings" link; for example, you can reduce the downsampling to view more detail on the EKG signals if needed.
-
+
+ {% else %}
+
-
+
+ {% endif %}
+
+
+
+
+
{% if user.practice_status == "CO" %}
- Results
- {% for project,events in results.items %} - {% for event,responses in events.items %} -- {{ project }} {{ event }} Correct answer: {{ responses.0 }} Your answer: {{ responses.1 }} - {% if responses.0 == responses.1 %} ✔️ {% else %} ❌ {% endif %} -
- {% endfor %} + {% if user.is_annotator == True %} +Results
+ {% else %} +Responses
+ {% endif %} + {% for response in results %} ++ {{ response.0 }} | {{ response.1 }} | {{ response.2 }} Your answer: {{ response.4 }} + {% if user.is_annotator %} + Correct answer: {{ response.3 }} + {% if response.3 == response.4 %} ✔️ {% else %} ❌ {% endif %} + {% endif %} +
{% endfor %} + {% if user.is_annotator %}Score: {{ correct }}/{{ total }}
+ {% else %} + Score is hidden until the staff review your responses + {% endif %} {% endif %}