Skip to content

Commit 8293cd5

Browse files
cmpajotcmpajot
authored
Improve Categorization API Error Messages (#182)
* Passing sync tests * Added async integration tests * Updated error messages
1 parent 9da55b5 commit 8293cd5

File tree

4 files changed

+120
-5
lines changed

4 files changed

+120
-5
lines changed

‎nucleus/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,7 @@ def annotate_dataset(
468468
DATASET_ID_KEY: dataset_id,
469469
ANNOTATIONS_PROCESSED_KEY: 0,
470470
ANNOTATIONS_IGNORED_KEY: 0,
471+
ERRORS_KEY: [],
471472
}
472473

473474
total_batches = len(batches) + len(semseg_batches)
@@ -490,6 +491,7 @@ def annotate_dataset(
490491
agg_response[ANNOTATIONS_IGNORED_KEY] += response[
491492
ANNOTATIONS_IGNORED_KEY
492493
]
494+
agg_response[ERRORS_KEY] += response[ERRORS_KEY]
493495

494496
for s_batch in semseg_batches:
495497
payload = construct_segmentation_payload(s_batch, update)
@@ -628,6 +630,8 @@ def predict(
628630
else:
629631
predictions_processed += response[PREDICTIONS_PROCESSED_KEY]
630632
predictions_ignored += response[PREDICTIONS_IGNORED_KEY]
633+
if ERRORS_KEY in response:
634+
errors += response[ERRORS_KEY]
631635

632636
for s_batch in s_batches:
633637
payload = construct_segmentation_payload(s_batch, update)

‎tests/helpers.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,14 @@ def reference_id_from_url(url):
180180
for i in range(len(TEST_IMG_URLS))
181181
]
182182

183+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_ANNOTATION = [
184+
{
185+
"label": "[Pytest] Category Label 0",
186+
"reference_id": reference_id_from_url(TEST_IMG_URLS[0]),
187+
"taxonomy_name": "[Pytest] Category Taxonomy Nonexistent",
188+
}
189+
]
190+
183191
TEST_MULTICATEGORY_ANNOTATIONS = [
184192
{
185193
"labels": [
@@ -290,6 +298,15 @@ def reference_id_from_url(url):
290298
for i in range(len(TEST_DEFAULT_CATEGORY_ANNOTATIONS))
291299
]
292300

301+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_PREDICTION = [
302+
{
303+
"label": "[Pytest] Category Label 0",
304+
"reference_id": reference_id_from_url(TEST_IMG_URLS[0]),
305+
"taxonomy_name": "[Pytest] Category Taxonomy Nonexistent",
306+
"confidence": 0.10,
307+
}
308+
]
309+
293310
TEST_INDEX_EMBEDDINGS_FILE = "https://raw.githubusercontent.com/scaleapi/nucleus-python-client/master/tests/testdata/pytest_embeddings_payload.json"
294311

295312

‎tests/test_annotation.py‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
SegmentationAnnotation,
1212
)
1313
from nucleus.constants import ERROR_PAYLOAD
14-
from nucleus.job import AsyncJob
14+
from nucleus.job import AsyncJob, JobError
1515

1616
from .helpers import (
1717
TEST_BOX_ANNOTATIONS,
@@ -21,6 +21,7 @@
2121
TEST_DEFAULT_MULTICATEGORY_ANNOTATIONS,
2222
TEST_IMG_URLS,
2323
TEST_MULTICATEGORY_ANNOTATIONS,
24+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_ANNOTATION,
2425
TEST_POLYGON_ANNOTATIONS,
2526
TEST_SEGMENTATION_ANNOTATIONS,
2627
assert_box_annotation_matches_dict,
@@ -162,6 +163,21 @@ def test_default_category_gt_upload(dataset):
162163
)
163164

164165

166+
def test_non_existent_taxonomy_category_gt_upload(dataset):
167+
annotation = CategoryAnnotation.from_json(
168+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_ANNOTATION[0]
169+
)
170+
response = dataset.annotate(annotations=[annotation])
171+
172+
assert response["dataset_id"] == dataset.id
173+
assert response["annotations_processed"] == 0
174+
assert response["annotations_ignored"] == 0
175+
assert (
176+
f'Input validation failed: Taxonomy {TEST_NONEXISTENT_TAXONOMY_CATEGORY_ANNOTATION[0]["taxonomy_name"]} does not exist in dataset {dataset.id}'
177+
in response["errors"][0]
178+
)
179+
180+
165181
def test_multicategory_gt_upload(dataset):
166182
annotation = MultiCategoryAnnotation.from_json(
167183
TEST_MULTICATEGORY_ANNOTATIONS[0]
@@ -745,3 +761,35 @@ def test_default_category_gt_upload_async(dataset):
745761
"completed_steps": 1,
746762
"total_steps": 1,
747763
}
764+
765+
766+
@pytest.mark.integration
767+
def test_non_existent_taxonomy_category_gt_upload_async(dataset):
768+
annotation = CategoryAnnotation.from_json(
769+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_ANNOTATION[0]
770+
)
771+
772+
try:
773+
job: AsyncJob = dataset.annotate(
774+
annotations=[
775+
annotation,
776+
],
777+
asynchronous=True,
778+
)
779+
job.sleep_until_complete()
780+
except JobError:
781+
assert (
782+
f'Input validation failed: Taxonomy {TEST_NONEXISTENT_TAXONOMY_CATEGORY_ANNOTATION[0]["taxonomy_name"]} does not exist in dataset {dataset.id}'
783+
in job.errors()[-1]
784+
)
785+
786+
assert job.status() == {
787+
"job_id": job.job_id,
788+
"status": "Errored",
789+
"message": {
790+
"status_log": "No additional information can be provided at this time."
791+
},
792+
"job_progress": "0.00",
793+
"completed_steps": 0,
794+
"total_steps": 1,
795+
}

‎tests/test_prediction.py‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
SegmentationPrediction,
1515
)
1616
from nucleus.constants import ERROR_PAYLOAD
17-
from nucleus.job import AsyncJob
17+
from nucleus.job import AsyncJob, JobError
1818

1919
from .helpers import (
2020
TEST_BOX_PREDICTIONS,
@@ -24,6 +24,7 @@
2424
TEST_IMG_URLS,
2525
TEST_MODEL_NAME,
2626
TEST_MODEL_RUN,
27+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_PREDICTION,
2728
TEST_POLYGON_PREDICTIONS,
2829
TEST_SEGMENTATION_PREDICTIONS,
2930
assert_box_prediction_matches_dict,
@@ -119,7 +120,7 @@ def test_polygon_pred_upload(model_run):
119120
response = model_run.predict(annotations=[prediction])
120121

121122
assert response["model_run_id"] == model_run.model_run_id
122-
assert response["predictions_ignored"] == 0
123+
assert response["predictions_processed"] == 1
123124
assert response["predictions_ignored"] == 0
124125

125126
response = model_run.refloc(prediction.reference_id)["polygon"]
@@ -134,7 +135,7 @@ def test_category_pred_upload(model_run):
134135
response = model_run.predict(annotations=[prediction])
135136

136137
assert response["model_run_id"] == model_run.model_run_id
137-
assert response["predictions_ignored"] == 0
138+
assert response["predictions_processed"] == 1
138139
assert response["predictions_ignored"] == 0
139140

140141
response = model_run.refloc(prediction.reference_id)["category"]
@@ -151,7 +152,7 @@ def test_default_category_pred_upload(model_run):
151152
response = model_run.predict(annotations=[prediction])
152153

153154
assert response["model_run_id"] == model_run.model_run_id
154-
assert response["predictions_ignored"] == 0
155+
assert response["predictions_processed"] == 1
155156
assert response["predictions_ignored"] == 0
156157

157158
response = model_run.refloc(prediction.reference_id)["category"]
@@ -161,6 +162,20 @@ def test_default_category_pred_upload(model_run):
161162
)
162163

163164

165+
def test_non_existent_taxonomy_category_gt_upload(model_run):
166+
prediction = CategoryPrediction.from_json(
167+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_PREDICTION[0]
168+
)
169+
response = model_run.predict(annotations=[prediction])
170+
assert response["model_run_id"] == model_run.model_run_id
171+
assert response["predictions_processed"] == 0
172+
assert response["predictions_ignored"] == 0
173+
assert (
174+
f'Input validation failed: Taxonomy {TEST_NONEXISTENT_TAXONOMY_CATEGORY_PREDICTION[0]["taxonomy_name"]} does not exist in dataset'
175+
in response["errors"][0]
176+
)
177+
178+
164179
def test_segmentation_pred_upload(model_run):
165180
prediction = SegmentationPrediction.from_json(
166181
TEST_SEGMENTATION_PREDICTIONS[0]
@@ -576,3 +591,34 @@ def test_default_category_pred_upload_async(model_run: ModelRun):
576591
"completed_steps": 1,
577592
"total_steps": 1,
578593
}
594+
595+
596+
@pytest.mark.integration
597+
def test_non_existent_taxonomy_category_gt_upload_async(model_run: ModelRun):
598+
prediction = CategoryPrediction.from_json(
599+
TEST_NONEXISTENT_TAXONOMY_CATEGORY_PREDICTION[0]
600+
)
601+
try:
602+
job: AsyncJob = model_run.predict(
603+
annotations=[
604+
prediction,
605+
],
606+
asynchronous=True,
607+
)
608+
job.sleep_until_complete()
609+
except JobError:
610+
assert (
611+
f'Input validation failed: Taxonomy {TEST_NONEXISTENT_TAXONOMY_CATEGORY_PREDICTION[0]["taxonomy_name"]} does not exist in dataset'
612+
in job.errors()[-1]
613+
)
614+
615+
assert job.status() == {
616+
"job_id": job.job_id,
617+
"status": "Errored",
618+
"message": {
619+
"status_log": "No additional information can be provided at this time."
620+
},
621+
"job_progress": "0.00",
622+
"completed_steps": 0,
623+
"total_steps": 1,
624+
}

0 commit comments

Comments
 (0)