From 8ff2bc5dff2c73352be6c9ae64d97c48f889869b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 11 Aug 2025 10:58:04 -0400 Subject: [PATCH 1/7] Introduce JSONEmpty lookup --- netbox/extras/lookups.py | 28 +++++++++++++++++++++++++++- netbox/extras/models/customfields.py | 6 +++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index 9e1fe4a0b6e..a43c0150810 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,4 +1,5 @@ -from django.db.models import CharField, Lookup +from django.db.models import CharField, JSONField, Lookup +from django.db.models.fields.json import KeyTextTransform from .fields import CachedValueField @@ -18,6 +19,30 @@ def as_sql(self, compiler, connection): return f"CAST(LENGTH({sql}) AS BOOLEAN) IS TRUE", params +class JSONEmpty(Lookup): + """ + Support "empty" lookups for JSONField keys. + + A key is considered empty if it is "", null, or does not exist. + """ + lookup_name = "empty" + + def as_sql(self, compiler, connection): + # self.lhs.lhs is the parent expression (could be a JSONField or another KeyTransform) + # Rebuild the expression using KeyTextTransform to guarantee ->> (text) + text_expr = KeyTextTransform(self.lhs.key_name, self.lhs.lhs) + lhs_sql, lhs_params = compiler.compile(text_expr) + + value = self.rhs + if value not in (True, False): + raise ValueError("The 'empty' lookup only accepts True or False.") + + condition = 'NOT ' if value else '' + sql = f"(NULLIF({lhs_sql}, '') IS {condition}NULL)" + + return sql, lhs_params + + class NetHost(Lookup): """ Similar to ipam.lookups.NetHost, but casts the field to INET. @@ -45,5 +70,6 @@ def as_sql(self, qn, connection): CharField.register_lookup(Empty) +JSONField.register_lookup(JSONEmpty) CachedValueField.register_lookup(NetHost) CachedValueField.register_lookup(NetContainsOrEquals) diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index aeeb1572895..cabaeada859 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -603,8 +603,12 @@ def to_filter(self, lookup_expr=None): if lookup_expr is not None: kwargs['lookup_expr'] = lookup_expr + # 'Empty' lookup is always a boolean + if lookup_expr == 'empty': + filter_class = django_filters.BooleanFilter + # Text/URL - if self.type in ( + elif self.type in ( CustomFieldTypeChoices.TYPE_TEXT, CustomFieldTypeChoices.TYPE_LONGTEXT, CustomFieldTypeChoices.TYPE_URL, From 78b55b8515cfc35065e27d029c52d3cf091a5c7a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 11 Aug 2025 13:49:42 -0400 Subject: [PATCH 2/7] Fix inverted logic --- netbox/extras/lookups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index a43c0150810..33296340ec2 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -37,7 +37,7 @@ def as_sql(self, compiler, connection): if value not in (True, False): raise ValueError("The 'empty' lookup only accepts True or False.") - condition = 'NOT ' if value else '' + condition = '' if value else 'NOT ' sql = f"(NULLIF({lhs_sql}, '') IS {condition}NULL)" return sql, lhs_params From b38a6116227c2162586007506007f63461b97c7f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 11 Aug 2025 15:11:20 -0400 Subject: [PATCH 3/7] Use empty lookup for numeric custom fields instead of isnull --- netbox/utilities/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index 9f027f8e16d..1ad0d484756 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -23,7 +23,7 @@ lt='lt', gte='gte', gt='gt', - empty='isnull', + empty='empty', ) FILTER_NEGATION_LOOKUP_MAP = dict( From a821981d276b9986293ada9c6f106ddfef42dd5e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 11 Aug 2025 15:11:52 -0400 Subject: [PATCH 4/7] Add custom field tests for empty lookup --- netbox/extras/tests/test_customfields.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 8bae8dfc958..8da8fcbf144 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -1615,6 +1615,7 @@ def setUpTestData(cls): 'cf11': manufacturers[2].pk, 'cf12': [manufacturers[2].pk, manufacturers[3].pk], }), + Site(name='Site 4', slug='site-4'), ]) def test_filter_integer(self): @@ -1624,6 +1625,7 @@ def test_filter_integer(self): self.assertEqual(self.filterset({'cf_cf1__gte': [200]}, self.queryset).qs.count(), 2) self.assertEqual(self.filterset({'cf_cf1__lt': [200]}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf1__lte': [200]}, self.queryset).qs.count(), 2) + self.assertEqual(self.filterset({'cf_cf1__empty': True}, self.queryset).qs.count(), 1) def test_filter_decimal(self): self.assertEqual(self.filterset({'cf_cf2': [100.1, 200.2]}, self.queryset).qs.count(), 2) @@ -1632,6 +1634,7 @@ def test_filter_decimal(self): self.assertEqual(self.filterset({'cf_cf2__gte': [200.2]}, self.queryset).qs.count(), 2) self.assertEqual(self.filterset({'cf_cf2__lt': [200.2]}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf2__lte': [200.2]}, self.queryset).qs.count(), 2) + self.assertEqual(self.filterset({'cf_cf2__empty': True}, self.queryset).qs.count(), 1) def test_filter_boolean(self): self.assertEqual(self.filterset({'cf_cf3': True}, self.queryset).qs.count(), 2) @@ -1648,6 +1651,7 @@ def test_filter_text_strict(self): self.assertEqual(self.filterset({'cf_cf4__niew': ['bar']}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf4__ie': ['FOO']}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf4__nie': ['FOO']}, self.queryset).qs.count(), 2) + self.assertEqual(self.filterset({'cf_cf4__empty': True}, self.queryset).qs.count(), 1) def test_filter_text_loose(self): self.assertEqual(self.filterset({'cf_cf5': ['foo']}, self.queryset).qs.count(), 2) @@ -1659,6 +1663,7 @@ def test_filter_date(self): self.assertEqual(self.filterset({'cf_cf6__gte': ['2016-06-27']}, self.queryset).qs.count(), 2) self.assertEqual(self.filterset({'cf_cf6__lt': ['2016-06-27']}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf6__lte': ['2016-06-27']}, self.queryset).qs.count(), 2) + self.assertEqual(self.filterset({'cf_cf6__empty': True}, self.queryset).qs.count(), 1) def test_filter_url_strict(self): self.assertEqual( @@ -1674,17 +1679,20 @@ def test_filter_url_strict(self): self.assertEqual(self.filterset({'cf_cf7__niew': ['.com']}, self.queryset).qs.count(), 0) self.assertEqual(self.filterset({'cf_cf7__ie': ['HTTP://A.EXAMPLE.COM']}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf7__nie': ['HTTP://A.EXAMPLE.COM']}, self.queryset).qs.count(), 2) + self.assertEqual(self.filterset({'cf_cf7__empty': True}, self.queryset).qs.count(), 1) def test_filter_url_loose(self): self.assertEqual(self.filterset({'cf_cf8': ['example.com']}, self.queryset).qs.count(), 3) def test_filter_select(self): self.assertEqual(self.filterset({'cf_cf9': ['A', 'B']}, self.queryset).qs.count(), 2) + self.assertEqual(self.filterset({'cf_cf9__empty': True}, self.queryset).qs.count(), 1) def test_filter_multiselect(self): self.assertEqual(self.filterset({'cf_cf10': ['A']}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf10': ['A', 'C']}, self.queryset).qs.count(), 2) self.assertEqual(self.filterset({'cf_cf10': ['null']}, self.queryset).qs.count(), 1) + self.assertEqual(self.filterset({'cf_cf10__empty': True}, self.queryset).qs.count(), 1) # Same as above def test_filter_object(self): manufacturer_ids = Manufacturer.objects.values_list('id', flat=True) @@ -1692,6 +1700,7 @@ def test_filter_object(self): self.filterset({'cf_cf11': [manufacturer_ids[0], manufacturer_ids[1]]}, self.queryset).qs.count(), 2 ) + self.assertEqual(self.filterset({'cf_cf11__empty': True}, self.queryset).qs.count(), 1) def test_filter_multiobject(self): manufacturer_ids = Manufacturer.objects.values_list('id', flat=True) @@ -1703,3 +1712,4 @@ def test_filter_multiobject(self): self.filterset({'cf_cf12': [manufacturer_ids[3]]}, self.queryset).qs.count(), 3 ) + self.assertEqual(self.filterset({'cf_cf12__empty': True}, self.queryset).qs.count(), 1) From 37abb4c8a865379c90e716efb991a5bbdcd32586 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 18 Aug 2025 09:46:57 -0400 Subject: [PATCH 5/7] Fix multiselect and multi-object CF filters --- netbox/extras/tests/test_customfields.py | 4 ++-- netbox/netbox/filtersets.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 8da8fcbf144..c3074aa415f 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -1691,8 +1691,8 @@ def test_filter_select(self): def test_filter_multiselect(self): self.assertEqual(self.filterset({'cf_cf10': ['A']}, self.queryset).qs.count(), 1) self.assertEqual(self.filterset({'cf_cf10': ['A', 'C']}, self.queryset).qs.count(), 2) - self.assertEqual(self.filterset({'cf_cf10': ['null']}, self.queryset).qs.count(), 1) - self.assertEqual(self.filterset({'cf_cf10__empty': True}, self.queryset).qs.count(), 1) # Same as above + self.assertEqual(self.filterset({'cf_cf10': ['null']}, self.queryset).qs.count(), 1) # Contains a literal null + self.assertEqual(self.filterset({'cf_cf10__empty': True}, self.queryset).qs.count(), 2) def test_filter_object(self): manufacturer_ids = Manufacturer.objects.values_list('id', flat=True) diff --git a/netbox/netbox/filtersets.py b/netbox/netbox/filtersets.py index f24b4e11c29..ea24efe48ae 100644 --- a/netbox/netbox/filtersets.py +++ b/netbox/netbox/filtersets.py @@ -29,6 +29,13 @@ 'OrganizationalModelFilterSet', ) +STANDARD_LOOKUPS = ( + 'exact', + 'iexact', + 'in', + 'contains', +) + # # FilterSets @@ -159,7 +166,7 @@ def get_additional_lookups(cls, existing_filter_name, existing_filter): return {} # Skip nonstandard lookup expressions - if existing_filter.method is not None or existing_filter.lookup_expr not in ['exact', 'iexact', 'in']: + if existing_filter.method is not None or existing_filter.lookup_expr not in STANDARD_LOOKUPS: return {} # Choose the lookup expression map based on the filter type From e01125a2adba59cba09f1a246976595578ba4114 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 18 Aug 2025 10:04:10 -0400 Subject: [PATCH 6/7] Revert FILTER_NUMERIC_BASED_LOOKUP_MAP change --- netbox/utilities/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index 1ad0d484756..9f027f8e16d 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -23,7 +23,7 @@ lt='lt', gte='gte', gt='gt', - empty='empty', + empty='isnull', ) FILTER_NEGATION_LOOKUP_MAP = dict( From 004bb69aa1478c925a8b037cb0c9d146bc0e7c86 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 21 Aug 2025 09:23:26 -0400 Subject: [PATCH 7/7] Override isnull lookup for custom fields --- netbox/extras/models/customfields.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index cabaeada859..19d9e1ded0a 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -600,6 +600,10 @@ def to_filter(self, lookup_expr=None): kwargs = { 'field_name': f'custom_field_data__{self.name}' } + # Native numeric filters will use `isnull` by default for empty lookups, but + # JSON fields require `empty` (see bug #20012). + if lookup_expr == 'isnull': + lookup_expr = 'empty' if lookup_expr is not None: kwargs['lookup_expr'] = lookup_expr