AssertionError in hypofuzz/database/models.py

[eivindjahren]

I tried to create a small example that tests the json module:

import hypothesis.strategies as st
from hypothesis import assume, given
from hypothesis.extra.lark import from_lark
import json
from contextlib import suppress
from lark import Lark

json_grammar = Lark(
    r"""
    value: dict
         | list
         | ESCAPED_STRING
         | SIGNED_NUMBER
         | "true" | "false" | "null"

    list : "[" [value ("," value)*] "]"

    dict : "{" [pair ("," pair)*] "}"
    pair : ESCAPED_STRING ":" value

    %import common.ESCAPED_STRING
    %import common.SIGNED_NUMBER
    %import common.WS
    %ignore WS

    """,
    start="value",
)


@given(st.one_of(st.text(), from_lark(json_grammar)))
def test_json(text):
    with suppress(json.JSONDecodeError):
        json.loads(text)


json_ground_values = st.one_of(
    st.text(), st.booleans(), st.integers(), st.floats(allow_nan=False)
)

json_data = st.recursive(
    json_ground_values, lambda x: st.one_of(st.lists(x), st.dictionaries(st.text(), x))
)


@given(json_data)
def test_json_roundtrip(data):
    assert json.loads(json.dumps(data)) == data

It seems to work for a while, but then I get the following crash:

Process Process-1:
Exception in thread Thread-3:
Traceback (most recent call last):
  File "...lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 364, in send_nowait_from_anywhere
  + Exception Group Traceback (most recent call last):
  |   File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
  |     self.run()
  |   File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run
  |     self._target(*self._args, **self._kwargs)
  |   File "...lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 419, in start_dashboard_process
  |     trio.run(run_dashboard, port, host)
  |   File "...lib/python3.12/site-packages/trio/_core/_run.py", line 2547, in run
  |     raise runner.main_task_outcome.error
  |   File "...lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 380, in run_dashboard
  |     async with trio.open_nursery() as nursery:
  |   File "...lib/python3.12/site-packages/trio/_core/_run.py", line 1125, in __aexit__
  |     raise combined_error_from_nursery
  | ExceptionGroup: Exceptions from Trio nursery (1 sub-exception)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "...lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 242, in handle_event
    |     event = _dashboard_event(db_event)
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "...lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 117, in _dashboard_event
    |     TESTS[value.nodeid].add_report(value)
    |   File "...lib/python3.12/site-packages/hypofuzz/dashboard/test.py", line 139, in add_report
    |     linear_report = ReportWithDiff.from_reports(
    |                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "...lib/python3.12/site-packages/hypofuzz/database/models.py", line 281, in from_reports
    |     return cls(
    |            ^^^^
    |   File "<string>", line 16, in __init__
    |   File "...lib/python3.12/site-packages/hypofuzz/database/models.py", line 244, in __post_init__
    |     assert all(
    |            ^^^^
    trio.lowlevel.current_task()
  File "...lib/python3.12/site-packages/trio/_core/_ run.py", line 2991, in current_task
    | AssertionError: {Status.OVERRUN: -2, Status.INVALID: 0, Status.VALID: 257, Status.INTERESTING: 0}

I have tried deleting the .hypothesis directory but it seems to not help. hypofuzz version 25.11.1, hypothesis version 6.142.1

[Liam-DeVoe]

Thanks! I think I see how this could happen, I'm looking into this today

[Stranger6667]

FWIW, I've also hit this while trying to integrate Hypofuzz with Schemathesis.

Almost right after the start:

> hypothesis fuzz -- test_app.py

Process Process-1:
  + Exception Group Traceback (most recent call last):
  |   File ".local/share/uv/python/cpython-3.12.9-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
  |     self.run()
  |   File ".local/share/uv/python/cpython-3.12.9-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
  |     self._target(*self._args, **self._kwargs)
  |   File "lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 419, in start_dashboard_process
  |     trio.run(run_dashboard, port, host)
  |   File "lib/python3.12/site-packages/trio/_core/_run.py", line 2549, in run
  |     raise runner.main_task_outcome.error
  |   File "lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 380, in run_dashboard
  |     async with trio.open_nursery() as nursery:
  |                ^^^^^^^^^^^^^^^^^^^
  |   File "...schemathesis/.venv/lib/python3.12/site-packages/trio/_core/_run.py", line 1125, in __aexit__
  |     raise combined_error_from_nursery
  | ExceptionGroup: Exceptions from Trio nursery (1 sub-exception)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 339, in load_initial_state
    |     await trio.to_thread.run_sync(_load_initial_state, fuzz_target)
    |   File "lib/python3.12/site-packages/trio/_threads.py", line 449, in to_thread_run_sync
    |     return msg_from_thread.unwrap()
    |            ^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "lib/python3.12/site-packages/outcome/_impl.py", line 213, in unwrap
    |     raise captured_error
    |   File "lib/python3.12/site-packages/trio/_threads.py", line 373, in do_release_then_return_result
    |     return result.unwrap()
    |            ^^^^^^^^^^^^^^^
    |   File "lib/python3.12/site-packages/outcome/_impl.py", line 213, in unwrap
    |     raise captured_error
    |   File "lib/python3.12/site-packages/trio/_threads.py", line 392, in worker_fn
    |     ret = context.run(sync_fn, *args)
    |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "lib/python3.12/site-packages/hypofuzz/dashboard/dashboard.py", line 319, in _load_initial_state
    |     test = Test(
    |            ^^^^^
    |   File "<string>", line 10, in __init__
    |   File "lib/python3.12/site-packages/hypofuzz/dashboard/test.py", line 64, in __post_init__
    |     self.add_report(report)
    |   File "lib/python3.12/site-packages/hypofuzz/dashboard/test.py", line 139, in add_report
    |     linear_report = ReportWithDiff.from_reports(
    |                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "lib/python3.12/site-packages/hypofuzz/database/models.py", line 281, in from_reports
    |     return cls(
    |            ^^^^
    |   File "<string>", line 16, in __init__
    |   File "lib/python3.12/site-packages/hypofuzz/database/models.py", line 244, in __post_init__
    |     assert all(
    |            ^^^^
    | AssertionError: {Status.OVERRUN: 0, Status.INVALID: 0, Status.VALID: -13, Status.INTERESTING: 0}
    +------------------------------------

test_app.py:

from hypothesis import settings

import schemathesis

schema = schemathesis.openapi.from_url("http://127.0.0.1:8081/schema.yaml")


@schema.parametrize()
@settings(database=settings().database)
def test_api(case):
    pass

where the test server comes from the schemathesis repo (after uv pip install -e '.[dev]' or similar)

./test_server.sh --spec=openapi3 8081

[Liam-DeVoe]

This ended up being more complicated to fix than I hoped. It's a subtle logic bug when reports arrive out of order from different workers. I have a tentative fix here if you want to test it out: https://github.com/Zac-HD/hypofuzz/compare/assertion-fix.

As a silly workaround, you can also limit hypofuzz to one worker with -n 1.

[Stranger6667]

Unfortunately, neither approach solves the exception (nor their combination) :(

 hypothesis fuzz -n 1 -- test_app.py
collecting tests...
        Now serving dashboard at http://localhost:9999/

done
using 1 process to fuzz 35 tests
Process Process-1:
  + Exception Group Traceback (most recent call last):
  |   File ".local/share/uv/python/cpython-3.12.9-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
  |     self.run()
  |   File ".local/share/uv/python/cpython-3.12.9-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
  |     self._target(*self._args, **self._kwargs)
  |   File "hypofuzz/src/hypofuzz/dashboard/dashboard.py", line 415, in start_dashboard_process
  |     trio.run(run_dashboard, port, host)
  |   File "lib/python3.12/site-packages/trio/_core/_run.py", line 2549, in run
  |     raise runner.main_task_outcome.error
  |   File "hypofuzz/src/hypofuzz/dashboard/dashboard.py", line 376, in run_dashboard
  |     async with trio.open_nursery() as nursery:
  |                ^^^^^^^^^^^^^^^^^^^
  |   File "lib/python3.12/site-packages/trio/_core/_run.py", line 1125, in __aexit__
  |     raise combined_error_from_nursery
  | ExceptionGroup: Exceptions from Trio nursery (14 sub-exceptions)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "hypofuzz/src/hypofuzz/dashboard/dashboard.py", line 335, in load_initial_state
    |     await trio.to_thread.run_sync(_load_initial_state, fuzz_target)
    |   File "lib/python3.12/site-packages/trio/_threads.py", line 449, in to_thread_run_sync
    |     return msg_from_thread.unwrap()
    |            ^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "lib/python3.12/site-packages/outcome/_impl.py", line 213, in unwrap
    |     raise captured_error
    |   File "lib/python3.12/site-packages/trio/_threads.py", line 373, in do_release_then_return_result
    |     return result.unwrap()
    |            ^^^^^^^^^^^^^^^
    |   File "lib/python3.12/site-packages/outcome/_impl.py", line 213, in unwrap
    |     raise captured_error
    |   File "lib/python3.12/site-packages/trio/_threads.py", line 392, in worker_fn
    |     ret = context.run(sync_fn, *args)
    |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "hypofuzz/src/hypofuzz/dashboard/dashboard.py", line 317, in _load_initial_state
    |     test = Test(
    |            ^^^^^
    |   File "<string>", line 10, in __init__
    |   File "hypofuzz/src/hypofuzz/dashboard/test.py", line 65, in __post_init__
    |     self.add_report(report)
    |   File "hypofuzz/src/hypofuzz/dashboard/test.py", line 146, in add_report
    |     linear_report = ReportWithDiff.from_reports(
    |                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "hypofuzz/src/hypofuzz/database/models.py", line 281, in from_reports
    |     return cls(
    |            ^^^^
    |   File "<string>", line 16, in __init__
    |   File "hypofuzz/src/hypofuzz/database/models.py", line 244, in __post_init__
    |     assert all(
    |            ^^^^
    | AssertionError: {Status.OVERRUN: 0, Status.INVALID: -2, Status.VALID: 2, Status.INTERESTING: 0}

[Liam-DeVoe]

hmm, you might also have to delete .hypothesis? If that also doesn't work, then this is a different problem than I was expecting!

[Stranger6667]

Right! It works after deleting .hypothesis :) Thanks!

[Zac-HD]

hmm, I thought we'd designed everything so that changes to the linearization logic didn't need to invalidate state on disk? That's a little worrying if I'm wrong.

re Liam's patch: lgtm, though pairwise is just needlessly expensive when there's already a total ordering over elements.