Use zigpy SerialProtocol (#256)

* Use zigpy flow control

* Bump minimum zigpy version

* Upgrade pytest-asyncio

* Use `znp.disconnect` instead of `znp.close`

* Try to improve joining test reliability

* Try to wait for device initialization tasks

Author: puddly

pyproject.toml

@@ -14,7 +14,7 @@ readme = "README.md"
 license = {text = "GPL-3.0"}
 requires-python = ">=3.8"
 dependencies = [
-    "zigpy>=0.69.0",
+    "zigpy>=0.70.0",
     "async_timeout",
     "voluptuous",
     "coloredlogs",
@@ -63,6 +63,7 @@ timeout = 20
 log_format = "%(asctime)s.%(msecs)03d %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
 
 [tool.flake8]
 exclude = ".venv,.git,.tox,docs,venv,bin,lib,deps,build"

tests/api/test_connect.py

@@ -19,7 +19,7 @@ async def test_connect_no_test(make_znp_server):
     # Nothing will be sent
     assert znp_server._uart.data_received.call_count == 0
 
-    znp.close()
+    await znp.disconnect()
 
 
 @pytest.mark.parametrize("work_after_attempt", [1, 2, 3])
@@ -44,7 +44,7 @@ def ping_rsp(req):
 
     await znp.connect(test_port=True)
 
-    znp.close()
+    await znp.disconnect()
 
 
 async def test_connect_skip_bootloader_batched_rsp(make_znp_server, mocker):
@@ -82,7 +82,7 @@ def ping_rsp(req):
 
     await znp.connect(test_port=True)
 
-    znp.close()
+    await znp.disconnect()
 
 
 async def test_connect_skip_bootloader_failure(make_znp_server):
@@ -92,7 +92,7 @@ async def test_connect_skip_bootloader_failure(make_znp_server):
     with pytest.raises(asyncio.TimeoutError):
         await znp.connect(test_port=True)
 
-    znp.close()
+    await znp.disconnect()
 
 
 async def test_connect_skip_bootloader_rts_dtr_pins(make_znp_server, mocker):
@@ -112,7 +112,7 @@ async def test_connect_skip_bootloader_rts_dtr_pins(make_znp_server, mocker):
     assert serial._mock_dtr_prop.mock_calls == [call(False), call(False), call(False)]
     assert serial._mock_rts_prop.mock_calls == [call(False), call(True), call(False)]
 
-    znp.close()
+    await znp.disconnect()
 
 
 async def test_connect_skip_bootloader_config(make_znp_server, mocker):
@@ -133,24 +133,24 @@ async def test_connect_skip_bootloader_config(make_znp_server, mocker):
     assert serial._mock_dtr_prop.called is False
     assert serial._mock_rts_prop.called is False
 
-    znp.close()
+    await znp.disconnect()
 
 
 async def test_api_close(connected_znp, mocker):
     znp, znp_server = connected_znp
     uart = znp._uart
     mocker.spy(uart, "close")
 
-    znp.close()
+    await znp.disconnect()
 
     # Make sure our UART was actually closed
     assert znp._uart is None
     assert znp._app is None
     assert uart.close.call_count == 1
 
-    # ZNP.close should not throw any errors if called multiple times
-    znp.close()
-    znp.close()
+    # ZNP.disconnect should not throw any errors if called multiple times
+    await znp.disconnect()
+    await znp.disconnect()
 
     def dict_minus(d, minus):
         return {k: v for k, v in d.items() if k not in minus}
@@ -165,8 +165,8 @@ def dict_minus(d, minus):
         znp2.__dict__, ignored_keys
     )
 
-    znp2.close()
-    znp2.close()
+    await znp2.disconnect()
+    await znp2.disconnect()
 
     assert dict_minus(znp.__dict__, ignored_keys) == dict_minus(
         znp2.__dict__, ignored_keys

tests/api/test_listeners.py

@@ -8,13 +8,13 @@
 from zigpy_znp.api import OneShotResponseListener, CallbackResponseListener
 
 
-async def test_resolve(event_loop, mocker):
+async def test_resolve(mocker):
     callback = mocker.Mock()
     callback_listener = CallbackResponseListener(
         [c.SYS.Ping.Rsp(partial=True)], callback
     )
 
-    future = event_loop.create_future()
+    future = asyncio.get_running_loop().create_future()
     one_shot_listener = OneShotResponseListener([c.SYS.Ping.Rsp(partial=True)], future)
 
     match = c.SYS.Ping.Rsp(Capabilities=t.MTCapabilities.SYS)
@@ -42,9 +42,9 @@ async def test_resolve(event_loop, mocker):
     assert one_shot_listener.cancel()
 
 
-async def test_cancel(event_loop):
+async def test_cancel():
     # Cancelling a one-shot listener prevents it from being fired
-    future = event_loop.create_future()
+    future = asyncio.get_running_loop().create_future()
     one_shot_listener = OneShotResponseListener([c.SYS.Ping.Rsp(partial=True)], future)
     one_shot_listener.cancel()
 
@@ -55,13 +55,13 @@ async def test_cancel(event_loop):
         await future
 
 
-async def test_multi_cancel(event_loop, mocker):
+async def test_multi_cancel(mocker):
     callback = mocker.Mock()
     callback_listener = CallbackResponseListener(
         [c.SYS.Ping.Rsp(partial=True)], callback
     )
 
-    future = event_loop.create_future()
+    future = asyncio.get_running_loop().create_future()
     one_shot_listener = OneShotResponseListener([c.SYS.Ping.Rsp(partial=True)], future)
 
     match = c.SYS.Ping.Rsp(Capabilities=t.MTCapabilities.SYS)
@@ -93,7 +93,7 @@ async def test_api_cancel_listeners(connected_znp, mocker):
     )
 
     assert not future.done()
-    znp.close()
+    await znp.disconnect()
 
     with pytest.raises(asyncio.CancelledError):
         await future

tests/api/test_network_state.py

@@ -23,7 +23,7 @@ async def test_state_transfer(from_device, to_device, make_connected_znp):
     formed_znp, _ = await make_connected_znp(server_cls=from_device)
 
     await formed_znp.load_network_info()
-    formed_znp.close()
+    await formed_znp.disconnect()
 
     empty_znp, _ = await make_connected_znp(server_cls=to_device)
 
@@ -72,15 +72,15 @@ async def test_broken_cc2531_load_state(device, make_connected_znp, caplog):
     await znp.load_network_info()
     assert "inconsistent" in caplog.text
 
-    znp.close()
+    await znp.disconnect()
 
 
 @pytest.mark.parametrize("device", [FormedZStack3CC2531])
 async def test_state_write_tclk_zstack3(device, make_connected_znp, caplog):
     formed_znp, _ = await make_connected_znp(server_cls=device)
 
     await formed_znp.load_network_info()
-    formed_znp.close()
+    await formed_znp.disconnect()
 
     empty_znp, _ = await make_connected_znp(server_cls=device)
 
@@ -106,7 +106,7 @@ async def test_state_write_tclk_zstack3(device, make_connected_znp, caplog):
 async def test_write_settings_fast(device, make_connected_znp):
     formed_znp, _ = await make_connected_znp(server_cls=FormedLaunchpadCC26X2R1)
     await formed_znp.load_network_info()
-    formed_znp.close()
+    await formed_znp.disconnect()
 
     znp, _ = await make_connected_znp(server_cls=device)
 
@@ -126,7 +126,7 @@ async def test_write_settings_fast(device, make_connected_znp):
 async def test_formation_failure_on_corrupted_nvram(device, make_connected_znp):
     formed_znp, _ = await make_connected_znp(server_cls=FormedLaunchpadCC26X2R1)
     await formed_znp.load_network_info()
-    formed_znp.close()
+    await formed_znp.disconnect()
 
     znp, znp_server = await make_connected_znp(server_cls=device)
 

tests/api/test_request.py

@@ -11,7 +11,7 @@
 from zigpy_znp.exceptions import CommandNotRecognized, InvalidCommandResponse
 
 
-async def test_callback_rsp(connected_znp, event_loop):
+async def test_callback_rsp(connected_znp):
     znp, znp_server = connected_znp
 
     def send_responses():
@@ -20,7 +20,7 @@ def send_responses():
             c.AF.DataConfirm.Callback(Endpoint=56, TSN=1, Status=t.Status.SUCCESS)
         )
 
-    event_loop.call_soon(send_responses)
+    asyncio.get_running_loop().call_soon(send_responses)
 
     # The UART sometimes replies with a SRSP and an AREQ faster than
     # we can register callbacks for both. This method is a workaround.
@@ -150,7 +150,7 @@ async def replier(req):
     assert len(znp._unhandled_command.mock_calls) == 0
 
 
-async def test_callback_rsp_cleanup_concurrent(connected_znp, event_loop, mocker):
+async def test_callback_rsp_cleanup_concurrent(connected_znp, mocker):
     znp, znp_server = connected_znp
 
     mocker.spy(znp, "_unhandled_command")
@@ -163,7 +163,7 @@ def send_responses():
         znp_server.send(c.SYS.OSALTimerExpired.Callback(Id=0xAB))
         znp_server.send(c.SYS.OSALTimerExpired.Callback(Id=0xCD))
 
-    event_loop.call_soon(send_responses)
+    asyncio.get_running_loop().call_soon(send_responses)
 
     callback_rsp = await znp.request_callback_rsp(
         request=c.UTIL.TimeAlive.Req(),
@@ -183,7 +183,7 @@ def send_responses():
     ]
 
 
-async def test_znp_request_kwargs(connected_znp, event_loop):
+async def test_znp_request_kwargs(connected_znp):
     znp, znp_server = connected_znp
 
     # Invalid format
@@ -196,7 +196,7 @@ async def test_znp_request_kwargs(connected_znp, event_loop):
 
     # Valid format, valid name
     ping_rsp = c.SYS.Ping.Rsp(Capabilities=t.MTCapabilities.SYS)
-    event_loop.call_soon(znp_server.send, ping_rsp)
+    asyncio.get_running_loop().call_soon(znp_server.send, ping_rsp)
     assert (
         await znp.request(c.SYS.Ping.Req(), RspCapabilities=t.MTCapabilities.SYS)
     ) == ping_rsp
@@ -227,7 +227,7 @@ async def test_znp_request_kwargs(connected_znp, event_loop):
         )
 
 
-async def test_znp_request_not_recognized(connected_znp, event_loop):
+async def test_znp_request_not_recognized(connected_znp):
     znp, _ = connected_znp
 
     # An error is raise when a bad request is sent
@@ -237,11 +237,11 @@ async def test_znp_request_not_recognized(connected_znp, event_loop):
     )
 
     with pytest.raises(CommandNotRecognized):
-        event_loop.call_soon(znp.frame_received, unknown_rsp.to_frame())
+        asyncio.get_running_loop().call_soon(znp.frame_received, unknown_rsp.to_frame())
         await znp.request(request)
 
 
-async def test_znp_request_wrong_params(connected_znp, event_loop):
+async def test_znp_request_wrong_params(connected_znp):
     znp, _ = connected_znp
 
     # You cannot specify response kwargs for responses with no response
@@ -250,14 +250,14 @@ async def test_znp_request_wrong_params(connected_znp, event_loop):
 
     # An error is raised when a response with bad params is received
     with pytest.raises(InvalidCommandResponse):
-        event_loop.call_soon(
+        asyncio.get_running_loop().call_soon(
             znp.frame_received,
             c.SYS.Ping.Rsp(Capabilities=t.MTCapabilities.SYS).to_frame(),
         )
         await znp.request(c.SYS.Ping.Req(), RspCapabilities=t.MTCapabilities.APP)
 
 
-async def test_znp_sreq_srsp(connected_znp, event_loop):
+async def test_znp_sreq_srsp(connected_znp):
     znp, _ = connected_znp
 
     # Each SREQ must have a corresponding SRSP, so this will fail
@@ -267,7 +267,7 @@ async def test_znp_sreq_srsp(connected_znp, event_loop):
 
     # This will work
     ping_rsp = c.SYS.Ping.Rsp(Capabilities=t.MTCapabilities.SYS)
-    event_loop.call_soon(znp.frame_received, ping_rsp.to_frame())
+    asyncio.get_running_loop().call_soon(znp.frame_received, ping_rsp.to_frame())
 
     await znp.request(c.SYS.Ping.Req())
 

tests/api/test_response.py

@@ -190,7 +190,7 @@ async def test_wait_responses_empty(connected_znp):
         await znp.wait_for_responses([])
 
 
-async def test_response_callback_simple(connected_znp, event_loop, mocker):
+async def test_response_callback_simple(connected_znp, mocker):
     znp, _ = connected_znp
 
     sync_callback = mocker.Mock()
@@ -207,7 +207,7 @@ async def test_response_callback_simple(connected_znp, event_loop, mocker):
     sync_callback.assert_called_once_with(good_response)
 
 
-async def test_response_callbacks(connected_znp, event_loop, mocker):
+async def test_response_callbacks(connected_znp, mocker):
     znp, _ = connected_znp
 
     sync_callback = mocker.Mock()
@@ -270,7 +270,7 @@ async def async_callback(response):
     assert len(async_callback_responses) == 3
 
 
-async def test_wait_for_responses(connected_znp, event_loop):
+async def test_wait_for_responses(connected_znp):
     znp, _ = connected_znp
 
     response1 = c.SYS.Ping.Rsp(Capabilities=t.MTCapabilities.SYS)

tests/application/test_joining.py

@@ -184,6 +184,10 @@ async def test_permit_join_with_key(device, permit_result, make_application, moc
     await app.shutdown()
 
 
+@mock.patch(
+    "zigpy.device.Device._initialize",
+    new=zigpy.device.Device._initialize.__wrapped__,  # to disable retries
+)
 @pytest.mark.parametrize("device", FORMED_DEVICES)
 async def test_on_zdo_device_join(device, make_application, mocker):
     app, znp_server = make_application(server_cls=device)
@@ -204,6 +208,10 @@ async def test_on_zdo_device_join(device, make_application, mocker):
     await app.shutdown()
 
 
+@mock.patch(
+    "zigpy.device.Device._initialize",
+    new=zigpy.device.Device._initialize.__wrapped__,  # to disable retries
+)
 @pytest.mark.parametrize("device", FORMED_DEVICES)
 async def test_on_zdo_device_join_and_announce_fast(device, make_application, mocker):
     app, znp_server = make_application(server_cls=device)
@@ -258,8 +266,12 @@ async def test_on_zdo_device_join_and_announce_fast(device, make_application, mo
     # Everything is cleaned up
     assert not app._join_announce_tasks
 
+    app.get_device(ieee=ieee).cancel_initialization()
     await app.shutdown()
 
+    with pytest.raises(asyncio.CancelledError):
+        await app.get_device(ieee=ieee)._initialize_task
+
 
 @mock.patch("zigpy_znp.zigbee.application.DEVICE_JOIN_MAX_DELAY", new=0.1)
 @mock.patch(
@@ -329,3 +341,6 @@ async def test_on_zdo_device_join_and_announce_slow(device, make_application, mo
 
     app.get_device(ieee=ieee).cancel_initialization()
     await app.shutdown()
+
+    with pytest.raises(asyncio.CancelledError):
+        await app.get_device(ieee=ieee)._initialize_task