From 400c064fc749705b8416e5ae6d20ab927f83c9f3 Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Wed, 24 Dec 2025 15:47:58 -0300 Subject: [PATCH 1/5] process: Use uvloop in pipeline process Faster --- runner/pyproject.toml | 1 + runner/src/runner/live/process/process.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/runner/pyproject.toml b/runner/pyproject.toml index 4ddcf4e51..f7f7b3074 100644 --- a/runner/pyproject.toml +++ b/runner/pyproject.toml @@ -67,6 +67,7 @@ realtime-base = [ "watchdog==5.0.2", "opencv-python==4.10.0.84", "av==14.4.0", + "uvloop>=0.22.0", ] # Actual extra for realtime pipelines with CUDA-enabled torch. This requires a GPU, so use realtime-dev if unavailable. realtime = [ diff --git a/runner/src/runner/live/process/process.py b/runner/src/runner/live/process/process.py index 07b11edeb..437315979 100644 --- a/runner/src/runner/live/process/process.py +++ b/runner/src/runner/live/process/process.py @@ -13,6 +13,7 @@ from typing import Any import torch +import uvloop from ..pipelines import Pipeline, BaseParams, PipelineSpec from ..pipelines.loader import load_pipeline, parse_pipeline_params @@ -197,6 +198,9 @@ def process_loop(self): # ( load_gpu_models is calling logging.info() for every frame ) logging.getLogger("comfy").setLevel(logging.WARNING) + # Use uvloop for better performance + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + try: asyncio.run(self._run_pipeline_loops()) except Exception as e: From aa340b8f72e602b3dc8dbccdb4a23b1774d3607f Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Wed, 24 Dec 2025 15:48:32 -0300 Subject: [PATCH 2/5] Revert "process: Reduce VRAM usage from parent process (#843)" This reverts commit 7d2ea10306aed07bc00c462684de82ebb327bc41. --- runner/src/runner/live/process/process.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/runner/src/runner/live/process/process.py b/runner/src/runner/live/process/process.py index 437315979..b994402a3 100644 --- a/runner/src/runner/live/process/process.py +++ b/runner/src/runner/live/process/process.py @@ -141,8 +141,8 @@ def update_params(self, params: dict): self.param_update_queue.put(params) def reset_stream(self, request_id: str, manifest_id: str, stream_id: str): - # Clear queues to avoid using frames from previous sessions - clear_queue(self.input_queue) + # we cannot clear the input_queue as we send CUDA tensors on it, which can't be received by the same process that sent it. + # So we clear only the other queues, and rely on the request_id checks to avoid using frames from previous sessions. clear_queue(self.output_queue) clear_queue(self.param_update_queue) clear_queue(self.error_queue) @@ -158,6 +158,8 @@ def reset_stream(self, request_id: str, manifest_id: str, stream_id: str): # TODO: Once audio is implemented, combined send_input with input_loop # We don't need additional queueing as comfystream already maintains a queue def send_input(self, frame: InputFrame): + if isinstance(frame, VideoFrame) and not frame.tensor.is_cuda and torch.cuda.is_available(): + frame = frame.replace_tensor(frame.tensor.cuda()) self._try_queue_put(self.input_queue, frame) async def recv_output(self) -> OutputFrame | None: @@ -276,10 +278,6 @@ async def _input_loop(self, pipeline: Pipeline, overlay: LoadingOverlayRenderer) if isinstance(input, VideoFrame): input.log_timestamps["pre_process_frame"] = time.time() - # Move CPU tensors to GPU before sending to pipeline - if not input.tensor.is_cuda and torch.cuda.is_available(): - input = input.replace_tensor(input.tensor.cuda()) - if self._is_loading() and self._last_params.show_reloading_frame: await self._render_loading_frame(overlay, input) else: @@ -298,10 +296,8 @@ async def _render_loading_frame(self, overlay: LoadingOverlayRenderer, input: Vi w, h = self._last_params.get_output_resolution() loading_tensor = await overlay.render(w, h) - - # Move to CPU before sending over multiprocessing queue to avoid CUDA IPC overhead - if loading_tensor.is_cuda: - loading_tensor = loading_tensor.cpu() + if torch.cuda.is_available() and not loading_tensor.is_cuda: + loading_tensor = loading_tensor.cuda() out_frame = input.replace_tensor(loading_tensor) out = VideoOutput(out_frame, self.request_id, is_loading_frame=True) @@ -321,9 +317,8 @@ async def _output_loop(self, pipeline: Pipeline, overlay: LoadingOverlayRenderer continue overlay.end_reload() - # Move to CPU before sending over multiprocessing queue to avoid CUDA IPC overhead - if isinstance(out, VideoOutput) and out.tensor.is_cuda: - out = out.replace_tensor(out.tensor.cpu()) + if isinstance(out, VideoOutput) and not out.tensor.is_cuda and torch.cuda.is_available(): + out = out.replace_tensor(out.tensor.cuda()) out.log_timestamps["post_process_frame"] = time.time() self._try_queue_put(self.output_queue, out) From fdbe80f1257d201017640c663d888c5475bc75e3 Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Wed, 24 Dec 2025 18:50:53 +0000 Subject: [PATCH 3/5] uv lock --- runner/uv.lock | 64 +++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/runner/uv.lock b/runner/uv.lock index 4548293c1..1c80584cb 100644 --- a/runner/uv.lock +++ b/runner/uv.lock @@ -153,6 +153,7 @@ realtime = [ { name = "pydantic", version = "2.11.7", source = { registry = "https://pypi.org/simple" } }, { name = "pytorch-triton" }, { name = "pyzmq" }, + { name = "uvloop" }, { name = "watchdog" }, ] realtime-base = [ @@ -160,6 +161,7 @@ realtime-base = [ { name = "opencv-python" }, { name = "pydantic", version = "2.11.7", source = { registry = "https://pypi.org/simple" } }, { name = "pyzmq" }, + { name = "uvloop" }, { name = "watchdog" }, ] realtime-dev = [ @@ -171,6 +173,7 @@ realtime-dev = [ { name = "torch", version = "2.9.1", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev')" }, { name = "torch", version = "2.9.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform != 'darwin' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" } }, + { name = "uvloop" }, { name = "watchdog" }, ] @@ -230,12 +233,13 @@ requires-dist = [ { name = "triton", marker = "extra == 'llm'" }, { name = "types-psutil", marker = "extra == 'batch'", specifier = "==6.0.0.20241011" }, { name = "uvicorn", specifier = "==0.38.0" }, + { name = "uvloop", marker = "extra == 'realtime-base'", specifier = ">=0.22.0" }, { name = "vllm", marker = "extra == 'llm'", specifier = "==0.6.5" }, { name = "watchdog", marker = "extra == 'realtime-base'", specifier = "==5.0.2" }, { name = "xformers", marker = "extra == 'batch'", specifier = "==0.0.23" }, { name = "xformers", marker = "extra == 'llm'" }, ] -provides-extras = ["batch", "realtime-base", "realtime", "llm", "dev", "realtime-dev"] +provides-extras = ["batch", "llm", "realtime-base", "realtime", "realtime-dev", "dev"] [[package]] name = "aiohappyeyeballs" @@ -519,7 +523,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "(implementation_name != 'PyPy' and extra == 'extra-9-ai-runner-llm') or (implementation_name != 'PyPy' and extra == 'extra-9-ai-runner-realtime') or (implementation_name != 'PyPy' and extra == 'extra-9-ai-runner-realtime-base') or (implementation_name != 'PyPy' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -757,7 +761,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11' or extra == 'extra-9-ai-runner-batch' or extra == 'extra-9-ai-runner-llm' or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-realtime' and extra != 'extra-9-ai-runner-realtime-base' and extra != 'extra-9-ai-runner-realtime-dev')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-batch' and extra != 'extra-9-ai-runner-llm' and extra != 'extra-9-ai-runner-realtime' and extra != 'extra-9-ai-runner-realtime-base' and extra != 'extra-9-ai-runner-realtime-dev')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -990,7 +994,7 @@ name = "importlib-metadata" version = "8.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "extra == 'extra-9-ai-runner-batch' or extra == 'extra-9-ai-runner-llm' or extra == 'extra-9-ai-runner-realtime-dev' or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base')" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } wheels = [ @@ -1020,7 +1024,7 @@ name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markupsafe", marker = "extra == 'extra-9-ai-runner-batch' or extra == 'extra-9-ai-runner-llm' or extra == 'extra-9-ai-runner-realtime-dev' or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base')" }, + { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ @@ -1194,13 +1198,13 @@ name = "mlx-lm" version = "0.28.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jinja2" }, + { name = "jinja2", marker = "sys_platform == 'darwin'" }, { name = "mlx", marker = "sys_platform == 'darwin'" }, - { name = "numpy" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "sentencepiece" }, - { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", marker = "sys_platform == 'darwin'" }, + { name = "protobuf", marker = "sys_platform == 'darwin'" }, + { name = "pyyaml", marker = "sys_platform == 'darwin'" }, + { name = "sentencepiece", marker = "sys_platform == 'darwin'" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f2/7f/94b3f7e00c4681a4fe2d47b519458245bd8a8f0506b1ce018d1850bbcf79/mlx_lm-0.28.4.tar.gz", hash = "sha256:3661d8ef5f0e2695d52993e0df1ed2c1f93ca1d094258146c18d9cec0c50514e", size = 232455, upload-time = "2025-12-03T22:39:59.122Z" } wheels = [ @@ -1550,7 +1554,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "nvidia-cublas-cu12", version = "12.4.5.8", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-9-ai-runner-llm' or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "nvidia-cublas-cu12", version = "12.4.5.8", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741, upload-time = "2024-04-22T15:24:15.253Z" }, @@ -1579,7 +1583,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "extra == 'extra-9-ai-runner-llm' or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/7a/8a/0e728f749baca3fbeffad762738276e5df60851958be7783af121a7221e7/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399", size = 211422548, upload-time = "2024-06-18T19:33:39.396Z" }, @@ -1641,9 +1645,9 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "nvidia-cublas-cu12", version = "12.4.5.8", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-9-ai-runner-llm' or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, - { name = "nvidia-cusparse-cu12", version = "12.3.1.170", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-9-ai-runner-llm' or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, - { name = "nvidia-nvjitlink-cu12", marker = "extra == 'extra-9-ai-runner-llm' or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "nvidia-cublas-cu12", version = "12.4.5.8", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "nvidia-cusparse-cu12", version = "12.3.1.170", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/46/6b/a5c33cf16af09166845345275c34ad2190944bcc6026797a39f8e0a282e0/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e", size = 127634111, upload-time = "2024-06-18T19:35:01.793Z" }, @@ -1676,7 +1680,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "extra == 'extra-9-ai-runner-llm' or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-9-ai-runner-llm') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'darwin' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (sys_platform == 'linux' and extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra != 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/96/a9/c0d2f83a53d40a4a41be14cea6a0bf9e668ffcf8b004bd65633f433050c0/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3", size = 207381987, upload-time = "2024-06-18T19:35:32.989Z" }, @@ -2470,7 +2474,7 @@ name = "pyzmq" version = "26.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "(implementation_name == 'pypy' and extra == 'extra-9-ai-runner-llm') or (implementation_name == 'pypy' and extra == 'extra-9-ai-runner-realtime') or (implementation_name == 'pypy' and extra == 'extra-9-ai-runner-realtime-base') or (implementation_name == 'pypy' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "cffi", marker = "implementation_name == 'pypy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fd/05/bed626b9f7bb2322cdbbf7b4bd8f54b1b617b0d2ab2d3547d6e39428a48e/pyzmq-26.2.0.tar.gz", hash = "sha256:070672c258581c8e4f640b5159297580a9974b026043bd4ab0470be9ed324f1f", size = 271975, upload-time = "2024-08-22T09:02:03.351Z" } wheels = [ @@ -2586,10 +2590,10 @@ name = "requests" version = "2.32.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi", marker = "extra == 'extra-9-ai-runner-batch' or extra == 'extra-9-ai-runner-llm' or extra == 'extra-9-ai-runner-realtime-dev' or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base')" }, - { name = "charset-normalizer", marker = "extra == 'extra-9-ai-runner-batch' or extra == 'extra-9-ai-runner-llm' or extra == 'extra-9-ai-runner-realtime-dev' or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base')" }, - { name = "idna", marker = "extra == 'extra-9-ai-runner-batch' or extra == 'extra-9-ai-runner-llm' or extra == 'extra-9-ai-runner-realtime-dev' or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base')" }, - { name = "urllib3", marker = "extra == 'extra-9-ai-runner-batch' or extra == 'extra-9-ai-runner-llm' or extra == 'extra-9-ai-runner-realtime-dev' or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base')" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ @@ -3106,9 +3110,9 @@ name = "torchvision" version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "pillow" }, - { name = "torch", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "pillow", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "torch", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8d/59/aea68d755da1451e1a0d894528a7edc9b58eb30d33e274bf21bef28dad1a/torchvision-0.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4878fefb96ef293d06c27210918adc83c399d9faaf34cda5a63e129f772328f1", size = 1787552, upload-time = "2024-10-29T17:40:34.071Z" }, @@ -3126,7 +3130,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(sys_platform == 'win32' and extra == 'extra-9-ai-runner-batch') or (sys_platform == 'win32' and extra == 'extra-9-ai-runner-llm') or (sys_platform == 'win32' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-llm') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-batch' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-llm' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime-base' and extra == 'extra-9-ai-runner-realtime-dev') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-base') or (extra == 'extra-9-ai-runner-realtime' and extra == 'extra-9-ai-runner-realtime-dev')" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } wheels = [ @@ -3512,10 +3516,10 @@ version = "0.1.21" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, - { name = "ninja" }, - { name = "pydantic", version = "2.12.5", source = { registry = "https://pypi.org/simple" } }, - { name = "torch", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, - { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" } }, + { name = "ninja", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "pydantic", version = "2.12.5", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "torch", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, { name = "triton", version = "3.1.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e3/52/ea664a56674f21c401b45f124c207a16ca4b2318364687172edbcf255375/xgrammar-0.1.21.tar.gz", hash = "sha256:2ce1e81417ff46aa7ef26d8c0627275cb20dd1f2e8ead5bb261aecde1cc8ba57", size = 2242013, upload-time = "2025-07-10T19:34:14.336Z" } From 4bb58416be51d3cb123d493a974241af73ab9966 Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Wed, 24 Dec 2025 18:52:13 +0000 Subject: [PATCH 4/5] uv lock pipelines --- live/comfyui/uv.lock | 16 ++++++++++++++++ live/scope/uv.lock | 18 +++++++++++++++++- live/streamdiffusion/uv.lock | 18 +++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/live/comfyui/uv.lock b/live/comfyui/uv.lock index c9990281e..60cd0c99b 100644 --- a/live/comfyui/uv.lock +++ b/live/comfyui/uv.lock @@ -61,6 +61,7 @@ realtime = [ { name = "pydantic" }, { name = "pytorch-triton" }, { name = "pyzmq" }, + { name = "uvloop" }, { name = "watchdog" }, ] @@ -120,6 +121,7 @@ requires-dist = [ { name = "triton", marker = "extra == 'llm'" }, { name = "types-psutil", marker = "extra == 'batch'", specifier = "==6.0.0.20241011" }, { name = "uvicorn", specifier = "==0.38.0" }, + { name = "uvloop", marker = "extra == 'realtime-base'", specifier = ">=0.22.0" }, { name = "vllm", marker = "extra == 'llm'", specifier = "==0.6.5" }, { name = "watchdog", marker = "extra == 'realtime-base'", specifier = "==5.0.2" }, { name = "xformers", marker = "extra == 'batch'", specifier = "==0.0.23" }, @@ -2898,6 +2900,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02", size = 68109, upload-time = "2025-10-18T13:46:42.958Z" }, ] +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/d5/69900f7883235562f1f50d8184bb7dd84a2fb61e9ec63f3782546fdbd057/uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9", size = 1352420, upload-time = "2025-10-16T22:16:21.187Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/c4e271b3bce59724e291465cc936c37758886a4868787da0278b3b56b905/uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77", size = 748677, upload-time = "2025-10-16T22:16:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" }, + { url = "https://files.pythonhosted.org/packages/74/4f/256aca690709e9b008b7108bc85fba619a2bc37c6d80743d18abad16ee09/uvloop-0.22.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702", size = 3804529, upload-time = "2025-10-16T22:16:25.246Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/03c05ae4737e871923d21a76fe28b6aad57f5c03b6e6bfcfa5ad616013e4/uvloop-0.22.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733", size = 3621267, upload-time = "2025-10-16T22:16:26.819Z" }, + { url = "https://files.pythonhosted.org/packages/75/be/f8e590fe61d18b4a92070905497aec4c0e64ae1761498cad09023f3f4b3e/uvloop-0.22.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473", size = 3723105, upload-time = "2025-10-16T22:16:28.252Z" }, +] + [[package]] name = "vtracer" version = "0.6.11" diff --git a/live/scope/uv.lock b/live/scope/uv.lock index bc5259099..487fb8174 100644 --- a/live/scope/uv.lock +++ b/live/scope/uv.lock @@ -55,6 +55,7 @@ realtime = [ { name = "pydantic" }, { name = "pytorch-triton" }, { name = "pyzmq" }, + { name = "uvloop" }, { name = "watchdog" }, ] @@ -114,12 +115,13 @@ requires-dist = [ { name = "triton", marker = "extra == 'llm'" }, { name = "types-psutil", marker = "extra == 'batch'", specifier = "==6.0.0.20241011" }, { name = "uvicorn", specifier = "==0.38.0" }, + { name = "uvloop", marker = "extra == 'realtime-base'", specifier = ">=0.22.0" }, { name = "vllm", marker = "extra == 'llm'", specifier = "==0.6.5" }, { name = "watchdog", marker = "extra == 'realtime-base'", specifier = "==5.0.2" }, { name = "xformers", marker = "extra == 'batch'", specifier = "==0.0.23" }, { name = "xformers", marker = "extra == 'llm'" }, ] -provides-extras = ["batch", "realtime-base", "realtime", "llm", "dev", "realtime-dev"] +provides-extras = ["batch", "llm", "realtime-base", "realtime", "realtime-dev", "dev"] [[package]] name = "aiohappyeyeballs" @@ -1722,6 +1724,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02", size = 68109, upload-time = "2025-10-18T13:46:42.958Z" }, ] +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/14/ecceb239b65adaaf7fde510aa8bd534075695d1e5f8dadfa32b5723d9cfb/uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c", size = 1343335, upload-time = "2025-10-16T22:16:11.43Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ae/6f6f9af7f590b319c94532b9567409ba11f4fa71af1148cab1bf48a07048/uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792", size = 742903, upload-time = "2025-10-16T22:16:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/09/bd/3667151ad0702282a1f4d5d29288fce8a13c8b6858bf0978c219cd52b231/uvloop-0.22.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac33ed96229b7790eb729702751c0e93ac5bc3bcf52ae9eccbff30da09194b86", size = 3648499, upload-time = "2025-10-16T22:16:14.451Z" }, + { url = "https://files.pythonhosted.org/packages/b3/f6/21657bb3beb5f8c57ce8be3b83f653dd7933c2fd00545ed1b092d464799a/uvloop-0.22.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:481c990a7abe2c6f4fc3d98781cc9426ebd7f03a9aaa7eb03d3bfc68ac2a46bd", size = 3700133, upload-time = "2025-10-16T22:16:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/09/e0/604f61d004ded805f24974c87ddd8374ef675644f476f01f1df90e4cdf72/uvloop-0.22.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a592b043a47ad17911add5fbd087c76716d7c9ccc1d64ec9249ceafd735f03c2", size = 3512681, upload-time = "2025-10-16T22:16:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ce/8491fd370b0230deb5eac69c7aae35b3be527e25a911c0acdffb922dc1cd/uvloop-0.22.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1489cf791aa7b6e8c8be1c5a080bae3a672791fcb4e9e12249b05862a2ca9cec", size = 3615261, upload-time = "2025-10-16T22:16:19.596Z" }, +] + [[package]] name = "watchdog" version = "5.0.2" diff --git a/live/streamdiffusion/uv.lock b/live/streamdiffusion/uv.lock index c6de9f9c6..6e7ae9096 100644 --- a/live/streamdiffusion/uv.lock +++ b/live/streamdiffusion/uv.lock @@ -61,6 +61,7 @@ realtime = [ { name = "pydantic" }, { name = "pytorch-triton" }, { name = "pyzmq" }, + { name = "uvloop" }, { name = "watchdog" }, ] @@ -120,12 +121,13 @@ requires-dist = [ { name = "triton", marker = "extra == 'llm'" }, { name = "types-psutil", marker = "extra == 'batch'", specifier = "==6.0.0.20241011" }, { name = "uvicorn", specifier = "==0.38.0" }, + { name = "uvloop", marker = "extra == 'realtime-base'", specifier = ">=0.22.0" }, { name = "vllm", marker = "extra == 'llm'", specifier = "==0.6.5" }, { name = "watchdog", marker = "extra == 'realtime-base'", specifier = "==5.0.2" }, { name = "xformers", marker = "extra == 'batch'", specifier = "==0.0.23" }, { name = "xformers", marker = "extra == 'llm'" }, ] -provides-extras = ["batch", "realtime-base", "realtime", "llm", "dev", "realtime-dev"] +provides-extras = ["batch", "llm", "realtime-base", "realtime", "realtime-dev", "dev"] [[package]] name = "aiohappyeyeballs" @@ -2103,6 +2105,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02", size = 68109, upload-time = "2025-10-18T13:46:42.958Z" }, ] +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/d5/69900f7883235562f1f50d8184bb7dd84a2fb61e9ec63f3782546fdbd057/uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9", size = 1352420, upload-time = "2025-10-16T22:16:21.187Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/c4e271b3bce59724e291465cc936c37758886a4868787da0278b3b56b905/uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77", size = 748677, upload-time = "2025-10-16T22:16:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" }, + { url = "https://files.pythonhosted.org/packages/74/4f/256aca690709e9b008b7108bc85fba619a2bc37c6d80743d18abad16ee09/uvloop-0.22.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702", size = 3804529, upload-time = "2025-10-16T22:16:25.246Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/03c05ae4737e871923d21a76fe28b6aad57f5c03b6e6bfcfa5ad616013e4/uvloop-0.22.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733", size = 3621267, upload-time = "2025-10-16T22:16:26.819Z" }, + { url = "https://files.pythonhosted.org/packages/75/be/f8e590fe61d18b4a92070905497aec4c0e64ae1761498cad09023f3f4b3e/uvloop-0.22.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473", size = 3723105, upload-time = "2025-10-16T22:16:28.252Z" }, +] + [[package]] name = "watchdog" version = "5.0.2" From 0302b83ac621f05c08eea37984ad63f9fdd54184 Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Wed, 24 Dec 2025 20:31:40 +0000 Subject: [PATCH 5/5] Reapply "process: Reduce VRAM usage from parent process (#843)" This reverts commit aa340b8f72e602b3dc8dbccdb4a23b1774d3607f. --- runner/src/runner/live/process/process.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/runner/src/runner/live/process/process.py b/runner/src/runner/live/process/process.py index b994402a3..437315979 100644 --- a/runner/src/runner/live/process/process.py +++ b/runner/src/runner/live/process/process.py @@ -141,8 +141,8 @@ def update_params(self, params: dict): self.param_update_queue.put(params) def reset_stream(self, request_id: str, manifest_id: str, stream_id: str): - # we cannot clear the input_queue as we send CUDA tensors on it, which can't be received by the same process that sent it. - # So we clear only the other queues, and rely on the request_id checks to avoid using frames from previous sessions. + # Clear queues to avoid using frames from previous sessions + clear_queue(self.input_queue) clear_queue(self.output_queue) clear_queue(self.param_update_queue) clear_queue(self.error_queue) @@ -158,8 +158,6 @@ def reset_stream(self, request_id: str, manifest_id: str, stream_id: str): # TODO: Once audio is implemented, combined send_input with input_loop # We don't need additional queueing as comfystream already maintains a queue def send_input(self, frame: InputFrame): - if isinstance(frame, VideoFrame) and not frame.tensor.is_cuda and torch.cuda.is_available(): - frame = frame.replace_tensor(frame.tensor.cuda()) self._try_queue_put(self.input_queue, frame) async def recv_output(self) -> OutputFrame | None: @@ -278,6 +276,10 @@ async def _input_loop(self, pipeline: Pipeline, overlay: LoadingOverlayRenderer) if isinstance(input, VideoFrame): input.log_timestamps["pre_process_frame"] = time.time() + # Move CPU tensors to GPU before sending to pipeline + if not input.tensor.is_cuda and torch.cuda.is_available(): + input = input.replace_tensor(input.tensor.cuda()) + if self._is_loading() and self._last_params.show_reloading_frame: await self._render_loading_frame(overlay, input) else: @@ -296,8 +298,10 @@ async def _render_loading_frame(self, overlay: LoadingOverlayRenderer, input: Vi w, h = self._last_params.get_output_resolution() loading_tensor = await overlay.render(w, h) - if torch.cuda.is_available() and not loading_tensor.is_cuda: - loading_tensor = loading_tensor.cuda() + + # Move to CPU before sending over multiprocessing queue to avoid CUDA IPC overhead + if loading_tensor.is_cuda: + loading_tensor = loading_tensor.cpu() out_frame = input.replace_tensor(loading_tensor) out = VideoOutput(out_frame, self.request_id, is_loading_frame=True) @@ -317,8 +321,9 @@ async def _output_loop(self, pipeline: Pipeline, overlay: LoadingOverlayRenderer continue overlay.end_reload() - if isinstance(out, VideoOutput) and not out.tensor.is_cuda and torch.cuda.is_available(): - out = out.replace_tensor(out.tensor.cuda()) + # Move to CPU before sending over multiprocessing queue to avoid CUDA IPC overhead + if isinstance(out, VideoOutput) and out.tensor.is_cuda: + out = out.replace_tensor(out.tensor.cpu()) out.log_timestamps["post_process_frame"] = time.time() self._try_queue_put(self.output_queue, out)