From 914fd8584f73a1d9e6d0d65bdae548d57c18e0a0 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Tue, 23 Sep 2025 11:42:38 -0400 Subject: [PATCH 01/44] refactor(workflows): remove ai-runner workflow trigger from docker.yaml (#414) --- .github/workflows/docker.yaml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 608a0edc4..68f374747 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -83,28 +83,6 @@ jobs: cache-from: type=registry,ref=livepeer/comfyui-base:build-cache cache-to: type=registry,mode=max,ref=livepeer/comfyui-base:build-cache - trigger: - name: Trigger ai-runner workflow - needs: base - if: ${{ github.repository == 'livepeer/comfystream' }} - runs-on: ubuntu-latest - steps: - - name: Send workflow dispatch event to ai-runner - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.CI_GITHUB_TOKEN }} - script: | - await github.rest.actions.createWorkflowDispatch({ - owner: context.repo.owner, - repo: "ai-runner", - workflow_id: "comfyui-trigger.yaml", - ref: "main", - inputs: { - "comfyui-base-digest": "${{ needs.base.outputs.image-digest }}", - "triggering-branch": "${{ github.head_ref || github.ref_name }}", - }, - }); - comfystream: name: comfystream image needs: base From 4f486957feea087ecc0652d93fcaccd9d4782e3e Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Tue, 23 Sep 2025 16:42:32 -0400 Subject: [PATCH 02/44] fix(nodes): add input timeout to LoadTensor and LoadAudioTensor nodes (#423) * nodes(video): add input timeout to LoadTensor and SaveTensor nodes --- nodes/audio_utils/load_audio_tensor.py | 79 ++++++++++++++---------- nodes/tensor_utils/load_tensor.py | 33 +++++++--- server/app.py | 6 +- server/byoc.py | 5 ++ server/frame_processor.py | 13 +--- src/comfystream/__init__.py | 5 +- src/comfystream/client.py | 6 +- src/comfystream/exceptions.py | 20 ++++++ src/comfystream/server/utils/__init__.py | 2 +- src/comfystream/server/utils/utils.py | 39 ++++++++++++ src/comfystream/utils.py | 1 + 11 files changed, 156 insertions(+), 53 deletions(-) create mode 100644 src/comfystream/exceptions.py diff --git a/nodes/audio_utils/load_audio_tensor.py b/nodes/audio_utils/load_audio_tensor.py index ece7fca31..eed09eea9 100644 --- a/nodes/audio_utils/load_audio_tensor.py +++ b/nodes/audio_utils/load_audio_tensor.py @@ -1,71 +1,88 @@ import numpy as np import torch - +import queue from comfystream import tensor_cache +from comfystream.exceptions import ComfyStreamInputTimeoutError, ComfyStreamAudioBufferError + class LoadAudioTensor: - CATEGORY = "audio_utils" + CATEGORY = "ComfyStream/Loaders" RETURN_TYPES = ("AUDIO",) RETURN_NAMES = ("audio",) FUNCTION = "execute" + DESCRIPTION = "Load audio tensor from ComfyStream input with timeout." def __init__(self): self.audio_buffer = np.empty(0, dtype=np.int16) self.buffer_samples = None self.sample_rate = None + self.leftover = np.empty(0, dtype=np.int16) @classmethod - def INPUT_TYPES(s): + def INPUT_TYPES(cls): return { "required": { - "buffer_size": ("FLOAT", {"default": 500.0}), + "buffer_size": ("FLOAT", { + "default": 500.0, + "tooltip": "Audio buffer size in milliseconds" + }), + }, + "optional": { + "timeout_seconds": ("FLOAT", { + "default": 1.0, + "min": 0.1, + "max": 30.0, + "step": 0.1, + "tooltip": "Timeout in seconds" + }), } } @classmethod - def IS_CHANGED(**kwargs): + def IS_CHANGED(cls, **kwargs): return float("nan") - - def execute(self, buffer_size): + + def execute(self, buffer_size: float, timeout_seconds: float = 1.0): + # Initialize if needed if self.sample_rate is None or self.buffer_samples is None: - frame = tensor_cache.audio_inputs.get(block=True) - self.sample_rate = frame.sample_rate - self.buffer_samples = int(self.sample_rate * buffer_size / 1000) - self.leftover = frame.side_data.input + try: + frame = tensor_cache.audio_inputs.get(block=True, timeout=timeout_seconds) + self.sample_rate = frame.sample_rate + self.buffer_samples = int(self.sample_rate * buffer_size / 1000) + self.leftover = frame.side_data.input + except queue.Empty: + raise ComfyStreamInputTimeoutError("audio", timeout_seconds) - if self.leftover.shape[0] < self.buffer_samples: + # Use leftover data if available + if self.leftover.shape[0] >= self.buffer_samples: + buffered_audio = self.leftover[:self.buffer_samples] + self.leftover = self.leftover[self.buffer_samples:] + else: + # Collect more audio chunks chunks = [self.leftover] if self.leftover.size > 0 else [] total_samples = self.leftover.shape[0] while total_samples < self.buffer_samples: - frame = tensor_cache.audio_inputs.get(block=True) - if frame.sample_rate != self.sample_rate: - raise ValueError("Sample rate mismatch") - chunks.append(frame.side_data.input) - total_samples += frame.side_data.input.shape[0] + try: + frame = tensor_cache.audio_inputs.get(block=True, timeout=timeout_seconds) + if frame.sample_rate != self.sample_rate: + raise ValueError(f"Sample rate mismatch: expected {self.sample_rate}Hz, got {frame.sample_rate}Hz") + chunks.append(frame.side_data.input) + total_samples += frame.side_data.input.shape[0] + except queue.Empty: + raise ComfyStreamAudioBufferError(timeout_seconds, self.buffer_samples, total_samples) merged_audio = np.concatenate(chunks, dtype=np.int16) buffered_audio = merged_audio[:self.buffer_samples] - self.leftover = merged_audio[self.buffer_samples:] - else: - buffered_audio = self.leftover[:self.buffer_samples] - self.leftover = self.leftover[self.buffer_samples:] + self.leftover = merged_audio[self.buffer_samples:] if merged_audio.shape[0] > self.buffer_samples else np.empty(0, dtype=np.int16) - # Convert numpy array to torch tensor and normalize int16 to float32 + # Convert to ComfyUI AUDIO format waveform_tensor = torch.from_numpy(buffered_audio.astype(np.float32) / 32768.0) # Ensure proper tensor shape: (batch, channels, samples) if waveform_tensor.dim() == 1: - # Mono: (samples,) -> (1, 1, samples) waveform_tensor = waveform_tensor.unsqueeze(0).unsqueeze(0) elif waveform_tensor.dim() == 2: - # Assume (channels, samples) and add batch dimension waveform_tensor = waveform_tensor.unsqueeze(0) - # Return AUDIO dictionary format - audio_dict = { - "waveform": waveform_tensor, - "sample_rate": self.sample_rate - } - - return (audio_dict,) + return ({"waveform": waveform_tensor, "sample_rate": self.sample_rate},) \ No newline at end of file diff --git a/nodes/tensor_utils/load_tensor.py b/nodes/tensor_utils/load_tensor.py index c39fe8a1d..9923a996a 100644 --- a/nodes/tensor_utils/load_tensor.py +++ b/nodes/tensor_utils/load_tensor.py @@ -1,20 +1,37 @@ +import torch +import queue from comfystream import tensor_cache +from comfystream.exceptions import ComfyStreamInputTimeoutError class LoadTensor: - CATEGORY = "tensor_utils" + CATEGORY = "ComfyStream/Loaders" RETURN_TYPES = ("IMAGE",) FUNCTION = "execute" + DESCRIPTION = "Load image tensor from ComfyStream input with timeout." @classmethod - def INPUT_TYPES(s): - return {} + def INPUT_TYPES(cls): + return { + "optional": { + "timeout_seconds": ("FLOAT", { + "default": 1.0, + "min": 0.1, + "max": 30.0, + "step": 0.1, + "tooltip": "Timeout in seconds" + }), + } + } @classmethod - def IS_CHANGED(): + def IS_CHANGED(cls, **kwargs): return float("nan") - def execute(self): - frame = tensor_cache.image_inputs.get(block=True) - frame.side_data.skipped = False - return (frame.side_data.input,) + def execute(self, timeout_seconds: float = 1.0): + try: + frame = tensor_cache.image_inputs.get(block=True, timeout=timeout_seconds) + frame.side_data.skipped = False + return (frame.side_data.input,) + except queue.Empty: + raise ComfyStreamInputTimeoutError("video", timeout_seconds) diff --git a/server/app.py b/server/app.py index a3a42fc44..c07065695 100644 --- a/server/app.py +++ b/server/app.py @@ -29,7 +29,7 @@ from aiortc.rtcrtpsender import RTCRtpSender from comfystream.pipeline import Pipeline from twilio.rest import Client -from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter +from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter, ComfyStreamTimeoutFilter from comfystream.server.metrics import MetricsManager, StreamStatsManager import time @@ -694,6 +694,10 @@ def force_print(*args, **kwargs): if args.comfyui_log_level: log_level = logging._nameToLevel.get(args.comfyui_log_level.upper()) logging.getLogger("comfy").setLevel(log_level) + + # Add ComfyStream timeout filter to suppress verbose execution logging + logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) + logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) if args.comfyui_inference_log_level: app["comfui_inference_log_level"] = args.comfyui_inference_log_level diff --git a/server/byoc.py b/server/byoc.py index 0735674b4..19a166be8 100644 --- a/server/byoc.py +++ b/server/byoc.py @@ -14,6 +14,7 @@ from pytrickle.utils.register import RegisterCapability from pytrickle.frame_skipper import FrameSkipConfig from frame_processor import ComfyStreamFrameProcessor +from comfystream.server.utils import ComfyStreamTimeoutFilter logger = logging.getLogger(__name__) @@ -117,6 +118,10 @@ def main(): if args.comfyui_log_level: log_level = logging._nameToLevel.get(args.comfyui_log_level.upper()) logging.getLogger("comfy").setLevel(log_level) + + # Add ComfyStream timeout filter to suppress verbose execution logging + logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) + logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) def force_print(*args, **kwargs): print(*args, **kwargs, flush=True) diff --git a/server/frame_processor.py b/server/frame_processor.py index bac139d4a..6cc79a712 100644 --- a/server/frame_processor.py +++ b/server/frame_processor.py @@ -157,28 +157,21 @@ async def load_model(self, **kwargs): ) async def warmup(self): - """Public warmup method that triggers pipeline warmup.""" + """Warm up the pipeline.""" if not self.pipeline: logger.warning("Warmup requested before pipeline initialization") return logger.info("Running pipeline warmup...") - """Run pipeline warmup.""" try: capabilities = self.pipeline.get_workflow_io_capabilities() - logger.info(f"Detected I/O capabilities for warmup: {capabilities}") + logger.info(f"Detected I/O capabilities: {capabilities}") - # Warm video if there are video inputs or outputs if capabilities.get("video", {}).get("input") or capabilities.get("video", {}).get("output"): - logger.info("Running video warmup...") await self.pipeline.warm_video() - logger.info("Video warmup completed") - # Warm audio if there are audio inputs or outputs if capabilities.get("audio", {}).get("input") or capabilities.get("audio", {}).get("output"): - logger.info("Running audio warmup...") await self.pipeline.warm_audio() - logger.info("Audio warmup completed") except Exception as e: logger.error(f"Warmup failed: {e}") @@ -265,7 +258,7 @@ async def _process_prompts(self, prompts): """Process and set prompts in the pipeline.""" try: converted = convert_prompt(prompts, return_dict=True) - + # Set prompts in pipeline await self.pipeline.set_prompts([converted]) logger.info(f"Prompts set successfully: {list(prompts.keys())}") diff --git a/src/comfystream/__init__.py b/src/comfystream/__init__.py index b58bf2e44..8aee624cf 100644 --- a/src/comfystream/__init__.py +++ b/src/comfystream/__init__.py @@ -3,6 +3,7 @@ from .server.utils import temporary_log_level from .server.utils import FPSMeter from .server.metrics import MetricsManager, StreamStatsManager +from .exceptions import ComfyStreamInputTimeoutError, ComfyStreamAudioBufferError __all__ = [ 'ComfyStreamClient', @@ -10,5 +11,7 @@ 'temporary_log_level', 'FPSMeter', 'MetricsManager', - 'StreamStatsManager' + 'StreamStatsManager', + 'ComfyStreamInputTimeoutError', + 'ComfyStreamAudioBufferError' ] diff --git a/src/comfystream/client.py b/src/comfystream/client.py index b5c7dca7f..291ecdda7 100644 --- a/src/comfystream/client.py +++ b/src/comfystream/client.py @@ -4,6 +4,7 @@ from comfystream import tensor_cache from comfystream.utils import convert_prompt +from comfystream.exceptions import ComfyStreamInputTimeoutError from comfy.api.components.schema.prompt import PromptDictInput from comfy.cli_args_types import Configuration @@ -68,6 +69,9 @@ async def run_prompt(self, prompt_index: int): await self.comfy_client.queue_prompt(self.current_prompts[prompt_index]) except asyncio.CancelledError: raise + except ComfyStreamInputTimeoutError: + # Timeout errors are expected during stream switching - just continue + continue except Exception as e: await self.cleanup() logger.error(f"Error running prompt: {str(e)}") @@ -265,4 +269,4 @@ async def get_available_nodes(self): except Exception as e: logger.error(f"Error getting node info: {str(e)}") - return {} + return {} \ No newline at end of file diff --git a/src/comfystream/exceptions.py b/src/comfystream/exceptions.py new file mode 100644 index 000000000..8382a4c30 --- /dev/null +++ b/src/comfystream/exceptions.py @@ -0,0 +1,20 @@ +"""ComfyStream specific exceptions.""" + + +class ComfyStreamInputTimeoutError(Exception): + """Raised when input tensors are not available within timeout.""" + + def __init__(self, input_type: str, timeout_seconds: float): + self.input_type = input_type + self.timeout_seconds = timeout_seconds + message = f"No {input_type} frames available after {timeout_seconds}s timeout" + super().__init__(message) + + +class ComfyStreamAudioBufferError(ComfyStreamInputTimeoutError): + """Audio buffer insufficient data error.""" + + def __init__(self, timeout_seconds: float, needed_samples: int, available_samples: int): + self.needed_samples = needed_samples + self.available_samples = available_samples + super().__init__("audio", timeout_seconds) diff --git a/src/comfystream/server/utils/__init__.py b/src/comfystream/server/utils/__init__.py index daa71bb1e..31a559085 100644 --- a/src/comfystream/server/utils/__init__.py +++ b/src/comfystream/server/utils/__init__.py @@ -1,2 +1,2 @@ -from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level +from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level, ComfyStreamTimeoutFilter from .fps_meter import FPSMeter diff --git a/src/comfystream/server/utils/utils.py b/src/comfystream/server/utils/utils.py index c7a7ac304..974aa74a3 100644 --- a/src/comfystream/server/utils/utils.py +++ b/src/comfystream/server/utils/utils.py @@ -83,3 +83,42 @@ async def temporary_log_level(logger_name: str, level: int): finally: if level is not None: logger.setLevel(original_level) + + +class ComfyStreamTimeoutFilter(logging.Filter): + """Filter to suppress verbose ComfyUI execution logs for ComfyStream timeout exceptions.""" + + def filter(self, record): + """Filter out ComfyUI execution error logs for ComfyStream timeout exceptions.""" + # Only filter ERROR level messages from ComfyUI execution system + if record.levelno != logging.ERROR: + return True + + # Check if this is from ComfyUI execution system + if not (record.name.startswith("comfy") and ("execution" in record.name or record.name == "comfy")): + return True + + # Get the full message including any exception info + message = record.getMessage() + + # Check if this is a ComfyStream timeout-related error + timeout_indicators = [ + "ComfyStreamInputTimeoutError", + "ComfyStreamAudioBufferError", + "No video frames available", + "No audio frames available" + ] + + # Suppress if any timeout indicator is found in the message + for indicator in timeout_indicators: + if indicator in message: + return False + + # Also check the exception info if present + if record.exc_info and record.exc_info[1]: + exc_str = str(record.exc_info[1]) + for indicator in timeout_indicators: + if indicator in exc_str: + return False + + return True diff --git a/src/comfystream/utils.py b/src/comfystream/utils.py index 7d8800c4a..c2cecd05b 100644 --- a/src/comfystream/utils.py +++ b/src/comfystream/utils.py @@ -73,6 +73,7 @@ def convert_prompt(prompt: PromptDictInput, return_dict: bool = False) -> Prompt for key in convertible_keys["PreviewImage"] + convertible_keys["SaveImage"]: node = prompt[key] prompt[key] = create_save_tensor_node(node["inputs"]) + # Return dict if requested (for downstream components that expect plain dicts) if return_dict: From bcebd510e2d8bcf147f4d4d7b36408f7b32f61eb Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Thu, 25 Sep 2025 15:23:29 -0400 Subject: [PATCH 03/44] cleanup(nodes): Improve input exception logging (#427) * refactor(exceptions): improve timeout error logging for ComfyStreamAudioBufferError, ComfyStreamInputTimeoutError --- server/app.py | 4 +- server/byoc.py | 3 +- server/frame_processor.py | 8 ++ src/comfystream/client.py | 1 + src/comfystream/exceptions.py | 113 ++++++++++++++++++++++- src/comfystream/server/utils/__init__.py | 2 +- src/comfystream/server/utils/utils.py | 38 -------- 7 files changed, 123 insertions(+), 46 deletions(-) diff --git a/server/app.py b/server/app.py index c07065695..028f12bbc 100644 --- a/server/app.py +++ b/server/app.py @@ -29,7 +29,8 @@ from aiortc.rtcrtpsender import RTCRtpSender from comfystream.pipeline import Pipeline from twilio.rest import Client -from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter, ComfyStreamTimeoutFilter +from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter +from comfystream.exceptions import ComfyStreamTimeoutFilter from comfystream.server.metrics import MetricsManager, StreamStatsManager import time @@ -697,7 +698,6 @@ def force_print(*args, **kwargs): # Add ComfyStream timeout filter to suppress verbose execution logging logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) - logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) if args.comfyui_inference_log_level: app["comfui_inference_log_level"] = args.comfyui_inference_log_level diff --git a/server/byoc.py b/server/byoc.py index 19a166be8..895667482 100644 --- a/server/byoc.py +++ b/server/byoc.py @@ -14,7 +14,7 @@ from pytrickle.utils.register import RegisterCapability from pytrickle.frame_skipper import FrameSkipConfig from frame_processor import ComfyStreamFrameProcessor -from comfystream.server.utils import ComfyStreamTimeoutFilter +from comfystream.exceptions import ComfyStreamTimeoutFilter logger = logging.getLogger(__name__) @@ -121,7 +121,6 @@ def main(): # Add ComfyStream timeout filter to suppress verbose execution logging logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) - logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) def force_print(*args, **kwargs): print(*args, **kwargs, flush=True) diff --git a/server/frame_processor.py b/server/frame_processor.py index 6cc79a712..65d1c0ac1 100644 --- a/server/frame_processor.py +++ b/server/frame_processor.py @@ -113,6 +113,14 @@ async def on_stream_stop(self): # Set stop event to signal all background tasks to stop self._stop_event.set() + # Stop the ComfyStream client's prompt execution + if self.pipeline and self.pipeline.client: + logger.info("Stopping ComfyStream client prompt execution") + try: + await self.pipeline.client.cleanup() + except Exception as e: + logger.error(f"Error stopping ComfyStream client: {e}") + # Stop text forwarder await self._stop_text_forwarder() diff --git a/src/comfystream/client.py b/src/comfystream/client.py index 291ecdda7..7686fca18 100644 --- a/src/comfystream/client.py +++ b/src/comfystream/client.py @@ -71,6 +71,7 @@ async def run_prompt(self, prompt_index: int): raise except ComfyStreamInputTimeoutError: # Timeout errors are expected during stream switching - just continue + logger.info(f"Input for prompt {prompt_index} timed out, continuing") continue except Exception as e: await self.cleanup() diff --git a/src/comfystream/exceptions.py b/src/comfystream/exceptions.py index 8382a4c30..aaf20c844 100644 --- a/src/comfystream/exceptions.py +++ b/src/comfystream/exceptions.py @@ -1,20 +1,127 @@ """ComfyStream specific exceptions.""" +import logging +from typing import Dict, Any, Optional + + +def log_comfystream_error( + exception: Exception, + logger: Optional[logging.Logger] = None, + level: int = logging.ERROR +) -> None: + """ + Centralized logging function for ComfyStream exceptions. + + Args: + exception: The exception to log + logger: Optional logger to use (defaults to module logger) + level: Log level (defaults to ERROR) + """ + if logger is None: + logger = logging.getLogger(__name__) + + # If it's a ComfyStream timeout error with structured details, use its logging method + if isinstance(exception, ComfyStreamInputTimeoutError): + exception.log_error(logger) + else: + # For other exceptions, provide basic logging + logger.log(level, f"ComfyStream error: {type(exception).__name__}: {str(exception)}") + class ComfyStreamInputTimeoutError(Exception): """Raised when input tensors are not available within timeout.""" - def __init__(self, input_type: str, timeout_seconds: float): + def __init__( + self, + input_type: str, + timeout_seconds: float, + details: Optional[Dict[str, Any]] = None + ): self.input_type = input_type self.timeout_seconds = timeout_seconds + self.details = details or {} message = f"No {input_type} frames available after {timeout_seconds}s timeout" super().__init__(message) + + def get_log_details(self) -> Dict[str, Any]: + """Get structured details for logging.""" + base_details = { + "input_type": self.input_type, + "timeout_seconds": self.timeout_seconds + } + base_details.update(self.details) + return base_details + + def log_error(self, logger: Optional[logging.Logger] = None) -> None: + """Log the error with detailed information.""" + if logger is None: + logger = logging.getLogger(__name__) + + details = self.get_log_details() + detail_str = ", ".join(f"{k}={v}" for k, v in details.items()) + logger.error(f"ComfyStream timeout error: {str(self)} | Details: {detail_str}") class ComfyStreamAudioBufferError(ComfyStreamInputTimeoutError): """Audio buffer insufficient data error.""" - def __init__(self, timeout_seconds: float, needed_samples: int, available_samples: int): + def __init__( + self, + timeout_seconds: float, + needed_samples: int, + available_samples: int + ): self.needed_samples = needed_samples self.available_samples = available_samples - super().__init__("audio", timeout_seconds) + + # Pass audio-specific details to the base class + audio_details = { + "needed_samples": needed_samples, + "available_samples": available_samples, + } + super().__init__("audio", timeout_seconds, details=audio_details) + + def get_log_details(self) -> Dict[str, Any]: + """Get structured details for logging, with audio-specific formatting.""" + details = super().get_log_details() + return details + + +class ComfyStreamTimeoutFilter(logging.Filter): + """Filter to suppress verbose ComfyUI execution logs for ComfyStream timeout exceptions.""" + + def filter(self, record): + """Filter out ComfyUI execution error logs for ComfyStream timeout exceptions.""" + try: + # Only filter ERROR level messages from ComfyUI execution system + if record.levelno != logging.ERROR: + return True + + # Check if this is from ComfyUI execution system + if not (record.name.startswith("comfy") and ("execution" in record.name or record.name == "comfy")): + return True + + # Get the full message including any exception info + message = record.getMessage() + + # Simple check: if this log contains ComfyStreamAudioBufferError or ComfyStreamInputTimeoutError, suppress it + if ("ComfyStreamAudioBufferError" in message or + "ComfyStreamInputTimeoutError" in message): + return False + + # Also check the exception info if present + if record.exc_info and record.exc_info[1]: + exc_str = str(record.exc_info[1]) + exc_type = str(type(record.exc_info[1])) + + if ("ComfyStreamAudioBufferError" in exc_str or + "ComfyStreamInputTimeoutError" in exc_str or + "ComfyStreamAudioBufferError" in exc_type or + "ComfyStreamInputTimeoutError" in exc_type): + return False + + return True + except Exception as e: + # If filter fails, allow the log through and print the error + print(f"[FILTER ERROR] Filter failed: {e}") + return True diff --git a/src/comfystream/server/utils/__init__.py b/src/comfystream/server/utils/__init__.py index 31a559085..daa71bb1e 100644 --- a/src/comfystream/server/utils/__init__.py +++ b/src/comfystream/server/utils/__init__.py @@ -1,2 +1,2 @@ -from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level, ComfyStreamTimeoutFilter +from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level from .fps_meter import FPSMeter diff --git a/src/comfystream/server/utils/utils.py b/src/comfystream/server/utils/utils.py index 974aa74a3..96e6661e9 100644 --- a/src/comfystream/server/utils/utils.py +++ b/src/comfystream/server/utils/utils.py @@ -84,41 +84,3 @@ async def temporary_log_level(logger_name: str, level: int): if level is not None: logger.setLevel(original_level) - -class ComfyStreamTimeoutFilter(logging.Filter): - """Filter to suppress verbose ComfyUI execution logs for ComfyStream timeout exceptions.""" - - def filter(self, record): - """Filter out ComfyUI execution error logs for ComfyStream timeout exceptions.""" - # Only filter ERROR level messages from ComfyUI execution system - if record.levelno != logging.ERROR: - return True - - # Check if this is from ComfyUI execution system - if not (record.name.startswith("comfy") and ("execution" in record.name or record.name == "comfy")): - return True - - # Get the full message including any exception info - message = record.getMessage() - - # Check if this is a ComfyStream timeout-related error - timeout_indicators = [ - "ComfyStreamInputTimeoutError", - "ComfyStreamAudioBufferError", - "No video frames available", - "No audio frames available" - ] - - # Suppress if any timeout indicator is found in the message - for indicator in timeout_indicators: - if indicator in message: - return False - - # Also check the exception info if present - if record.exc_info and record.exc_info[1]: - exc_str = str(record.exc_info[1]) - for indicator in timeout_indicators: - if indicator in exc_str: - return False - - return True From 5c66950c44261572fc04f9af354e236ea9527357 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Fri, 26 Sep 2025 12:59:05 -0400 Subject: [PATCH 04/44] Update documentation links to docs.comfystream.org (#431) Co-authored-by: Cursor Agent --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 52c864f13..a9edf4be5 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This repo also includes a WebRTC server and UI that uses comfystream to support Refer to [.devcontainer/README.md](.devcontainer/README.md) to setup ComfyStream in a devcontainer using a pre-configured ComfyUI docker environment. -For other installation options, refer to [Install ComfyUI and ComfyStream](https://pipelines.livepeer.org/docs/technical/install/local-testing) in the Livepeer pipelines documentation. +For other installation options, refer to [Install ComfyUI and ComfyStream](https://docs.comfystream.org/technical/get-started/install) in the ComfyStream documentation. For additional information, refer to the remaining sections below. @@ -35,7 +35,7 @@ For additional information, refer to the remaining sections below. You can quickly deploy ComfyStream using the docker image `livepeer/comfystream` -Refer to the documentation at [https://pipelines.livepeer.org/docs/technical/getting-started/install-comfystream](https://pipelines.livepeer.org/docs/technical/getting-started/install-comfystream) for instructions to run locally or on a remote server. +Refer to the documentation at [https://docs.comfystream.org/technical/get-started/install](https://docs.comfystream.org/technical/get-started/install) for instructions to run locally or on a remote server. #### RunPod From e25d25a19659ce4d5f1a5cb882e3abb1cf7b762d Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Fri, 26 Sep 2025 14:24:43 -0400 Subject: [PATCH 05/44] launch.json: update capability name for byoc (#437) --- .vscode/launch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4d442c585..f05e02f5e 100755 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -56,7 +56,7 @@ "env": { "ORCH_URL": "https://172.17.0.1:9995", "ORCH_SECRET": "orch-secret", - "CAPABILITY_NAME": "comfystream-byoc-processor", + "CAPABILITY_NAME": "comfystream", "CAPABILITY_DESCRIPTION": "ComfyUI streaming processor for BYOC mode", "CAPABILITY_URL": "http://172.17.0.1:8000", "CAPABILITY_PRICE_PER_UNIT": "0", From 29a51d5592624c6ae2d1c8a75fde1517f4ba653a Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Fri, 26 Sep 2025 15:15:54 -0400 Subject: [PATCH 06/44] fix(docker): add missing development libraries for Cairo and Pango (#438) Co-authored-by: Jason Stone --- docker/Dockerfile.base | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index c8f6b7ff1..7d3facdac 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -27,6 +27,12 @@ RUN apt update && apt install -yqq --no-install-recommends \ swig \ libprotobuf-dev \ protobuf-compiler \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf2.0-dev \ + libffi-dev \ + libgirepository1.0-dev \ + pkg-config \ && rm -rf /var/lib/apt/lists/* #enable opengl support with nvidia gpu From 70f924ca702db8f7d64fc889c357c1373e7bc5af Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sat, 27 Sep 2025 06:40:27 +0300 Subject: [PATCH 07/44] feat: GitHub workflow opencv-cuda compilation (#432) * feat: GitHub workflow for automated OpenCV CUDA builds --- .github/workflows/opencv-cuda-artifact.yml | 196 +++++++++++++++++++++ docker/Dockerfile.opencv | 124 +++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 .github/workflows/opencv-cuda-artifact.yml create mode 100644 docker/Dockerfile.opencv diff --git a/.github/workflows/opencv-cuda-artifact.yml b/.github/workflows/opencv-cuda-artifact.yml new file mode 100644 index 000000000..5572d09e7 --- /dev/null +++ b/.github/workflows/opencv-cuda-artifact.yml @@ -0,0 +1,196 @@ +name: Build OpenCV CUDA Artifact + +on: + push: + branches: + - main + paths: + - 'docker/Dockerfile.opencv' + - 'docker/Dockerfile.base' + pull_request: + branches: + - main + paths: + - 'docker/Dockerfile.opencv' + - 'docker/Dockerfile.base' + workflow_dispatch: + inputs: + python_version: + description: 'Python version to build' + required: false + default: '3.12' + type: string + cuda_version: + description: 'CUDA version to build' + required: false + default: '12.8' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + PYTHON_VERSION: ${{ github.event.inputs.python_version || '3.12' }} + CUDA_VERSION: ${{ github.event.inputs.cuda_version || '12.8' }} + +jobs: + build-opencv-artifact: + name: Build OpenCV CUDA Artifact + runs-on: [self-hosted, linux, gpu] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build OpenCV CUDA Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile.opencv + build-args: | + BASE_IMAGE=nvidia/cuda:${{ env.CUDA_VERSION }}.1-cudnn-devel-ubuntu22.04 + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + CUDA_VERSION=${{ env.CUDA_VERSION }} + tags: opencv-cuda-artifact:latest + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Extract OpenCV libraries from Docker container + run: | + echo "Creating temporary container..." + docker create --name opencv-extract opencv-cuda-artifact:latest + + echo "Creating workspace directory..." + mkdir -p ./opencv-artifacts + + # Try to copy from system installation + docker cp opencv-extract:/usr/local/lib/python${{ env.PYTHON_VERSION }}/site-packages/cv2 ./opencv-artifacts/cv2 || echo "cv2 not found in system site-packages" + + echo "Copying OpenCV source directories..." + # Copy opencv and opencv_contrib source directories + docker cp opencv-extract:/workspace/opencv ./opencv-artifacts/ || echo "opencv source not found" + docker cp opencv-extract:/workspace/opencv_contrib ./opencv-artifacts/ || echo "opencv_contrib source not found" + + echo "Cleaning up container..." + docker rm opencv-extract + + echo "Contents of opencv-artifacts:" + ls -la ./opencv-artifacts/ + + - name: Create tarball artifact + run: | + echo "Creating opencv-cuda-release.tar.gz..." + cd ./opencv-artifacts + tar -czf ../opencv-cuda-release.tar.gz . || echo "Failed to create tarball" + cd .. + + echo "Generating checksums..." + sha256sum opencv-cuda-release.tar.gz > opencv-cuda-release.tar.gz.sha256 + md5sum opencv-cuda-release.tar.gz > opencv-cuda-release.tar.gz.md5 + + echo "Verifying archive contents..." + echo "Archive size: $(ls -lh opencv-cuda-release.tar.gz | awk '{print $5}')" + echo "First 20 files in archive:" + tar -tzf opencv-cuda-release.tar.gz | head -20 + + - name: Extract and verify tarball + run: | + echo "Testing tarball extraction..." + mkdir -p test-extract + cd test-extract + tar -xzf ../opencv-cuda-release.tar.gz + echo "Extracted contents:" + find . -maxdepth 2 -type d | sort + cd .. + rm -rf test-extract + + - name: Upload OpenCV CUDA Release Artifact + uses: actions/upload-artifact@v4 + with: + name: opencv-cuda-release-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: | + opencv-cuda-release.tar.gz + opencv-cuda-release.tar.gz.sha256 + opencv-cuda-release.tar.gz.md5 + retention-days: 30 + + - name: Create Release Notes + run: | + cat > release-info.txt << EOF + OpenCV CUDA Release Artifact + + Build Details: + - Python Version: ${{ env.PYTHON_VERSION }} + - CUDA Version: ${{ env.CUDA_VERSION }} + - OpenCV Version: 4.11.0 + - Built on: $(date -u) + - Commit SHA: ${{ github.sha }} + + Contents: + - cv2: Python OpenCV module with CUDA support + - opencv: OpenCV source code + - opencv_contrib: OpenCV contrib modules source + - lib: Compiled OpenCV libraries + - include: OpenCV header files + + Installation: + 1. Download opencv-cuda-release.tar.gz + 2. Extract: tar -xzf opencv-cuda-release.tar.gz + 3. Copy cv2 to your Python environment's site-packages + 4. Ensure CUDA libraries are in your system PATH + + Checksums: + SHA256: $(cat opencv-cuda-release.tar.gz.sha256) + MD5: $(cat opencv-cuda-release.tar.gz.md5) + EOF + + - name: Upload Release Info + uses: actions/upload-artifact@v4 + with: + name: release-info-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: release-info.txt + retention-days: 30 + + create-release-draft: + name: Create Release Draft + needs: build-opencv-artifact + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: opencv-cuda-release-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: ./artifacts + + - name: Download release info + uses: actions/download-artifact@v4 + with: + name: release-info-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: ./artifacts + + - name: Create Release Draft + uses: softprops/action-gh-release@v1 + with: + tag_name: opencv-cuda-v${{ env.PYTHON_VERSION }}-${{ env.CUDA_VERSION }}-${{ github.run_number }} + name: OpenCV CUDA Release - Python ${{ env.PYTHON_VERSION }} CUDA ${{ env.CUDA_VERSION }} + body_path: ./artifacts/release-info.txt + draft: true + files: | + ./artifacts/opencv-cuda-release.tar.gz + ./artifacts/opencv-cuda-release.tar.gz.sha256 + ./artifacts/opencv-cuda-release.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/docker/Dockerfile.opencv b/docker/Dockerfile.opencv new file mode 100644 index 000000000..848db1fe8 --- /dev/null +++ b/docker/Dockerfile.opencv @@ -0,0 +1,124 @@ +ARG BASE_IMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 \ + CONDA_VERSION=latest \ + PYTHON_VERSION=3.12 \ + CUDA_VERSION=12.8 + +FROM "${BASE_IMAGE}" + +ARG CONDA_VERSION \ + PYTHON_VERSION \ + CUDA_VERSION + +ENV DEBIAN_FRONTEND=noninteractive \ + CONDA_VERSION="${CONDA_VERSION}" \ + PATH="/workspace/miniconda3/bin:${PATH}" \ + PYTHON_VERSION="${PYTHON_VERSION}" \ + CUDA_VERSION="${CUDA_VERSION}" + +# System dependencies +RUN apt update && apt install -yqq \ + git \ + wget \ + nano \ + socat \ + libsndfile1 \ + build-essential \ + llvm \ + tk-dev \ + cmake \ + libgflags-dev \ + libgoogle-glog-dev \ + libjpeg-dev \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libswscale-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /workspace/comfystream && \ + wget "https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh" -O /tmp/miniconda.sh && \ + bash /tmp/miniconda.sh -b -p /workspace/miniconda3 && \ + eval "$(/workspace/miniconda3/bin/conda shell.bash hook)" && \ + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main && \ + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r && \ + conda create -n comfystream python="${PYTHON_VERSION}" -c conda-forge -y && \ + rm /tmp/miniconda.sh && \ + conda run -n comfystream --no-capture-output pip install numpy==1.26.4 aiortc aiohttp requests tqdm pyyaml --root-user-action=ignore + +# Clone ComfyUI +ADD --link https://github.com/comfyanonymous/ComfyUI.git /workspace/ComfyUI + +# OpenCV with CUDA support +WORKDIR /workspace + +# Clone OpenCV repositories +RUN git clone --depth 1 --branch 4.11.0 https://github.com/opencv/opencv.git && \ + git clone --depth 1 --branch 4.11.0 https://github.com/opencv/opencv_contrib.git + +# Create build directory +RUN mkdir -p /workspace/opencv/build + +# Create a toolchain file with absolute path +RUN echo '# Custom toolchain file to exclude Conda paths\n\ +\n\ +# Set system compilers\n\ +set(CMAKE_C_COMPILER "/usr/bin/gcc")\n\ +set(CMAKE_CXX_COMPILER "/usr/bin/g++")\n\ +\n\ +# Set system root directories\n\ +set(CMAKE_FIND_ROOT_PATH "/usr")\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\n\ +\n\ +# Explicitly exclude Conda paths\n\ +list(APPEND CMAKE_IGNORE_PATH \n\ + "/workspace/miniconda3"\n\ + "/workspace/miniconda3/envs"\n\ + "/workspace/miniconda3/envs/comfystream"\n\ + "/workspace/miniconda3/envs/comfystream/lib"\n\ +)\n\ +\n\ +# Set RPATH settings\n\ +set(CMAKE_SKIP_BUILD_RPATH FALSE)\n\ +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)\n\ +set(CMAKE_INSTALL_RPATH "/usr/local/lib:/usr/lib/x86_64-linux-gnu")\n\ +set(PYTHON_LIBRARY "/workspace/miniconda3/envs/comfystream/lib/")\n\ +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)' > /workspace/custom_toolchain.cmake + +# Set environment variables for OpenCV +RUN echo 'export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH' >> /root/.bashrc + +# Build and install OpenCV with CUDA support +RUN cd /workspace/opencv/build && \ + # Build OpenCV + cmake \ + -D CMAKE_TOOLCHAIN_FILE=/workspace/custom_toolchain.cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_CUDA=ON \ + -D WITH_CUDNN=ON \ + -D WITH_CUBLAS=ON \ + -D WITH_TBB=ON \ + -D CUDA_ARCH_LIST="8.0+PTX" \ + -D OPENCV_DNN_CUDA=ON \ + -D OPENCV_ENABLE_NONFREE=ON \ + -D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \ + -D OPENCV_EXTRA_MODULES_PATH=/workspace/opencv_contrib/modules \ + -D PYTHON3_EXECUTABLE=/workspace/miniconda3/envs/comfystream/bin/python3.12 \ + -D PYTHON_INCLUDE_DIR=/workspace/miniconda3/envs/comfystream/include/python3.12 \ + -D PYTHON_LIBRARY=/workspace/miniconda3/envs/comfystream/lib/libpython3.12.so \ + -D HAVE_opencv_python3=ON \ + -D WITH_NVCUVID=OFF \ + -D WITH_NVCUVENC=OFF \ + .. && \ + make -j$(nproc) && \ + make install && \ + ldconfig + +# Configure no environment activation by default +RUN conda config --set auto_activate_base false && \ + conda init bash + +WORKDIR /workspace/comfystream From daf4fc557b92d54fa7c297c0e1f68295b8fd5356 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sat, 27 Sep 2025 08:58:16 +0300 Subject: [PATCH 08/44] chore: update opencv-cuda for Python 3.12, add FasterLivePortrait (#413) * chore: update opencv-cuda for Python 3.12, add FasterLivePortrait add JoyVASA models, add ComfyUI-FasterLivePortrait, add engine build script fix(constraints): add cuda-python dependency to constraints file docker(opencv-cuda): pin move apt pkg installs from entrypoint to dockerfile.base dockerfile: add opencv-cuda installation, update to build v2.1 --------- Co-authored-by: John | Elite Encoder --- configs/models.yaml | 74 +++++++++++++++++++++++++ configs/nodes.yaml | 6 ++ docker/Dockerfile.base | 48 +++++++++------- docker/entrypoint.sh | 19 +++---- src/comfystream/scripts/constraints.txt | 1 + 5 files changed, 118 insertions(+), 30 deletions(-) diff --git a/configs/models.yaml b/configs/models.yaml index 09149f954..be3fce6b0 100644 --- a/configs/models.yaml +++ b/configs/models.yaml @@ -74,6 +74,80 @@ models: path: "text_encoders/CLIPText/model.fp16.safetensors" type: "text_encoder" + # JoyVASA models for ComfyUI-FasterLivePortrait + joyvasa_motion_generator: + name: "JoyVASA Motion Generator" + url: "https://huggingface.co/jdh-algo/JoyVASA/resolve/main/motion_generator/motion_generator_hubert_chinese.pt?download=true" + path: "liveportrait_onnx/joyvasa_models/motion_generator_hubert_chinese.pt" + type: "torch" + + joyvasa_audio_model: + name: "JoyVASA Hubert Chinese" + url: "https://huggingface.co/TencentGameMate/chinese-hubert-base/resolve/main/chinese-hubert-base-fairseq-ckpt.pt?download=true" + path: "liveportrait_onnx/joyvasa_models/chinese-hubert-base-fairseq-ckpt.pt" + type: "torch" + + joyvasa_motion_template: + name: "JoyVASA Motion Template" + url: "https://huggingface.co/jdh-algo/JoyVASA/resolve/main/motion_template/motion_template.pkl?download=true" + path: "liveportrait_onnx/joyvasa_models/motion_template.pkl" + type: "pickle" + + # LivePortrait ONNX models - only necessary to build TRT engines + warping_spade: + name: "WarpingSpadeModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/warping_spade-fix.onnx?download=true" + path: "liveportrait_onnx/warping_spade-fix.onnx" + type: "onnx" + + motion_extractor: + name: "MotionExtractorModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/motion_extractor.onnx?download=true" + path: "liveportrait_onnx/motion_extractor.onnx" + type: "onnx" + + landmark: + name: "LandmarkModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/landmark.onnx?download=true" + path: "liveportrait_onnx/landmark.onnx" + type: "onnx" + + face_analysis_retinaface: + name: "FaceAnalysisModel - RetinaFace" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/retinaface_det_static.onnx?download=true" + path: "liveportrait_onnx/retinaface_det_static.onnx" + type: "onnx" + + face_analysis_2dpose: + name: "FaceAnalysisModel - 2DPose" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/face_2dpose_106_static.onnx?download=true" + path: "liveportrait_onnx/face_2dpose_106_static.onnx" + type: "onnx" + + appearance_feature_extractor: + name: "AppearanceFeatureExtractorModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/appearance_feature_extractor.onnx?download=true" + path: "liveportrait_onnx/appearance_feature_extractor.onnx" + type: "onnx" + + stitching: + name: "StitchingModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/stitching.onnx?download=true" + path: "liveportrait_onnx/stitching.onnx" + type: "onnx" + + stitching_eye_retarget: + name: "StitchingModel (Eye Retargeting)" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/stitching_eye.onnx?download=true" + path: "liveportrait_onnx/stitching_eye.onnx" + type: "onnx" + + stitching_lip_retarget: + name: "StitchingModel (Lip Retargeting)" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/stitching_lip.onnx?download=true" + path: "liveportrait_onnx/stitching_lip.onnx" + type: "onnx" + sd-turbo: name: "SD-Turbo" url: "https://huggingface.co/stabilityai/sd-turbo/resolve/main/sd_turbo.safetensors" diff --git a/configs/nodes.yaml b/configs/nodes.yaml index 49d422a57..cf497e07c 100644 --- a/configs/nodes.yaml +++ b/configs/nodes.yaml @@ -19,6 +19,12 @@ nodes: branch: "main" type: "tensorrt" + comfyui-fasterliveportrait: + name: "ComfyUI FasterLivePortrait" + url: "https://github.com/pschroedl/ComfyUI-FasterLivePortrait.git" + branch: "main" + type: "tensorrt" + # Ryan's nodes comfyui-ryanontheinside: name: "ComfyUI RyanOnTheInside" diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 7d3facdac..00c214552 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -8,31 +8,20 @@ ARG CONDA_VERSION \ PYTHON_VERSION ENV DEBIAN_FRONTEND=noninteractive \ + TensorRT_ROOT=/opt/TensorRT-10.12.0.36 \ CONDA_VERSION="${CONDA_VERSION}" \ PATH="/workspace/miniconda3/bin:${PATH}" \ PYTHON_VERSION="${PYTHON_VERSION}" - + # System dependencies RUN apt update && apt install -yqq --no-install-recommends \ - git \ - wget \ - nano \ - socat \ - libsndfile1 \ - build-essential \ - llvm \ - tk-dev \ - libglvnd-dev \ - cmake \ - swig \ - libprotobuf-dev \ - protobuf-compiler \ - libcairo2-dev \ - libpango1.0-dev \ - libgdk-pixbuf2.0-dev \ - libffi-dev \ - libgirepository1.0-dev \ - pkg-config \ + git wget nano socat \ + libsndfile1 build-essential llvm tk-dev \ + libglvnd-dev cmake swig libprotobuf-dev \ + protobuf-compiler libcairo2-dev libpango1.0-dev libgdk-pixbuf2.0-dev \ + libffi-dev libgirepository1.0-dev pkg-config libgflags-dev \ + libgoogle-glog-dev libjpeg-dev libavcodec-dev libavformat-dev \ + libavutil-dev libswscale-dev \ && rm -rf /var/lib/apt/lists/* #enable opengl support with nvidia gpu @@ -61,6 +50,21 @@ conda run -n comfystream --no-capture-output pip install wheel COPY ./src/comfystream/scripts /workspace/comfystream/src/comfystream/scripts COPY ./configs /workspace/comfystream/configs +# TensorRT SDK +WORKDIR /opt +RUN wget --progress=dot:giga \ + https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.12.0/tars/TensorRT-10.12.0.36.Linux.x86_64-gnu.cuda-12.9.tar.gz \ + && tar -xzf TensorRT-10.12.0.36.Linux.x86_64-gnu.cuda-12.9.tar.gz \ + && rm TensorRT-10.12.0.36.Linux.x86_64-gnu.cuda-12.9.tar.gz + +# Link libraries and update linker cache +RUN echo "${TensorRT_ROOT}/lib" > /etc/ld.so.conf.d/tensorrt.conf \ + && ldconfig + +# Install matching TensorRT Python bindings for CPython 3.12 +RUN conda run -n comfystream pip install --no-cache-dir \ + ${TensorRT_ROOT}/python/tensorrt-10.12.0.36-cp312-none-linux_x86_64.whl + # Clone ComfyUI RUN git clone --branch v0.3.56 --depth 1 https://github.com/comfyanonymous/ComfyUI.git /workspace/ComfyUI @@ -72,6 +76,10 @@ RUN conda run -n comfystream --cwd /workspace/comfystream --no-capture-output pi # Copy comfystream and example workflows to ComfyUI COPY ./workflows/comfyui/* /workspace/ComfyUI/user/default/workflows/ COPY ./test/example-512x512.png /workspace/ComfyUI/input +COPY ./docker/entrypoint.sh /workspace/comfystream/docker/entrypoint.sh + +# Install OpenCV CUDA +RUN conda run -n comfystream --no-capture-output --cwd /workspace/comfystream --no-capture-output docker/entrypoint.sh --opencv-cuda # Install ComfyUI requirements RUN conda run -n comfystream --no-capture-output --cwd /workspace/ComfyUI pip install -r requirements.txt --root-user-action=ignore diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ef55c77e2..e6d44463a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -130,6 +130,14 @@ if [ "$1" = "--build-engines" ]; then echo "Engine for DepthAnything2 (large) already exists at ${DEPTH_ANYTHING_DIR}/${DEPTH_ANYTHING_ENGINE_LARGE}, skipping..." fi + # Build Engines for FasterLivePortrait + if [ ! -f "$FASTERLIVEPORTRAIT_DIR/warping_spade-fix.trt" ]; then + cd "$FASTERLIVEPORTRAIT_DIR" + bash /workspace/ComfyUI/custom_nodes/ComfyUI-FasterLivePortrait/scripts/build_fasterliveportrait_trt.sh "${FASTERLIVEPORTRAIT_DIR}" "${FASTERLIVEPORTRAIT_DIR}" "${FASTERLIVEPORTRAIT_DIR}" + else + echo "Engines for FasterLivePortrait already exists, skipping..." + fi + # Build Engine for StreamDiffusion if [ ! -f "$TENSORRT_DIR/StreamDiffusion-engines/stabilityai/sd-turbo--lcm_lora-True--tiny_vae-True--max_batch-3--min_batch-3--mode-img2img/unet.engine.opt.onnx" ]; then cd /workspace/ComfyUI/custom_nodes/ComfyUI-StreamDiffusion @@ -158,7 +166,7 @@ if [ "$1" = "--opencv-cuda" ]; then if [ ! -f "/workspace/comfystream/opencv-cuda-release.tar.gz" ]; then # Download and extract OpenCV CUDA build DOWNLOAD_NAME="opencv-cuda-release.tar.gz" - wget -q -O "$DOWNLOAD_NAME" https://github.com/JJassonn69/ComfyUI-Stream-Pack/releases/download/v2/opencv-cuda-release.tar.gz + wget -q -O "$DOWNLOAD_NAME" https://github.com/JJassonn69/ComfyUI-Stream-Pack/releases/download/v2.1/opencv-cuda-release.tar.gz tar -xzf "$DOWNLOAD_NAME" -C /workspace/comfystream/ rm "$DOWNLOAD_NAME" else @@ -166,15 +174,6 @@ if [ "$1" = "--opencv-cuda" ]; then fi # Install required libraries - apt-get update && apt-get install -y \ - libgflags-dev \ - libgoogle-glog-dev \ - libjpeg-dev \ - libavcodec-dev \ - libavformat-dev \ - libavutil-dev \ - libswscale-dev - # Remove existing cv2 package SITE_PACKAGES_DIR="/workspace/miniconda3/envs/comfystream/lib/python3.12/site-packages" rm -rf "${SITE_PACKAGES_DIR}/cv2"* diff --git a/src/comfystream/scripts/constraints.txt b/src/comfystream/scripts/constraints.txt index 529030f2c..9d1bdccb6 100644 --- a/src/comfystream/scripts/constraints.txt +++ b/src/comfystream/scripts/constraints.txt @@ -2,6 +2,7 @@ --extra-index-url https://pypi.nvidia.com numpy<2.0.0 torch==2.7.1+cu128 +cuda-python<13.0 torchvision==0.22.1+cu128 torchaudio==2.7.1+cu128 tensorrt==10.12.0.36 From 2909ea4e36aac50856c812fc4376775409ed7a5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Sep 2025 02:09:32 -0400 Subject: [PATCH 09/44] chore(deps): bump actions/setup-node from 4 to 5 (#377) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 5. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 01900b71a..b46ff8ba2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: node-version: 18 cache: npm From cc292f89f349a17f42b244e128c5a156235f0a21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Sep 2025 02:33:57 -0400 Subject: [PATCH 10/44] chore(deps-dev): bump brace-expansion from 1.1.11 to 1.1.12 in /ui (#443) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 1.1.11 to 1.1.12. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/1.1.11...v1.1.12) ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/package-lock.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 1261f8422..0938cc8b8 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -2367,9 +2367,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2866,9 +2866,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -7389,9 +7389,9 @@ } }, "node_modules/sucrase/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" From 50717e0f1970987d85fe16518629cfc8ab273c70 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Mon, 29 Sep 2025 17:23:35 +0300 Subject: [PATCH 11/44] Feat/updated UI remove inline (#444) * refactor: remove unused HTTP streaming modules and clean up related code --- server/app.py | 29 ----- server/frame_buffer.py | 42 ------- server/http_streaming/__init__.py | 5 - server/http_streaming/routes.py | 69 ----------- server/http_streaming/tokens.py | 86 ------------- server/public/stream.html | 60 --------- ui/src/app/webrtc-preview/page.tsx | 176 +++++++++++++++++++++++++++ ui/src/components/room.tsx | 6 +- ui/src/components/stream-control.tsx | 127 ++++++++----------- ui/src/hooks/use-peer.ts | 26 +++- 10 files changed, 252 insertions(+), 374 deletions(-) delete mode 100644 server/frame_buffer.py delete mode 100644 server/http_streaming/__init__.py delete mode 100644 server/http_streaming/routes.py delete mode 100644 server/http_streaming/tokens.py delete mode 100644 server/public/stream.html create mode 100644 ui/src/app/webrtc-preview/page.tsx diff --git a/server/app.py b/server/app.py index 028f12bbc..b496a4585 100644 --- a/server/app.py +++ b/server/app.py @@ -12,9 +12,6 @@ if torch.cuda.is_available(): torch.cuda.init() - -from aiohttp import web, MultipartWriter -from aiohttp_cors import setup as setup_cors, ResourceOptions from aiohttp import web from aiortc import ( MediaStreamTrack, @@ -24,7 +21,6 @@ RTCSessionDescription, ) # Import HTTP streaming modules -from http_streaming.routes import setup_routes from aiortc.codecs import h264 from aiortc.rtcrtpsender import RTCRtpSender from comfystream.pipeline import Pipeline @@ -112,15 +108,6 @@ async def recv(self): """ processed_frame = await self.pipeline.get_processed_video_frame() - # Update the frame buffer with the processed frame - try: - from frame_buffer import FrameBuffer - frame_buffer = FrameBuffer.get_instance() - frame_buffer.update_frame(processed_frame) - except Exception as e: - # Don't let frame buffer errors affect the main pipeline - print(f"Error updating frame buffer: {e}") - # Increment the frame count to calculate FPS. await self.fps_meter.increment_frame_count() @@ -637,16 +624,6 @@ async def on_shutdown(app: web.Application): app = web.Application() app["media_ports"] = args.media_ports.split(",") if args.media_ports else None app["workspace"] = args.workspace - - # Setup CORS - cors = setup_cors(app, defaults={ - "*": ResourceOptions( - allow_credentials=True, - expose_headers="*", - allow_headers="*", - allow_methods=["GET", "POST", "OPTIONS"] - ) - }) app.on_startup.append(on_startup) app.on_shutdown.append(on_shutdown) @@ -658,12 +635,6 @@ async def on_shutdown(app: web.Application): app.router.add_post("/offer", offer) app.router.add_post("/prompt", set_prompt) - # Setup HTTP streaming routes - setup_routes(app, cors) - - # Serve static files from the public directory - app.router.add_static("/", path=os.path.join(os.path.dirname(__file__), "public"), name="static") - # Add routes for getting stream statistics. stream_stats_manager = StreamStatsManager(app) app.router.add_get( diff --git a/server/frame_buffer.py b/server/frame_buffer.py deleted file mode 100644 index 2a16407ae..000000000 --- a/server/frame_buffer.py +++ /dev/null @@ -1,42 +0,0 @@ -import threading -import time -import numpy as np -import cv2 -import av -from typing import Optional - -class FrameBuffer: - _instance = None - - @classmethod - def get_instance(cls): - if cls._instance is None: - cls._instance = FrameBuffer() - return cls._instance - - def __init__(self): - self.current_frame = None - self.frame_lock = threading.Lock() - self.last_update_time = 0 - self.quality = 70 # JPEG quality (0-100) - - def update_frame(self, frame): - """Update the current frame in the buffer""" - with self.frame_lock: - # Convert frame to numpy array if it's an av.VideoFrame - if isinstance(frame, av.VideoFrame): - frame_np = frame.to_ndarray(format="rgb24") - else: - frame_np = frame - - # Store the frame as a JPEG-encoded bytes object for efficient serving - _, jpeg_frame = cv2.imencode('.jpg', cv2.cvtColor(frame_np, cv2.COLOR_RGB2BGR), - [cv2.IMWRITE_JPEG_QUALITY, self.quality]) - - self.current_frame = jpeg_frame.tobytes() - self.last_update_time = time.time() - - def get_current_frame(self) -> Optional[bytes]: - """Get the current frame from the buffer""" - with self.frame_lock: - return self.current_frame diff --git a/server/http_streaming/__init__.py b/server/http_streaming/__init__.py deleted file mode 100644 index 4fad17f79..000000000 --- a/server/http_streaming/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -HTTP Streaming module for ComfyStream. - -This module contains components for token management and HTTP streaming routes. -""" diff --git a/server/http_streaming/routes.py b/server/http_streaming/routes.py deleted file mode 100644 index ac309bae5..000000000 --- a/server/http_streaming/routes.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -HTTP streaming routes for ComfyStream. - -This module contains the routes for HTTP streaming. -""" -import asyncio -import logging -from aiohttp import web -from frame_buffer import FrameBuffer -from .tokens import cleanup_expired_sessions, validate_token, create_stream_token - -logger = logging.getLogger(__name__) - -async def stream_mjpeg(request): - """Serve an MJPEG stream with token validation""" - # Clean up expired sessions - cleanup_expired_sessions() - - stream_id = request.query.get("token") - - # Validate the stream token - is_valid, error_message = validate_token(stream_id) - if not is_valid: - return web.Response(status=403, text=error_message) - - frame_buffer = FrameBuffer.get_instance() - - # Use a fixed frame delay for 30 FPS - frame_delay = 1.0 / 30 - - response = web.StreamResponse( - status=200, - reason='OK', - headers={ - 'Content-Type': 'multipart/x-mixed-replace; boundary=frame', - 'Cache-Control': 'no-cache', - 'Connection': 'close', - } - ) - await response.prepare(request) - - try: - while True: - jpeg_frame = frame_buffer.get_current_frame() - if jpeg_frame is not None: - await response.write( - b'--frame\r\n' - b'Content-Type: image/jpeg\r\n\r\n' + jpeg_frame + b'\r\n' - ) - await asyncio.sleep(frame_delay) - except (ConnectionResetError, asyncio.CancelledError): - logger.info("MJPEG stream connection closed") - except Exception as e: - logger.error(f"Error in MJPEG stream: {e}") - finally: - return response - -def setup_routes(app, cors): - """Setup HTTP streaming routes - - Args: - app: The aiohttp web application - cors: The CORS setup object - """ - # Stream token endpoints - cors.add(app.router.add_post("/api/stream-token", create_stream_token)) - - # Stream endpoint with token validation - cors.add(app.router.add_get("/api/stream", stream_mjpeg)) diff --git a/server/http_streaming/tokens.py b/server/http_streaming/tokens.py deleted file mode 100644 index d424cf36d..000000000 --- a/server/http_streaming/tokens.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Token management system for ComfyStream HTTP streaming. - -This module handles the creation, validation, and management of stream tokens. -""" -import time -import secrets -import logging -from aiohttp import web - -logger = logging.getLogger(__name__) - -# Constants -SESSION_CLEANUP_INTERVAL = 60 # Clean up expired sessions every 60 seconds - -# Global token storage -active_stream_sessions = {} -last_cleanup_time = 0 - -def cleanup_expired_sessions(): - """Clean up expired stream sessions""" - global active_stream_sessions, last_cleanup_time - - current_time = time.time() - - # Only clean up if it's been at least SESSION_CLEANUP_INTERVAL since last cleanup - if current_time - last_cleanup_time < SESSION_CLEANUP_INTERVAL: - return - - # Update the last cleanup time - last_cleanup_time = current_time - - # Find expired sessions - expired_sessions = [sid for sid, expires in active_stream_sessions.items() if current_time > expires] - - # Remove expired sessions - for sid in expired_sessions: - logger.info(f"Removing expired session: {sid[:8]}...") - del active_stream_sessions[sid] - - if expired_sessions: - logger.info(f"Cleaned up {len(expired_sessions)} expired sessions. {len(active_stream_sessions)} active sessions remaining.") - -async def create_stream_token(request): - """Create a unique stream token for secure access to the stream""" - global active_stream_sessions - - # Clean up expired sessions - cleanup_expired_sessions() - - current_time = time.time() - - # Generate a new unique token - stream_id = secrets.token_urlsafe(32) - expires_at = current_time + 3600 # 1 hour from now - - # Store the new session - active_stream_sessions[stream_id] = expires_at - - logger.info(f"Generated new stream token: {stream_id[:8]}... ({len(active_stream_sessions)} active sessions)") - - return web.json_response({ - "stream_id": stream_id, - "expires_at": int(expires_at) - }) - -def validate_token(token): - """Validate a stream token and return whether it's valid - - Args: - token: The token to validate - - Returns: - tuple: (is_valid, error_message) - """ - if not token or token not in active_stream_sessions: - return False, "Invalid stream token" - - # Check if token is expired - current_time = time.time() - if current_time > active_stream_sessions[token]: - # Remove expired token - del active_stream_sessions[token] - return False, "Stream token expired" - - return True, None diff --git a/server/public/stream.html b/server/public/stream.html deleted file mode 100644 index 536781f97..000000000 --- a/server/public/stream.html +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - ComfyStream - OBS Capture - - - - - Video Stream - - diff --git a/ui/src/app/webrtc-preview/page.tsx b/ui/src/app/webrtc-preview/page.tsx new file mode 100644 index 000000000..b2d2045a2 --- /dev/null +++ b/ui/src/app/webrtc-preview/page.tsx @@ -0,0 +1,176 @@ +"use client"; +// WebRTC Preview Popup Page (client-only) + +import React, { useEffect, useRef, useState, useCallback } from "react"; + +const POLL_INTERVAL_MS = 300; +const MAX_ATTEMPTS = 200; // ~60s + +export default function WebRTCPopupPage() { + const videoRef = useRef(null); + const localStreamRef = useRef(null); + const parentStreamRef = useRef(null); + const clonedIdsRef = useRef>(new Set()); + const attemptsRef = useRef(0); + const intervalRef = useRef(null); + const [status, setStatus] = useState("Initializing…"); + + const clearIntervalInternal = useCallback(() => { + if (intervalRef.current !== null) { + window.clearInterval(intervalRef.current); + intervalRef.current = null; + } + }, []); + + const scheduleClose = useCallback((delay = 800) => { + window.setTimeout(() => { + try { window.close(); } catch { /* noop */ } + }, delay); + }, []); + + const validateOpener = useCallback((): boolean => { + try { + if (!window.opener) { + setStatus("Opener lost. Closing…"); + scheduleClose(); + return false; + } + void window.opener.location.href; // cross-origin check + return true; + } catch { + setStatus("Cross-origin opener. Closing…"); + scheduleClose(); + return false; + } + }, [scheduleClose]); + + const attachVideoIfNeeded = () => { + if (!localStreamRef.current) return; + const video = videoRef.current; + if (video && video.srcObject !== localStreamRef.current) { + video.srcObject = localStreamRef.current; + } + }; + + const cloneTracks = useCallback(() => { + if (!validateOpener()) return; + // @ts-ignore - global from opener context + const parentStream: MediaStream | undefined = window.opener?.__comfystreamRemoteStream; + if (!parentStream) { + setStatus("Waiting for stream…"); + return; + } + if (!localStreamRef.current) { + localStreamRef.current = new MediaStream(); + } + // Parent stream changed -> reset + if (parentStreamRef.current && parentStreamRef.current !== parentStream) { + localStreamRef.current.getTracks().forEach(t => { try { t.stop(); } catch { /* */ } }); + localStreamRef.current = new MediaStream(); + clonedIdsRef.current.clear(); + } + parentStreamRef.current = parentStream; + + let added = false; + parentStream.getTracks().forEach(src => { + if (src.readyState === "ended") return; + if (!clonedIdsRef.current.has(src.id)) { + try { + const clone = src.clone(); + clone.addEventListener("ended", () => { + clonedIdsRef.current.delete(src.id); + }); + localStreamRef.current!.addTrack(clone); + clonedIdsRef.current.add(src.id); + added = true; + } catch { + /* skip */ + } + } + }); + // Cleanup ended clones + localStreamRef.current.getTracks().forEach(t => { + if (t.readyState === "ended") { + localStreamRef.current!.removeTrack(t); + try { t.stop(); } catch { /* */ } + } + }); + if (added) { + attachVideoIfNeeded(); + setStatus("Live"); + videoRef.current?.play().catch(() => {}); + } + }, [validateOpener]); + + useEffect(() => { + if (typeof window === "undefined") return; // safety + attemptsRef.current = 0; + setStatus("Initializing…"); + + const tick = () => { + attemptsRef.current += 1; + if (!validateOpener()) { + clearIntervalInternal(); + return; + } + // @ts-ignore + if (!window.opener.__comfystreamRemoteStream) { + setStatus("Parent stream ended"); + clearIntervalInternal(); + scheduleClose(1200); + return; + } + cloneTracks(); + if (attemptsRef.current >= MAX_ATTEMPTS && (!localStreamRef.current || localStreamRef.current.getTracks().length === 0)) { + setStatus("Timeout waiting for stream"); + clearIntervalInternal(); + scheduleClose(1500); + } + }; + + intervalRef.current = window.setInterval(tick, POLL_INTERVAL_MS); + cloneTracks(); + + const beforeUnload = () => { + clearIntervalInternal(); + localStreamRef.current?.getTracks().forEach(t => { try { t.stop(); } catch { /* */ } }); + }; + window.addEventListener("beforeunload", beforeUnload); + return () => { + window.removeEventListener("beforeunload", beforeUnload); + clearIntervalInternal(); + localStreamRef.current?.getTracks().forEach(t => { try { t.stop(); } catch { /* */ } }); + }; + }, [cloneTracks, clearIntervalInternal, validateOpener, scheduleClose]); + + return ( +
+
+ ); +} diff --git a/ui/src/components/room.tsx b/ui/src/components/room.tsx index 85c6a8f85..df03ed50f 100644 --- a/ui/src/components/room.tsx +++ b/ui/src/components/room.tsx @@ -178,12 +178,11 @@ interface StageProps { onStreamReady: () => void; onComfyUIReady: () => void; resolution: { width: number; height: number }; - backendUrl: string; onOutputStreamReady: (stream: MediaStream | null) => void; prompts: Prompt[] | null; } -function Stage({ connected, onStreamReady, onComfyUIReady, resolution, backendUrl, onOutputStreamReady, prompts }: StageProps) { +function Stage({ connected, onStreamReady, onComfyUIReady, resolution, onOutputStreamReady, prompts }: StageProps) { const { remoteStream, peerConnection } = usePeerContext(); const [frameRate, setFrameRate] = useState(0); // Add state and refs for tracking frames @@ -310,7 +309,7 @@ function Stage({ connected, onStreamReady, onComfyUIReady, resolution, backendUr )} {/* Add StreamControlIcon at the bottom right corner of the video box */} - + ); } @@ -598,7 +597,6 @@ export const Room = () => { => { - try { - // Validate backendUrl - if (!backendUrl) { - console.error("Backend URL is not configured."); - throw new Error("Backend URL is not configured in settings."); - } - // Parse base URL from the provided backendUrl - let baseUrl: string; + // Open popup which polls opener for stream and clones tracks locally (no postMessage MediaStream cloning) + const openWebRTCPopup = useCallback(() => { + const features = 'width=1024,height=1024'; + const getBasePath = (): string => { try { - // The origin property gives us "http://hostname:port" - baseUrl = new URL(backendUrl).origin; - } catch (e) { - console.error("Invalid backend URL configured:", backendUrl, e); - throw new Error(`Invalid backend URL configured: ${backendUrl}`); - } - - // Check if we're in a hosted environment by looking at the current URL - // This might need adjustment depending on how hosted environments are detected - const isHosted = window.location.pathname.includes('/live'); - const pathPrefix = isHosted ? '/live' : ''; - - // Request a unique streamID from the server using the derived baseUrl - const response = await fetch(`${baseUrl}${pathPrefix}/api/stream-token`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' + const scripts = document.querySelectorAll('script[src]'); + for (const s of Array.from(scripts)) { + const src = (s as HTMLScriptElement).src; + // Look for /_next/static/ which precedes hashed chunks + const idx = src.indexOf('/_next/static/'); + if (idx !== -1) { + const urlObj = new URL(src); + const before = urlObj.pathname.substring(0, urlObj.pathname.indexOf('/_next/static/')); + if (before !== undefined) { + return before.replace(/\/$/, ''); + } + } } - }); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.error || `Failed to get stream token: ${response.status}`); - } - - const data = await response.json(); - const streamId = data.stream_id; - - // Return the URL with the unique streamID, using the derived baseUrl - // Note: Token will be removed from URL in a later step - return `${baseUrl}${pathPrefix}/stream.html?token=${streamId}`; - } catch (error) { - console.error('Error getting stream URL:', error); - return null; - } - }; - - // Open the stream in a new window - const openStreamWindow = async () => { - try { - setIsLoading(true); - const streamUrl = await getStreamUrl(); - - if (!streamUrl) { - throw new Error('Failed to get stream URL'); - } - - const newWindow = window.open(streamUrl, 'ComfyStream OBS Capture', 'width=1024,height=1024'); - - if (!newWindow) { - throw new Error('Failed to open stream window. Please check your popup blocker settings.'); - } - } catch (error) { - console.error('Error opening stream window:', error); - alert(error instanceof Error ? error.message : 'Failed to open stream window. Please try again.'); - } finally { - setIsLoading(false); + } catch { /* ignore */ } + + try { + const { pathname } = window.location; + // If pathname points to a file (no trailing slash and contains a dot), strip file portion + if (/\.[a-zA-Z0-9]{2,8}$/.test(pathname.split('/').pop() || '')) { + const parts = pathname.split('/'); + parts.pop(); + return parts.join('/') || '/'; + } + return pathname.replace(/\/$/, ''); + } catch { /* ignore */ } + + return ''; + }; + + const basePath = getBasePath(); + const isDev = process.env.NEXT_PUBLIC_DEV === 'true'; + const previewPath = (basePath ? basePath : '') + (isDev ? '/webrtc-preview' : '/webrtc-preview.html'); + + const popup = window.open(previewPath, 'comfystream_preview', features) || window.open(previewPath); + if (!popup) { + alert('Popup blocked. Please allow popups for this site.'); } + }, []); + + const openStreamWindow = () => { + openWebRTCPopup(); }; return (