From f7fba9073ef9df32c273f8a4fb77c72b373c34c0 Mon Sep 17 00:00:00 2001 From: Maxime Leroy <19607336+maxime1907@users.noreply.github.com> Date: Sun, 26 Jan 2025 15:05:14 +0100 Subject: [PATCH] feat: youtube proxy config, bump yt-dlp, drop uneeded dev deps --- VERSION | 2 +- config/config.json | 4 +- pyproject.toml | 4 +- requirements-dev.txt | 20 +--- requirements.txt | 2 +- src/torchlight/AsyncClient.py | 4 +- src/torchlight/Commands.py | 12 ++- src/torchlight/FFmpegAudioPlayer.py | 152 ++++++++++++++++++---------- src/torchlight/SourceRCONClient.py | 7 +- src/torchlight/SourceRCONServer.py | 11 +- src/torchlight/TriggerManager.py | 2 +- src/torchlight/URLInfo.py | 9 +- 12 files changed, 126 insertions(+), 103 deletions(-) diff --git a/VERSION b/VERSION index 943f9cb..27f9cd3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.1 +1.8.0 diff --git a/config/config.json b/config/config.json index 5ac4241..975a2b2 100644 --- a/config/config.json +++ b/config/config.json @@ -29,7 +29,8 @@ "Host": "127.0.0.1", "Port": 27019, "SampleRate": 22050, - "Volume": 1.0 + "Volume": 1.0, + "Proxy": "" }, "GeoIP": @@ -222,6 +223,7 @@ } ], "parameters": { + "proxy": "", "keywords_banned": [ "earrape", "rape", diff --git a/pyproject.toml b/pyproject.toml index c96899f..7358300 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,11 +41,9 @@ version = {file = "VERSION"} [project.optional-dependencies] dev = [ - "ruff", "memory_profiler", "mypy", - "pip-tools", - "pyupgrade", + "ruff", ] [tool.mypy] diff --git a/requirements-dev.txt b/requirements-dev.txt index 22ae250..f20cb4f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,8 +17,6 @@ beautifulsoup4==4.12.3 # via # -c requirements.txt # torchlight (pyproject.toml) -build==1.0.3 - # via pip-tools certifi==2023.11.17 # via # -c requirements.txt @@ -32,7 +30,6 @@ click==8.1.7 # -c requirements.txt # torchlight (pyproject.toml) # gtts - # pip-tools defusedxml==0.7.1 # via # -c requirements.txt @@ -74,26 +71,16 @@ mypy==1.8.0 # via torchlight (pyproject.toml) mypy-extensions==1.0.0 # via mypy -packaging==23.2 - # via build pillow==10.2.0 # via # -c requirements.txt # torchlight (pyproject.toml) -pip==24.3.1 - # via pip-tools -pip-tools==7.3.0 - # via torchlight (pyproject.toml) psutil==6.1.1 # via memory-profiler -pyproject-hooks==1.0.0 - # via build python-magic==0.4.27 # via # -c requirements.txt # torchlight (pyproject.toml) -pyupgrade==3.15.0 - # via torchlight (pyproject.toml) requests==2.32.3 # via # -c requirements.txt @@ -106,26 +93,21 @@ setuptools==75.8.0 # -c requirements.txt # geoip2 # maxminddb - # pip-tools soupsieve==2.5 # via # -c requirements.txt # beautifulsoup4 -tokenize-rt==5.2.0 - # via pyupgrade typing-extensions==4.9.0 # via mypy urllib3==2.1.0 # via # -c requirements.txt # requests -wheel==0.42.0 - # via pip-tools yarl==1.9.4 # via # -c requirements.txt # aiohttp -yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@af2c821d74049b519895288aca23cee81fc4b049#egg=yt-dlp +yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@5ff7a43623e3a92270f66a7e37b5fc53d7a57fdf#egg=yt-dlp # via # -c requirements.txt # torchlight (pyproject.toml) diff --git a/requirements.txt b/requirements.txt index 747bda8..d176bbc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,5 +58,5 @@ urllib3==2.1.0 # via requests yarl==1.9.4 # via aiohttp -yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@af2c821d74049b519895288aca23cee81fc4b049#egg=yt-dlp +yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@5ff7a43623e3a92270f66a7e37b5fc53d7a57fdf#egg=yt-dlp # via torchlight (pyproject.toml) diff --git a/src/torchlight/AsyncClient.py b/src/torchlight/AsyncClient.py index bdffeaf..149d952 100644 --- a/src/torchlight/AsyncClient.py +++ b/src/torchlight/AsyncClient.py @@ -32,7 +32,7 @@ def __init__( # @profile async def Connect(self) -> None: while True: - self.logger.warn("Reconnecting...") + self.logger.warning("Reconnecting...") try: _, self.protocol = await self.loop.create_connection( lambda: ClientProtocol(self.loop), @@ -64,7 +64,7 @@ def OnReceive(self, data: str | bytes) -> None: try: json_obj = json.loads(data) except Exception: - self.logger.warn("OnReceive: Unable to decode data as json, skipping") + self.logger.warning("OnReceive: Unable to decode data as json, skipping") return if "method" in json_obj and json_obj["method"] == "publish": diff --git a/src/torchlight/Commands.py b/src/torchlight/Commands.py index 93d6f92..e019b08 100644 --- a/src/torchlight/Commands.py +++ b/src/torchlight/Commands.py @@ -752,6 +752,8 @@ async def _func(self, message: list[str], player: Player) -> int: if self.check_disabled(player): return -1 + command_config = self.get_config() + input_keywords = message[1] if URLFilter.youtube_regex.search(input_keywords): input_url = input_keywords @@ -760,8 +762,11 @@ async def _func(self, message: list[str], player: Player) -> int: real_time = get_url_real_time(url=input_url) + if "parameters" in command_config and "proxy" in command_config["parameters"]: + proxy = command_config["parameters"]["proxy"] + try: - info = get_url_youtube_info(url=input_url) + info = get_url_youtube_info(url=input_url, proxy=proxy) except Exception as e: self.logger.error(f"Failed to extract youtube info from: {input_url}") self.logger.error(e) @@ -772,16 +777,15 @@ async def _func(self, message: list[str], player: Player) -> int: return 1 if "title" not in info and "url" in info: - info = get_url_youtube_info(url=info["url"]) + info = get_url_youtube_info(url=info["url"], proxy=proxy) if info["extractor_key"] == "YoutubeSearch": - info = get_first_valid_entry(entries=info["entries"]) + info = get_first_valid_entry(entries=info["entries"], proxy=proxy) title = info["title"] url = get_audio_format(info=info) title_words = title.split() keywords_banned: list[str] = [] - command_config = self.get_config() if "parameters" in command_config and "keywords_banned" in command_config["parameters"]: keywords_banned = command_config["parameters"]["keywords_banned"] diff --git a/src/torchlight/FFmpegAudioPlayer.py b/src/torchlight/FFmpegAudioPlayer.py index 779b532..394d9bc 100644 --- a/src/torchlight/FFmpegAudioPlayer.py +++ b/src/torchlight/FFmpegAudioPlayer.py @@ -29,13 +29,15 @@ def __init__(self, torchlight: Torchlight) -> None: self.port = self.config["Port"] self.sample_rate = float(self.config["SampleRate"]) self.volume = float(self.config["Volume"]) + self.proxy = self.config["Proxy"] self.started_playing: float | None = None self.stopped_playing: float | None = None self.seconds = 0.0 self.writer: StreamWriter | None = None - self.sub_process: Process | None = None + self.ffmpeg_process: Process | None = None + self.curl_process: Process | None = None self.callbacks: list[tuple[str, Callable]] = [] @@ -45,53 +47,48 @@ def __del__(self) -> None: # @profile def PlayURI(self, uri: str, position: int | None, *args: Any) -> bool: + curl_command = ["/usr/bin/curl", "-L", uri] + if self.proxy: + curl_command.extend( + [ + "-x", + self.proxy, + ] + ) + ffmpeg_command = [ + "/usr/bin/ffmpeg", + "-i", + "pipe:0", + "-acodec", + "pcm_s16le", + "-ac", + "1", + "-ar", + str(int(self.sample_rate)), + "-filter:a", + f"volume={str(float(self.volume))}", + "-f", + "s16le", + "-vn", + *args, + "-", + ] + if position is not None: pos_str = str(datetime.timedelta(seconds=position)) - command = [ - "/usr/bin/ffmpeg", - "-ss", - pos_str, - "-i", - uri, - "-acodec", - "pcm_s16le", - "-ac", - "1", - "-ar", - str(int(self.sample_rate)), - "-filter:a", - f"volume={str(float(self.volume))}", - "-f", - "s16le", - "-vn", - *args, - "-", - ] + ffmpeg_command.extend( + [ + "-ss", + pos_str, + ] + ) self.position = position - else: - command = [ - "/usr/bin/ffmpeg", - "-i", - uri, - "-acodec", - "pcm_s16le", - "-ac", - "1", - "-ar", - str(int(self.sample_rate)), - "-filter:a", - f"volume={str(float(self.volume))}", - "-f", - "s16le", - "-vn", - *args, - "-", - ] - - self.logger.debug(command) + + self.logger.debug(curl_command) + self.logger.debug(ffmpeg_command) self.playing = True - asyncio.ensure_future(self._stream_subprocess(command)) + asyncio.ensure_future(self._stream_subprocess(curl_command, ffmpeg_command)) return True # @profile @@ -99,11 +96,20 @@ def Stop(self, force: bool = True) -> bool: if not self.playing: return False - if self.sub_process: + if self.ffmpeg_process: + try: + self.ffmpeg_process.terminate() + self.ffmpeg_process.kill() + self.ffmpeg_process = None + except ProcessLookupError as exc: + self.logger.debug(exc) + pass + + if self.curl_process: try: - self.sub_process.terminate() - self.sub_process.kill() - self.sub_process = None + self.curl_process.terminate() + self.curl_process.kill() + self.curl_process = None except ProcessLookupError as exc: self.logger.debug(exc) pass @@ -128,7 +134,7 @@ def Stop(self, force: bool = True) -> bool: else: loop.run_until_complete(self.writer.wait_closed()) except Exception as exc: - self.logger.warn(exc) + self.logger.warning(exc) pass self.playing = False @@ -173,7 +179,7 @@ async def _updater(self) -> None: if seconds_elapsed >= self.seconds: if not self.stopped_playing: - self.logger.warn("BUFFER UNDERRUN!") + self.logger.debug("BUFFER UNDERRUN!") self.Stop(False) return @@ -208,21 +214,55 @@ async def _read_stream(self, stream: StreamReader | None, writer: StreamWriter) self.stopped_playing = time.time() # @profile - async def _stream_subprocess(self, cmd: list[str]) -> None: + async def _stream_subprocess(self, curl_command: list[str], ffmpeg_command: list[str]) -> None: if not self.playing: return _, self.writer = await asyncio.open_connection(self.host, self.port) - self.sub_process = await asyncio.create_subprocess_exec( - *cmd, + self.ffmpeg_process = await asyncio.create_subprocess_exec( + *ffmpeg_command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + self.curl_process = await asyncio.create_subprocess_exec( + *curl_command, stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.DEVNULL, ) - await self._read_stream(self.sub_process.stdout, self.writer) - if self.sub_process is not None: - await self.sub_process.wait() + asyncio.create_task(self._wait_for_process_exit(self.curl_process)) + + asyncio.create_task(self._write_stream(self.curl_process.stdout, self.ffmpeg_process.stdin)) + + await self._read_stream(self.ffmpeg_process.stdout, self.writer) + + if self.ffmpeg_process is not None: + if self.ffmpeg_process.stdin: + self.ffmpeg_process.stdin.close() + await self.ffmpeg_process.wait() + + self.writer.close() + await self.writer.wait_closed() if self.seconds == 0.0: self.Stop() + + async def _write_stream(self, stream: StreamReader | None, writer: StreamWriter | None) -> None: + while True: + if not stream: + break + chunk = await stream.read(65536) + if not chunk: + break + + if writer: + writer.write(chunk) + await writer.drain() + + async def _wait_for_process_exit(self, curl_process: Process) -> None: + await curl_process.wait() + if curl_process.returncode != 0: + self.logger.error(f"Curl process exited with error code {curl_process.returncode}") + self.Stop() diff --git a/src/torchlight/SourceRCONClient.py b/src/torchlight/SourceRCONClient.py index bceb2cd..afb6721 100644 --- a/src/torchlight/SourceRCONClient.py +++ b/src/torchlight/SourceRCONClient.py @@ -3,7 +3,7 @@ import socket import struct import sys -from collections.abc import Awaitable, Generator +from collections.abc import Awaitable from typing import Any from torchlight.CommandHandler import CommandHandler @@ -32,10 +32,9 @@ def send(self, data: bytes) -> Awaitable: return self.loop.sock_sendall(self._sock, data) # @profile - @asyncio.coroutine - def _peer_loop(self) -> Generator: + async def _peer_loop(self) -> None: while True: - data = yield from self.loop.sock_recv(self._sock, 1024) + data = await self.loop.sock_recv(self._sock, 1024) if data == b"": break diff --git a/src/torchlight/SourceRCONServer.py b/src/torchlight/SourceRCONServer.py index b7f6843..6218f91 100644 --- a/src/torchlight/SourceRCONServer.py +++ b/src/torchlight/SourceRCONServer.py @@ -2,7 +2,6 @@ import logging import socket import sys -from collections.abc import Generator from typing import Any from torchlight.SourceRCONClient import SourceRCONClient @@ -27,12 +26,11 @@ def Remove(self, peer: SourceRCONClient) -> None: self.logger.info(sys._getframe().f_code.co_name + f" Peer {peer.name} disconnected!") self.peers.remove(peer) - @asyncio.coroutine - def _server(self) -> Generator: + async def _server(self) -> None: while True: peer_socket: socket.socket peer_name: Any - peer_socket, peer_name = yield from self.loop.sock_accept(self._serv_sock) + peer_socket, peer_name = await self.loop.sock_accept(self._serv_sock) peer_socket.setblocking(False) peer = SourceRCONClient( self.loop, @@ -45,10 +43,9 @@ def _server(self) -> Generator: self.peers.append(peer) self.logger.info(sys._getframe().f_code.co_name + f" Peer {peer.name} connected!") - @asyncio.coroutine - def _peer_handler(self, peer: SourceRCONClient) -> Generator: + async def _peer_handler(self, peer: SourceRCONClient) -> None: try: - yield from peer._peer_loop() + await peer._peer_loop() except OSError: pass finally: diff --git a/src/torchlight/TriggerManager.py b/src/torchlight/TriggerManager.py index 91b35f5..bf59cd3 100644 --- a/src/torchlight/TriggerManager.py +++ b/src/torchlight/TriggerManager.py @@ -41,4 +41,4 @@ def Load(self) -> None: for sound in sounds: sound_path = os.path.abspath(os.path.join(self.sound_path, sound)) if not os.path.exists(sound_path): - self.logger.warn(f"Sound path {sound_path} does not exist") + self.logger.warning(f"Sound path {sound_path} does not exist") diff --git a/src/torchlight/URLInfo.py b/src/torchlight/URLInfo.py index 8b739fc..48abd14 100644 --- a/src/torchlight/URLInfo.py +++ b/src/torchlight/URLInfo.py @@ -102,7 +102,7 @@ def get_url_real_time(url: str) -> int: # @profile -def get_url_youtube_info(url: str) -> dict: +def get_url_youtube_info(url: str, proxy: str = "") -> dict: # https://github.com/ytdl-org/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312 # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L192 ydl_opts = { @@ -114,6 +114,7 @@ def get_url_youtube_info(url: str) -> dict: "format": "m4a/bestaudio/best", "simulate": True, "keepvideo": False, + "proxy": proxy, } ydl = yt_dlp.YoutubeDL(ydl_opts) ydl.add_default_info_extractors() @@ -121,14 +122,14 @@ def get_url_youtube_info(url: str) -> dict: # @profile -def get_first_valid_entry(entries: list[Any]) -> dict[str, Any]: +def get_first_valid_entry(entries: list[Any], proxy: str = "") -> dict[str, Any]: for entry in entries: input_url = f"https://youtube.com/watch?v={entry['id']}" try: - info = get_url_youtube_info(url=input_url) + info = get_url_youtube_info(url=input_url, proxy=proxy) return info except yt_dlp.utils.DownloadError: - logger.warn(f"Error trying to download <{input_url}>") + logger.warning(f"Error trying to download <{input_url}>") pass raise Exception("No compatible youtube video found, try something else")