From ce13f01a4beacc047502cd8e489fa38e5854e4d3 Mon Sep 17 00:00:00 2001 From: Olezhich Date: Wed, 22 Oct 2025 20:53:40 +0300 Subject: [PATCH 1/5] dev side work is integrated with mypy --- audiostats/db/api.py | 2 +- audiostats/db/models.py | 7 ++- audiostats/db/session.py | 25 ++++---- audiostats/handlers/plst_handler.py | 23 +++++--- migrations/env.py | 6 +- poetry.lock | 88 ++++++++++++++++++++++++++++- pyproject.toml | 5 ++ 7 files changed, 130 insertions(+), 26 deletions(-) diff --git a/audiostats/db/api.py b/audiostats/db/api.py index 87efa2d..72b35f0 100644 --- a/audiostats/db/api.py +++ b/audiostats/db/api.py @@ -13,7 +13,7 @@ class DBApi: def __init__(self, db_url : str, workers : int = 5, queue_sz : int = 10): self._session_factory = SessionFactory(db_url) - self._queue = asyncio.Queue(maxsize=queue_sz) + self._queue: asyncio.Queue[AlbumDTO | None] = asyncio.Queue(maxsize=queue_sz) self._num_workers = workers async def _album_upserter(self): diff --git a/audiostats/db/models.py b/audiostats/db/models.py index a798ad6..eed9e8d 100644 --- a/audiostats/db/models.py +++ b/audiostats/db/models.py @@ -1,15 +1,16 @@ from datetime import datetime from sqlalchemy import String, Integer, UniqueConstraint, ForeignKey, Float, Enum, DateTime, func -from sqlalchemy.orm import declarative_base, Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship, DeclarativeBase from audiostats.domain import Status, Success -Base = declarative_base() - MAX_PATH_FIELD_LEN = 200 MAX_STR_FIELD_LEN = 50 +class Base(DeclarativeBase): + ... + class Album(Base): """Represents **albums** table line as orm object diff --git a/audiostats/db/session.py b/audiostats/db/session.py index 2d811a0..4775815 100644 --- a/audiostats/db/session.py +++ b/audiostats/db/session.py @@ -1,5 +1,6 @@ from asyncio import Semaphore from contextlib import asynccontextmanager +from typing import AsyncIterator from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession @@ -17,7 +18,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): await self._engine.dispose() @asynccontextmanager - async def get_session(self) -> AsyncSession: + async def get_session(self) -> AsyncIterator[AsyncSession]: session = None try: await self._semaphore.acquire() @@ -30,14 +31,14 @@ async def get_session(self) -> AsyncSession: await session.close() self._semaphore.release() - async def __call__(self, *args, **kwargs) -> AsyncSession: - await self._semaphore.acquire() - session = self._session_maker() - - async def close_session(): - await session.close() - self._semaphore.release() - - session.close = close_session() - - return session \ No newline at end of file + # async def __call__(self, *args, **kwargs) -> AsyncSession: + # await self._semaphore.acquire() + # session = self._session_maker() + # + # async def close_session(): + # await session.close() + # self._semaphore.release() + # + # session.close = close_session() + # + # return session \ No newline at end of file diff --git a/audiostats/handlers/plst_handler.py b/audiostats/handlers/plst_handler.py index e5b1d17..fbae4ba 100644 --- a/audiostats/handlers/plst_handler.py +++ b/audiostats/handlers/plst_handler.py @@ -7,14 +7,18 @@ from typing import Any from collections.abc import Iterator from cuetools import TrackData +from types import ModuleType + from .models import AlbumDTO, TrackDTO +librosa: ModuleType | None +LIBROSA_AVAILABLE = False + try: import librosa LIBROSA_AVAILABLE = True except ImportError: librosa = None - LIBROSA_AVAILABLE = False MIN_TRACK_DURATION = 10 #Used to decide whether there are more tracks in the file or whether a new file should be started @@ -78,8 +82,10 @@ def _process_cue(self, path: str) -> AlbumDTO | None: return album except FileNotFoundError: logger.warning(f'No such file: {path}') + return None except UnicodeDecodeError: logger.warning(f'Cant read file: {path}') + return None def _get_cover_path(self, current_dir: str) -> str | None: @@ -90,29 +96,32 @@ def _get_cover_path(self, current_dir: str) -> str | None: return None def _process_cue_tracks(self, cue : cuetools.AlbumData, current_dir : str) -> Iterator[TrackDTO]: - offset = 0 + offset: float = 0 + duration: float for track_cue in sorted(cue.tracks, reverse=True, key=lambda x: int(x.track)): - offset, duration = self._get_offset_duration(current_dir, track_cue, offset) if LIBROSA_AVAILABLE else (None, None) + offset, duration = self._get_offset_duration(current_dir, track_cue, offset) if LIBROSA_AVAILABLE else (0.0,0.0) if not (title:=track_cue.title): logger.warning(f'No title in track: {track_cue.track}]') else: track = TrackDTO(title=title, number=int(track_cue.track), path=os.path.join(current_dir, track_cue.link), - offset=offset, - duration=duration) + offset=offset if LIBROSA_AVAILABLE else None, + duration=duration if LIBROSA_AVAILABLE else None) yield track def _get_offset_duration(self, current_dir : str, track_cue : TrackData, next_offset : float | None) -> tuple[float, float]: offset = frame_t_sec(track_cue.index['01']) if track_cue.index['01'] else 0 - duration = next_offset - offset if next_offset >= MIN_TRACK_DURATION else self._get_audiofile_duration(os.path.join(current_dir,track_cue.link)) - offset + duration = next_offset - offset if next_offset is not None and next_offset >= MIN_TRACK_DURATION else self._get_audiofile_duration(os.path.join(current_dir,track_cue.link)) - offset return offset, duration def _get_audiofile_duration(self, path_to_file: str) -> float: if path_to_file.endswith('.ape'): return self._get_ape_duration(path_to_file) else: - return librosa.get_duration(path=path_to_file) + if librosa is not None: + return librosa.get_duration(path=path_to_file) + raise RuntimeError('librosa is unavailable') def _get_ape_duration(self, path_to_file: str) -> float: result = subprocess.run( diff --git a/migrations/env.py b/migrations/env.py index 5a2b077..25a9e02 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -16,7 +16,11 @@ # access to the values within the .ini file in use. config = context.config -config.set_main_option('sqlalchemy.url', os.getenv('SYNC_DB_URL')) +# get config from env +sync_db_url = os.getenv('SYNC_DB_URL') +if not sync_db_url: + raise ValueError('SYNC_DB_URL env variable is required for Alembic') +config.set_main_option('sqlalchemy.url', sync_db_url) # Interpret the config file for Python logging. diff --git a/poetry.lock b/poetry.lock index d612d10..0dc6396 100644 --- a/poetry.lock +++ b/poetry.lock @@ -861,6 +861,78 @@ files = [ {file = "msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd"}, ] +[[package]] +name = "mypy" +version = "1.18.2" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "mypy-1.18.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eab0cf6294dafe397c261a75f96dc2c31bffe3b944faa24db5def4e2b0f77c"}, + {file = "mypy-1.18.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7a780ca61fc239e4865968ebc5240bb3bf610ef59ac398de9a7421b54e4a207e"}, + {file = "mypy-1.18.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448acd386266989ef11662ce3c8011fd2a7b632e0ec7d61a98edd8e27472225b"}, + {file = "mypy-1.18.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f9e171c465ad3901dc652643ee4bffa8e9fef4d7d0eece23b428908c77a76a66"}, + {file = "mypy-1.18.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:592ec214750bc00741af1f80cbf96b5013d81486b7bb24cb052382c19e40b428"}, + {file = "mypy-1.18.2-cp310-cp310-win_amd64.whl", hash = "sha256:7fb95f97199ea11769ebe3638c29b550b5221e997c63b14ef93d2e971606ebed"}, + {file = "mypy-1.18.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:807d9315ab9d464125aa9fcf6d84fde6e1dc67da0b6f80e7405506b8ac72bc7f"}, + {file = "mypy-1.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:776bb00de1778caf4db739c6e83919c1d85a448f71979b6a0edd774ea8399341"}, + {file = "mypy-1.18.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1379451880512ffce14505493bd9fe469e0697543717298242574882cf8cdb8d"}, + {file = "mypy-1.18.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1331eb7fd110d60c24999893320967594ff84c38ac6d19e0a76c5fd809a84c86"}, + {file = "mypy-1.18.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ca30b50a51e7ba93b00422e486cbb124f1c56a535e20eff7b2d6ab72b3b2e37"}, + {file = "mypy-1.18.2-cp311-cp311-win_amd64.whl", hash = "sha256:664dc726e67fa54e14536f6e1224bcfce1d9e5ac02426d2326e2bb4e081d1ce8"}, + {file = "mypy-1.18.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33eca32dd124b29400c31d7cf784e795b050ace0e1f91b8dc035672725617e34"}, + {file = "mypy-1.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3c47adf30d65e89b2dcd2fa32f3aeb5e94ca970d2c15fcb25e297871c8e4764"}, + {file = "mypy-1.18.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d6c838e831a062f5f29d11c9057c6009f60cb294fea33a98422688181fe2893"}, + {file = "mypy-1.18.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01199871b6110a2ce984bde85acd481232d17413868c9807e95c1b0739a58914"}, + {file = "mypy-1.18.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a2afc0fa0b0e91b4599ddfe0f91e2c26c2b5a5ab263737e998d6817874c5f7c8"}, + {file = "mypy-1.18.2-cp312-cp312-win_amd64.whl", hash = "sha256:d8068d0afe682c7c4897c0f7ce84ea77f6de953262b12d07038f4d296d547074"}, + {file = "mypy-1.18.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:07b8b0f580ca6d289e69209ec9d3911b4a26e5abfde32228a288eb79df129fcc"}, + {file = "mypy-1.18.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed4482847168439651d3feee5833ccedbf6657e964572706a2adb1f7fa4dfe2e"}, + {file = "mypy-1.18.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3ad2afadd1e9fea5cf99a45a822346971ede8685cc581ed9cd4d42eaf940986"}, + {file = "mypy-1.18.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a431a6f1ef14cf8c144c6b14793a23ec4eae3db28277c358136e79d7d062f62d"}, + {file = "mypy-1.18.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7ab28cc197f1dd77a67e1c6f35cd1f8e8b73ed2217e4fc005f9e6a504e46e7ba"}, + {file = "mypy-1.18.2-cp313-cp313-win_amd64.whl", hash = "sha256:0e2785a84b34a72ba55fb5daf079a1003a34c05b22238da94fcae2bbe46f3544"}, + {file = "mypy-1.18.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:62f0e1e988ad41c2a110edde6c398383a889d95b36b3e60bcf155f5164c4fdce"}, + {file = "mypy-1.18.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8795a039bab805ff0c1dfdb8cd3344642c2b99b8e439d057aba30850b8d3423d"}, + {file = "mypy-1.18.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ca1e64b24a700ab5ce10133f7ccd956a04715463d30498e64ea8715236f9c9c"}, + {file = "mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb"}, + {file = "mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075"}, + {file = "mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf"}, + {file = "mypy-1.18.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25a9c8fb67b00599f839cf472713f54249a62efd53a54b565eb61956a7e3296b"}, + {file = "mypy-1.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2b9c7e284ee20e7598d6f42e13ca40b4928e6957ed6813d1ab6348aa3f47133"}, + {file = "mypy-1.18.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6985ed057513e344e43a26cc1cd815c7a94602fb6a3130a34798625bc2f07b6"}, + {file = "mypy-1.18.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22f27105f1525ec024b5c630c0b9f36d5c1cc4d447d61fe51ff4bd60633f47ac"}, + {file = "mypy-1.18.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:030c52d0ea8144e721e49b1f68391e39553d7451f0c3f8a7565b59e19fcb608b"}, + {file = "mypy-1.18.2-cp39-cp39-win_amd64.whl", hash = "sha256:aa5e07ac1a60a253445797e42b8b2963c9675563a94f11291ab40718b016a7a0"}, + {file = "mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e"}, + {file = "mypy-1.18.2.tar.gz", hash = "sha256:06a398102a5f203d7477b2923dda3634c36727fa5c237d8f859ef90c42a9924b"}, +] + +[package.dependencies] +mypy_extensions = ">=1.0.0" +pathspec = ">=0.9.0" +typing_extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, + {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, +] + [[package]] name = "numba" version = "0.61.2" @@ -973,6 +1045,18 @@ files = [ {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + [[package]] name = "platformdirs" version = "4.4.0" @@ -1550,7 +1634,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "ML", "db"] +groups = ["main", "ML", "db", "dev"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -1577,4 +1661,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "483aabdbb8ca42dccf64e6d4f49362af5758448962b58a3c85861742cb915836" +content-hash = "ce38720afde62d0476520ff6ca2900d5f35be9020b8e9afb631b8a7417641269" diff --git a/pyproject.toml b/pyproject.toml index f9c1861..8c9f1cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,11 @@ pytest = "^8.4.1" pytest-cov = "^6.2.1" pytest-asyncio = "^1.1.0" dotenv = "^0.9.9" +mypy = "^1.18.2" + +[tool.mypy] +python_version = "3.12" +ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests/"] From d065eb8cee4ca230e31d4d165d7caf238f79d852 Mon Sep 17 00:00:00 2001 From: Olezhich Date: Wed, 22 Oct 2025 20:56:40 +0300 Subject: [PATCH 2/5] ci upgrade, ready to add mypy test --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0f72f06..22a685c 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -5,7 +5,7 @@ on: branches: [ main ] jobs: - test: + ci: runs-on: ubuntu-latest env: From 75ff8c8bbeb3a07bd65f3bffa8dfc5cb1198f486 Mon Sep 17 00:00:00 2001 From: Olezhich Date: Wed, 22 Oct 2025 21:15:51 +0300 Subject: [PATCH 3/5] added mypy to ci --- .github/workflows/run_tests.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 22a685c..e191d02 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -2,7 +2,11 @@ name: Run Tests on PR on: pull_request: + types: + [opened, edited, reopened] branches: [ main ] + paths-ignore: + - 'README.md' jobs: ci: @@ -28,7 +32,8 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - name: Get repository code + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -47,9 +52,13 @@ jobs: run: | poetry run alembic upgrade head + - name: Run Mypy check + run: | + poetry run mypy . + - name: Run tests and generate coverage report run: | - poetry run pytest --cov=audiostats --cov-report=lcov tests/ + poetry run pytest --cov-report=lcov tests/ - name: Upload coverage to Coveralls uses: coverallsapp/github-action@master From a40b5d5d29972876eced43521993c93fe5d3f402 Mon Sep 17 00:00:00 2001 From: Olezhich Date: Wed, 22 Oct 2025 21:28:10 +0300 Subject: [PATCH 4/5] fixed librosa import issue --- audiostats/handlers/plst_handler.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/audiostats/handlers/plst_handler.py b/audiostats/handlers/plst_handler.py index fbae4ba..307bf35 100644 --- a/audiostats/handlers/plst_handler.py +++ b/audiostats/handlers/plst_handler.py @@ -4,21 +4,24 @@ import subprocess import logging -from typing import Any +from typing import Any, TYPE_CHECKING from collections.abc import Iterator from cuetools import TrackData from types import ModuleType from .models import AlbumDTO, TrackDTO -librosa: ModuleType | None +librosa: ModuleType | None = None LIBROSA_AVAILABLE = False try: - import librosa + #import librosa + import librosa as _librosa + librosa = _librosa + LIBROSA_AVAILABLE = True except ImportError: - librosa = None + pass MIN_TRACK_DURATION = 10 #Used to decide whether there are more tracks in the file or whether a new file should be started From be012b2a1de8e56469c17f00fea7792879b17b29 Mon Sep 17 00:00:00 2001 From: Olezhich Date: Wed, 22 Oct 2025 21:33:16 +0300 Subject: [PATCH 5/5] fixed ci pr issue --- .github/workflows/run_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index e191d02..a65ebb4 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -2,8 +2,6 @@ name: Run Tests on PR on: pull_request: - types: - [opened, edited, reopened] branches: [ main ] paths-ignore: - 'README.md'