From 9d9376ca7b2e1d68d1571be66ae9c8b44cfbfa84 Mon Sep 17 00:00:00 2001 From: Alex <25013571+alexhb1@users.noreply.github.com> Date: Wed, 15 Apr 2026 18:48:50 +0100 Subject: [PATCH] Direct source refactor --- .github/workflows/ci.yml | 85 +-- Makefile | 106 ++-- docs/configuration.md | 4 +- docs/environment-variables.md | 178 ++++--- docs/index.md | 22 +- docs/installation.md | 32 +- docs/url-search-parameters.md | 4 +- readme.md | 65 +-- scripts/generate_env_docs.py | 3 + shelfmark/bypass/internal_bypasser.py | 12 +- shelfmark/config/settings.py | 278 +++++----- shelfmark/core/mirrors.py | 285 +++++------ shelfmark/core/onboarding.py | 482 ++++++++++++++---- shelfmark/core/settings_registry.py | 277 ++++++---- shelfmark/download/http.py | 23 +- shelfmark/download/network.py | 16 +- shelfmark/main.py | 5 +- shelfmark/release_sources/direct_download.py | 67 ++- src/README.md | 4 +- src/frontend/src/App.tsx | 1 + src/frontend/src/components/Dropdown.tsx | 135 +++-- .../src/components/OnboardingModal.tsx | 160 +++--- src/frontend/src/components/SearchSection.tsx | 4 +- .../components/UrlSearchBootstrapMount.tsx | 2 +- .../users/UserSearchPreferencesSection.tsx | 4 +- src/frontend/src/hooks/useSearch.ts | 4 +- src/frontend/src/services/api.ts | 1 + src/frontend/src/types/index.ts | 1 + tests/README.md | 5 +- tests/bypass/test_internal_bypasser.py | 46 ++ tests/config/test_download_settings.py | 76 +++ tests/config/test_generate_env_docs.py | 40 ++ .../config/test_mirror_settings_migration.py | 203 ++++++++ tests/core/test_config_api.py | 2 + tests/core/test_mirrors_config.py | 115 ++++- tests/core/test_onboarding.py | 180 +++++++ tests/direct_download/test_search_queries.py | 24 +- .../test_source_availability.py | 97 ++++ .../download/test_http_bypasser_fallbacks.py | 1 + tests/download/test_network_dns_failover.py | 17 + 40 files changed, 2162 insertions(+), 904 deletions(-) create mode 100644 tests/config/test_generate_env_docs.py create mode 100644 tests/config/test_mirror_settings_migration.py create mode 100644 tests/core/test_onboarding.py create mode 100644 tests/direct_download/test_source_availability.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c16f5f0b..3e4c6d8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,8 @@ permissions: contents: read jobs: - backend-quality: - name: Backend Quality + python-quality: + name: Python Quality runs-on: ubuntu-latest steps: - name: Checkout @@ -25,23 +25,17 @@ jobs: - name: Sync dependencies run: make install-python-dev - - name: Lint backend + - name: Lint run: make python-lint - - name: Check backend formatting - run: make python-format-check + - name: Check formatting + run: make python-format - - name: Check backend dead code + - name: Check dead code run: make python-dead-code - - name: Lint tests - run: make python-test-lint - - - name: Check test formatting - run: make python-test-format-check - - backend-typechecks: - name: Backend Typechecks + python-typechecks: + name: Python Typechecks runs-on: ubuntu-latest steps: - name: Checkout @@ -57,14 +51,11 @@ jobs: - name: Sync dependencies run: make install-python-dev - - name: Typecheck backend + - name: Typecheck run: make python-typecheck - - name: Typecheck tests - run: make python-test-typecheck - - backend-tests: - name: Backend Tests + python-tests: + name: Python Tests runs-on: ubuntu-latest steps: - name: Checkout @@ -81,7 +72,7 @@ jobs: run: make install-python-dev - name: Run tests - run: uv run pytest tests/ -x --tb=short -m "not integration and not e2e" + run: make python-test docker-build-check: runs-on: ubuntu-latest @@ -103,7 +94,8 @@ jobs: BUILD_VERSION=pr-${{ github.sha }} RELEASE_VERSION=pr-${{ github.event.pull_request.number }} - frontend-checks: + frontend-quality: + name: Frontend Quality runs-on: ubuntu-latest steps: - name: Checkout @@ -117,21 +109,50 @@ jobs: cache-dependency-path: src/frontend/package-lock.json - name: Install dependencies - working-directory: src/frontend - run: npm ci + run: make install-ci - name: Lint - working-directory: src/frontend - run: npm run lint + run: make frontend-lint - name: Check formatting - working-directory: src/frontend - run: npm run format:check + run: make frontend-format + + frontend-typechecks: + name: Frontend Typechecks + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Node + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + cache: "npm" + cache-dependency-path: src/frontend/package-lock.json + + - name: Install dependencies + run: make install-ci - name: Typecheck - working-directory: src/frontend - run: npm run typecheck + run: make frontend-typecheck + + frontend-tests: + name: Frontend Tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Node + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + cache: "npm" + cache-dependency-path: src/frontend/package-lock.json + + - name: Install dependencies + run: make install-ci - name: Unit tests - working-directory: src/frontend - run: npm run test:unit + run: make frontend-test diff --git a/Makefile b/Makefile index 443f06f9..72aafb47 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install install-python-dev dev build preview typecheck frontend-lint frontend-format frontend-format-check frontend-checks frontend-test clean up up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-check python-typecheck python-dead-code python-checks python-test-lint python-test-lint-fix python-test-format python-test-format-check python-test-typecheck python-test-checks python-coverage prek-install +.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov checks fix # Frontend directory FRONTEND_DIR := src/frontend @@ -10,34 +10,34 @@ COMPOSE_FILE := docker-compose.dev.yml help: @echo "Available targets:" @echo "" + @echo "Quality:" + @echo " checks - Run ALL static analysis checks (frontend + Python)" + @echo " fix - Auto-fix lint + format issues (frontend + Python)" + @echo "" @echo "Frontend:" @echo " install - Install frontend dependencies" @echo " dev - Start development server" @echo " build - Build frontend for production" @echo " build-serve - Build and serve via Flask (test prod build without Docker)" @echo " preview - Preview production build" - @echo " typecheck - Run TypeScript type checking" + @echo " frontend-typecheck - Run TypeScript type checking" @echo " frontend-lint - Run Oxlint against frontend code" - @echo " frontend-format - Format frontend code with Oxfmt" - @echo " frontend-format-check - Check frontend formatting with Oxfmt" + @echo " frontend-format - Check frontend formatting with Oxfmt" + @echo " frontend-format-fix - Format frontend code with Oxfmt" @echo " frontend-checks - Run all frontend static analysis checks" @echo " frontend-test - Run frontend unit tests" + @echo "" + @echo "Python:" @echo " install-python-dev - Sync Python runtime + dev tooling with uv" - @echo " python-lint - Run Ruff against Python backend code" + @echo " python-lint - Run Ruff against Python code (backend + tests)" @echo " python-lint-fix - Run Ruff with safe auto-fixes" - @echo " python-format - Format Python backend code with Ruff" - @echo " python-format-check - Check Python backend formatting with Ruff" - @echo " python-typecheck - Run BasedPyright against Python backend code" - @echo " python-dead-code - Run Vulture against Python backend code" + @echo " python-format - Check Python formatting with Ruff" + @echo " python-format-fix - Format Python code with Ruff" + @echo " python-typecheck - Run BasedPyright against backend + tests" + @echo " python-dead-code - Run Vulture against backend code" @echo " python-checks - Run all Python static analysis checks" - @echo " python-test-lint - Run Ruff against Python tests with the relaxed tests profile" - @echo " python-test-lint-fix - Run Ruff with safe auto-fixes against Python tests" - @echo " python-test-format - Format Python tests with Ruff" - @echo " python-test-format-check - Check Python test formatting with Ruff" - @echo " python-test-typecheck - Run lightweight BasedPyright checks against Python tests" - @echo " python-test-checks - Run all relaxed Python test static analysis checks" - @echo " python-coverage - Run tests with coverage report" - @echo " prek-install - Install prek git hooks" + @echo " python-test - Run unit tests" + @echo " python-test-cov - Run unit tests with coverage report" @echo " clean - Remove node_modules and build artifacts" @echo "" @echo "Backend (Docker):" @@ -52,6 +52,10 @@ install: @echo "Installing frontend dependencies..." cd $(FRONTEND_DIR) && npm install +install-ci: + @echo "Installing frontend dependencies (CI, lockfile-strict)..." + cd $(FRONTEND_DIR) && npm ci + # Install Python development dependencies install-python-dev: @echo "Syncing Python runtime and dev tooling with uv..." @@ -82,67 +86,47 @@ preview: cd $(FRONTEND_DIR) && npm run preview # Type checking -typecheck: +frontend-typecheck: @echo "Running TypeScript type checking..." cd $(FRONTEND_DIR) && npm run typecheck -# Python linting +# Python linting (backend + tests) python-lint: @echo "Running Ruff..." - uv run ruff check shelfmark + uv run ruff check shelfmark tests python-lint-fix: @echo "Running Ruff with safe auto-fixes..." - uv run ruff check shelfmark --fix + uv run ruff check shelfmark tests --fix python-format: - @echo "Formatting Python backend code with Ruff..." - uv run ruff format shelfmark + @echo "Checking Python formatting with Ruff..." + uv run ruff format --check shelfmark tests -python-format-check: - @echo "Checking Python backend formatting with Ruff..." - uv run ruff format --check shelfmark +python-format-fix: + @echo "Formatting Python code with Ruff..." + uv run ruff format shelfmark tests python-typecheck: @echo "Running BasedPyright..." uv run basedpyright + @echo "Running BasedPyright against tests..." + uv run basedpyright tests --skipunannotated python-dead-code: @echo "Running Vulture..." uv run vulture shelfmark -python-checks: python-lint python-format-check python-typecheck python-dead-code - -python-test-lint: - @echo "Running Ruff against tests with the relaxed tests profile..." - uv run ruff check tests - -python-test-lint-fix: - @echo "Running Ruff with safe auto-fixes against tests..." - uv run ruff check tests --fix +python-checks: python-lint python-format python-typecheck python-dead-code -python-test-format: - @echo "Formatting Python tests with Ruff..." - uv run ruff format tests +python-test: + @echo "Running tests..." + uv run pytest tests/ -x --tb=short -m "not integration and not e2e" -python-test-format-check: - @echo "Checking Python test formatting with Ruff..." - uv run ruff format --check tests - -python-test-typecheck: - @echo "Running lightweight BasedPyright checks against tests..." - uv run basedpyright tests --skipunannotated - -python-test-checks: python-test-lint python-test-format-check python-test-typecheck - -python-coverage: +python-test-cov: @echo "Running tests with coverage..." uv run pytest tests/ -x --tb=short -m "not integration and not e2e" --cov --cov-report=term-missing -prek-install: - @echo "Installing prek git hooks..." - uv run prek install - # Frontend linting frontend-lint: @echo "Running Oxlint..." @@ -150,21 +134,27 @@ frontend-lint: # Frontend formatting frontend-format: - @echo "Formatting frontend code with Oxfmt..." - cd $(FRONTEND_DIR) && npm run format - -frontend-format-check: @echo "Checking frontend formatting with Oxfmt..." cd $(FRONTEND_DIR) && npm run format:check +frontend-format-fix: + @echo "Formatting frontend code with Oxfmt..." + cd $(FRONTEND_DIR) && npm run format + # All frontend static analysis -frontend-checks: frontend-lint frontend-format-check typecheck +frontend-checks: frontend-lint frontend-format frontend-typecheck # Run frontend unit tests frontend-test: @echo "Running frontend unit tests..." cd $(FRONTEND_DIR) && npm run test:unit +# All static analysis checks (frontend + Python) +checks: frontend-checks python-checks + +# Auto-fix lint + format issues (frontend + Python) +fix: python-lint-fix python-format-fix frontend-format-fix + # Clean build artifacts and dependencies clean: @echo "Cleaning build artifacts and dependencies..." diff --git a/docs/configuration.md b/docs/configuration.md index a82474a4..5b1632f8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -18,9 +18,9 @@ Prowlarr -> Download client saves to Key point: For torrent and usenet downloads, Shelfmark must see the same file path that your download client reports. The container path must match in both containers. -## Direct Download Setup +## Direct Download Volume Setup -Direct downloads do not use an external download client. A simple two-folder setup is enough. +If you plan to use Direct Download, it does not use an external download client. A simple two-folder setup is enough. Required volumes: diff --git a/docs/environment-variables.md b/docs/environment-variables.md index b2ce52d0..71521efa 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -21,7 +21,6 @@ This document lists all configuration options that can be set via environment va - [Hardcover](#metadata-providers-hardcover) - [Open Library](#metadata-providers-open-library) - [Google Books](#metadata-providers-google-books) - - [Direct Download](#direct-download) - [Download Sources](#direct-download-download-sources) - [Cloudflare Bypass](#direct-download-cloudflare-bypass) @@ -125,7 +124,7 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme | Variable | Description | Type | Default | |----------|-------------|------|---------| -| `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Booklore, etc). | string | _none_ | +| `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). | string | _none_ | | `AUDIOBOOK_LIBRARY_URL` | Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text. | string | _none_ | | `SUPPORTED_FORMATS` | Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found. | string (comma-separated) | `epub,mobi,azw3,fb2,djvu,cbz,cbr` | | `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3` | @@ -138,7 +137,7 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme **Library URL** -Adds a navigation button to your book library (Calibre-Web Automated, Booklore, etc). +Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). - **Type:** string - **Default:** _none_ @@ -185,12 +184,14 @@ Default language filter for searches. | Variable | Description | Type | Default | |----------|-------------|------|---------| -| `SEARCH_MODE` | How you want to search for and download books. | string (choice) | `direct` | +| `SEARCH_MODE` | How you want to search for and download books. | string (choice) | `universal` | | `AA_DEFAULT_SORT` | Default sort order for search results. | string (choice) | `relevance` | | `SHOW_RELEASE_SOURCE_LINKS` | Show clickable release-source links in release and details modals. Metadata provider links stay enabled. | boolean | `true` | +| `SHOW_COMBINED_SELECTOR` | Show the option to search for and download both a book and audiobook together. | boolean | `true` | | `METADATA_PROVIDER` | Choose which metadata provider to use for book searches. | string (choice) | `openlibrary` | | `METADATA_PROVIDER_AUDIOBOOK` | Metadata provider for audiobook searches. Uses the book provider if not set. | string (choice) | _empty string_ | -| `DEFAULT_RELEASE_SOURCE` | The release source tab to open by default in the release modal for books. | string (choice) | `direct_download` | +| `METADATA_PROVIDER_COMBINED` | Metadata provider for combined mode searches. Uses the book provider if not set. | string (choice) | _empty string_ | +| `DEFAULT_RELEASE_SOURCE` | The release source tab to open by default in the release modal for books. Leave unset to use the first available source. | string (choice) | _empty string_ | | `DEFAULT_RELEASE_SOURCE_AUDIOBOOK` | The release source tab to open by default in the release modal for audiobooks. Uses the book release source if not set. | string (choice) | _empty string_ |
@@ -203,7 +204,7 @@ Default language filter for searches. How you want to search for and download books. - **Type:** string (choice) -- **Default:** `direct` +- **Default:** `universal` - **Options:** `direct` (Direct), `universal` (Universal) #### `AA_DEFAULT_SORT` @@ -225,6 +226,15 @@ Show clickable release-source links in release and details modals. Metadata prov - **Type:** boolean - **Default:** `true` +#### `SHOW_COMBINED_SELECTOR` + +**Show Combined Download Selector** + +Show the option to search for and download both a book and audiobook together. + +- **Type:** boolean +- **Default:** `true` + #### `METADATA_PROVIDER` **Book Metadata Provider** @@ -233,7 +243,7 @@ Choose which metadata provider to use for book searches. - **Type:** string (choice) - **Default:** `openlibrary` -- **Options:** `hardcover` (Hardcover), `openlibrary` (Open Library), `googlebooks` (Google Books) +- **Options:** `""` (No providers enabled) #### `METADATA_PROVIDER_AUDIOBOOK` @@ -243,17 +253,27 @@ Metadata provider for audiobook searches. Uses the book provider if not set. - **Type:** string (choice) - **Default:** _empty string_ -- **Options:** `""` (Use book provider), `hardcover` (Hardcover), `openlibrary` (Open Library), `googlebooks` (Google Books) +- **Options:** `""` (Use book provider), `""` (No providers enabled) + +#### `METADATA_PROVIDER_COMBINED` + +**Combined Mode Metadata Provider** + +Metadata provider for combined mode searches. Uses the book provider if not set. + +- **Type:** string (choice) +- **Default:** _empty string_ +- **Options:** `""` (Use book provider), `""` (No providers enabled) #### `DEFAULT_RELEASE_SOURCE` **Default Book Release Source** -The release source tab to open by default in the release modal for books. +The release source tab to open by default in the release modal for books. Leave unset to use the first available source. - **Type:** string (choice) -- **Default:** `direct_download` -- **Options:** `direct_download` (Direct Download), `prowlarr` (Prowlarr) +- **Default:** _empty string_ +- **Options:** `""` (Use first available source) #### `DEFAULT_RELEASE_SOURCE_AUDIOBOOK` @@ -263,7 +283,7 @@ The release source tab to open by default in the release modal for audiobooks. U - **Type:** string (choice) - **Default:** _empty string_ -- **Options:** `""` (Use book release source), `prowlarr` (Prowlarr), `audiobookbay` (AudiobookBay) +- **Options:** `""` (Use book release source)
@@ -277,12 +297,12 @@ The release source tab to open by default in the release modal for audiobooks. U | `TEMPLATE_RENAME` | Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title} ({Year})` | | `TEMPLATE_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title} ({Year})` | | `HARDLINK_TORRENTS` | Create hardlinks instead of copying. Preserves seeding but archives won't be extracted. Don't use if destination is a library ingest folder. | boolean | `false` | -| `BOOKLORE_HOST` | Base URL of your Booklore instance | string | _none_ | -| `BOOKLORE_USERNAME` | Booklore account username | string | _none_ | -| `BOOKLORE_PASSWORD` | Booklore account password | string (secret) | _none_ | +| `BOOKLORE_HOST` | Base URL of your Grimmory instance | string | _none_ | +| `BOOKLORE_USERNAME` | Grimmory account username | string | _none_ | +| `BOOKLORE_PASSWORD` | Grimmory account password | string (secret) | _none_ | | `BOOKLORE_DESTINATION` | Choose whether uploads go directly to a specific library path or to Bookdrop for review. | string (choice) | `library` | -| `BOOKLORE_LIBRARY_ID` | Booklore library to upload into. | string (choice) | _none_ | -| `BOOKLORE_PATH_ID` | Booklore library path for uploads. | string (choice) | _none_ | +| `BOOKLORE_LIBRARY_ID` | Grimmory library to upload into. | string (choice) | _none_ | +| `BOOKLORE_PATH_ID` | Grimmory library path for uploads. | string (choice) | _none_ | | `EMAIL_RECIPIENT` | Optional fallback email address when no per-user email recipient override is configured. | string | _none_ | | `EMAIL_ATTACHMENT_SIZE_LIMIT_MB` | Maximum total attachment size per email. Email encoding adds overhead; keep this below your provider's limit. | number | `25` | | `EMAIL_SMTP_HOST` | SMTP server hostname or IP (e.g., smtp.gmail.com). | string | _none_ | @@ -315,7 +335,7 @@ Choose where completed book files are sent. - **Type:** string (choice) - **Default:** `folder` -- **Options:** `folder` (Folder), `email` (Email (SMTP)), `booklore` (Booklore (API)) +- **Options:** `folder` (Folder), `email` (Email (SMTP)), `booklore` (Grimmory (API)) #### `INGEST_DIR` @@ -366,9 +386,9 @@ Create hardlinks instead of copying. Preserves seeding but archives won't be ext #### `BOOKLORE_HOST` -**Booklore URL** +**Grimmory URL** -Base URL of your Booklore instance +Base URL of your Grimmory instance - **Type:** string - **Default:** _none_ @@ -378,7 +398,7 @@ Base URL of your Booklore instance **Username** -Booklore account username +Grimmory account username - **Type:** string - **Default:** _none_ @@ -388,7 +408,7 @@ Booklore account username **Password** -Booklore account password +Grimmory account password - **Type:** string (secret) - **Default:** _none_ @@ -408,7 +428,7 @@ Choose whether uploads go directly to a specific library path or to Bookdrop for **Library** -Booklore library to upload into. +Grimmory library to upload into. - **Type:** string (choice) - **Default:** _none_ @@ -418,7 +438,7 @@ Booklore library to upload into. **Path** -Booklore library path for uploads. +Grimmory library path for uploads. - **Type:** string (choice) - **Default:** _none_ @@ -627,7 +647,7 @@ How long to keep completed/failed downloads in the queue display. | `OIDC_DISCOVERY_URL` | OpenID Connect discovery endpoint URL. Usually ends with /.well-known/openid-configuration. | string | _none_ | | `OIDC_CLIENT_ID` | OAuth2 client ID from your identity provider. | string | _none_ | | `OIDC_CLIENT_SECRET` | OAuth2 client secret from your identity provider. | string (secret) | _none_ | -| `OIDC_SCOPES` | OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization. | string | `openid,email,profile` | +| `OIDC_SCOPES` | OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization. | string (comma-separated) | `openid,email,profile` | | `OIDC_GROUP_CLAIM` | The name of the claim in the ID token that contains user groups. | string | `groups` | | `OIDC_ADMIN_GROUP` | Users in this group will be given admin access (if enabled below). Leave empty to use database roles only. | string | _empty string_ | | `OIDC_USE_ADMIN_GROUP` | When enabled, users in the Admin Group are granted admin access. When disabled, admin access is determined solely by database roles. | boolean | `true` | @@ -719,7 +739,7 @@ OAuth2 client secret from your identity provider. OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization. -- **Type:** string +- **Type:** string (comma-separated) - **Default:** `openid,email,profile` #### `OIDC_GROUP_CLAIM` @@ -1253,7 +1273,7 @@ How long to keep cached search results before they expire. | `QBITTORRENT_CATEGORY` | Category to assign to book downloads in qBittorrent | string | `books` | | `QBITTORRENT_CATEGORY_AUDIOBOOK` | Category for audiobook downloads. Leave empty to use the book category. | string | _empty string_ | | `QBITTORRENT_DOWNLOAD_DIR` | Server-side directory where torrents are downloaded (optional, uses qBittorrent default if not specified) | string | _none_ | -| `QBITTORRENT_TAG` | Tag(s) to assign to qBittorrent downloads. Leave empty for no tags. | string | _empty list_ | +| `QBITTORRENT_TAG` | Tag(s) to assign to qBittorrent downloads. Leave empty for no tags. | string (comma-separated) | _empty list_ | | `TRANSMISSION_URL` | URL of your Transmission instance (use https:// for TLS) | string | _none_ | | `TRANSMISSION_USERNAME` | Transmission RPC username (if authentication enabled) | string | _none_ | | `TRANSMISSION_PASSWORD` | Transmission RPC password | string (secret) | _none_ | @@ -1357,7 +1377,7 @@ Server-side directory where torrents are downloaded (optional, uses qBittorrent Tag(s) to assign to qBittorrent downloads. Leave empty for no tags. -- **Type:** string +- **Type:** string (comma-separated) - **Default:** _empty list_ #### `TRANSMISSION_URL` @@ -1637,6 +1657,7 @@ Move deletes the job from your usenet client after import; Copy keeps it in the | `HARDCOVER_DEFAULT_SORT` | Default sort order for Hardcover search results. | string (choice) | `relevance` | | `HARDCOVER_EXCLUDE_COMPILATIONS` | Filter out compilations, anthologies, and omnibus editions from search results | boolean | `false` | | `HARDCOVER_EXCLUDE_UNRELEASED` | Filter out books with a release year in the future | boolean | `false` | +| `HARDCOVER_AUTO_REMOVE_ON_DOWNLOAD` | Automatically remove a book from the active Hardcover list when you download it | boolean | `true` |
Detailed descriptions @@ -1688,6 +1709,15 @@ Filter out books with a release year in the future - **Type:** boolean - **Default:** `false` +#### `HARDCOVER_AUTO_REMOVE_ON_DOWNLOAD` + +**Auto-Remove from List on Download** + +Automatically remove a book from the active Hardcover list when you download it + +- **Type:** boolean +- **Default:** `true` +
### Metadata Providers: Open Library @@ -1769,6 +1799,7 @@ Default sort order for Google Books search results. | Variable | Description | Type | Default | |----------|-------------|------|---------| +| `DIRECT_DOWNLOAD_ENABLED` | Show Direct Download in release-source lists and allow Direct mode searches. Add your own mirror URLs in the Mirrors tab before using it. | boolean | `false` | | `AA_DONATOR_KEY` | Enables fast download access on AA. Get this from your donator account page. | string (secret) | _none_ | | `FAST_SOURCES_DISPLAY` | Always tried first, no waiting or bypass required. | JSON array | _see UI for defaults_ | | `SOURCE_PRIORITY` | Fallback sources, may have waiting. Requires bypasser. Drag to reorder. | JSON array | _see UI for defaults_ | @@ -1787,6 +1818,15 @@ Default sort order for Google Books search results.
Detailed descriptions +#### `DIRECT_DOWNLOAD_ENABLED` + +**Enable Direct Download Source** + +Show Direct Download in release-source lists and allow Direct mode searches. Add your own mirror URLs in the Mirrors tab before using it. + +- **Type:** boolean +- **Default:** `false` + #### `AA_DONATOR_KEY` **Account Donator Key** @@ -1971,14 +2011,11 @@ Timeout for external bypasser requests in milliseconds. | Variable | Description | Type | Default | |----------|-------------|------|---------| -| `AA_BASE_URL` | Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback). | string (choice) | `auto` | -| `AA_MIRROR_URLS` | Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation | string | `https://annas-archive.gl,https://annas-archive.pk,https://annas-archive.vg,https://annas-archive.gd` | -| `AA_ADDITIONAL_URLS` | Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables. | string | _none_ | -| `LIBGEN_ADDITIONAL_URLS` | Comma-separated list of custom LibGen mirrors to add to the defaults. | string | _none_ | -| `ZLIB_PRIMARY_URL` | Z-Library mirror to use for downloads. | string (choice) | `https://z-lib.fm` | -| `ZLIB_ADDITIONAL_URLS` | Comma-separated list of custom Z-Library mirror URLs. | string | _none_ | -| `WELIB_PRIMARY_URL` | Welib mirror to use for downloads. | string (choice) | `https://welib.org` | -| `WELIB_ADDITIONAL_URLS` | Comma-separated list of custom Welib mirror URLs. | string | _none_ | +| `AA_BASE_URL` | Select Auto to try mirrors from your list on startup and fail over on errors. Choosing a specific mirror pins Shelfmark to that URL. | string (choice) | `auto` | +| `AA_MIRROR_URLS` | List the Anna's Archive mirror URLs you want Shelfmark to use. Type a URL and press Enter to add it. Order matters when Auto is selected. | string (comma-separated) | _empty list_ | +| `LIBGEN_MIRROR_URLS` | Mirrors are tried in the order you add them until one works. | string (comma-separated) | _empty list_ | +| `ZLIB_MIRROR_URLS` | Only the first mirror in the list is used. | string (comma-separated) | _empty list_ | +| `WELIB_MIRROR_URLS` | Only the first mirror in the list is used. | string (comma-separated) | _empty list_ |
Detailed descriptions @@ -1987,75 +2024,46 @@ Timeout for external bypasser requests in milliseconds. **Primary Mirror** -Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback). +Select Auto to try mirrors from your list on startup and fail over on errors. Choosing a specific mirror pins Shelfmark to that URL. - **Type:** string (choice) - **Default:** `auto` -- **Options:** `auto` (Auto (Recommended)), `https://annas-archive.gl` (annas-archive.gl), `https://annas-archive.pk` (annas-archive.pk), `https://annas-archive.vg` (annas-archive.vg), `https://annas-archive.gd` (annas-archive.gd) +- **Options:** `auto` (Auto (Recommended)) #### `AA_MIRROR_URLS` **Mirrors** -Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation - -- **Type:** string -- **Default:** `https://annas-archive.gl,https://annas-archive.pk,https://annas-archive.vg,https://annas-archive.gd` +List the Anna's Archive mirror URLs you want Shelfmark to use. Type a URL and press Enter to add it. Order matters when Auto is selected. -#### `AA_ADDITIONAL_URLS` - -**Additional Mirrors (Legacy)** - -Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables. - -- **Type:** string -- **Default:** _none_ - -#### `LIBGEN_ADDITIONAL_URLS` - -**Additional Mirrors** - -Comma-separated list of custom LibGen mirrors to add to the defaults. - -- **Type:** string -- **Default:** _none_ - -#### `ZLIB_PRIMARY_URL` - -**Primary Mirror** - -Z-Library mirror to use for downloads. - -- **Type:** string (choice) -- **Default:** `https://z-lib.fm` -- **Options:** `https://z-lib.fm` (z-lib.fm), `https://z-lib.gs` (z-lib.gs), `https://z-lib.id` (z-lib.id), `https://z-library.sk` (z-library.sk), `https://zlibrary-global.se` (zlibrary-global.se) +- **Type:** string (comma-separated) +- **Default:** _empty list_ -#### `ZLIB_ADDITIONAL_URLS` +#### `LIBGEN_MIRROR_URLS` -**Additional Mirrors** +**LibGen** -Comma-separated list of custom Z-Library mirror URLs. +Mirrors are tried in the order you add them until one works. -- **Type:** string -- **Default:** _none_ +- **Type:** string (comma-separated) +- **Default:** _empty list_ -#### `WELIB_PRIMARY_URL` +#### `ZLIB_MIRROR_URLS` -**Primary Mirror** +**Z-Library** -Welib mirror to use for downloads. +Only the first mirror in the list is used. -- **Type:** string (choice) -- **Default:** `https://welib.org` -- **Options:** `https://welib.org` (welib.org) +- **Type:** string (comma-separated) +- **Default:** _empty list_ -#### `WELIB_ADDITIONAL_URLS` +#### `WELIB_MIRROR_URLS` -**Additional Mirrors** +**Welib** -Comma-separated list of custom Welib mirror URLs. +Only the first mirror in the list is used. -- **Type:** string -- **Default:** _none_ +- **Type:** string (comma-separated) +- **Default:** _empty list_
diff --git a/docs/index.md b/docs/index.md index bfbd3b29..97d861c9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,3 +1,23 @@ # Shelfmark Documentation -TODO +Shelfmark is a self-hosted interface for searching, requesting, and delivering books and audiobooks through the sources and services you choose to configure. + +Use the guides below to set up the app, connect your library tools, and understand the main configuration areas. + +## Getting Started + +- [Installation](installation.md) +- [Directory and Volume Setup](configuration.md) +- [Environment Variables](environment-variables.md) + +## Core Guides + +- [Users & Requests](users-and-requests.md) +- [Reverse Proxy](reverse-proxy.md) +- [OIDC](oidc.md) +- [URL Search Parameters](url-search-parameters.md) +- [Custom Scripts](custom-scripts.md) + +## Help + +- [Troubleshooting](troubleshooting.md) diff --git a/docs/installation.md b/docs/installation.md index 7cd42228..b41c2cf9 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,3 +1,33 @@ # Installation -TODO +Shelfmark is typically deployed with Docker Compose. + +## Quick Start + +1. Download the compose file from the repository: + +```bash +curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml +``` + +2. Start the service: + +```bash +docker compose up -d +``` + +3. Open `http://localhost:8084` + +4. Configure the sources, metadata providers, and delivery settings you want to use + +## Next Steps + +- For volume and path setup, see [Directory and Volume Setup](configuration.md) +- For environment-based setup, see [Environment Variables](environment-variables.md) +- For authentication and user management, see [Users & Requests](users-and-requests.md) and [OIDC](oidc.md) + +## Notes + +- Universal search is the default mode for new installs +- Direct Download is optional and must be enabled and configured before it can be used +- Torrent and usenet setups require matching download paths between Shelfmark and your download client diff --git a/docs/url-search-parameters.md b/docs/url-search-parameters.md index aaf49120..131b4386 100644 --- a/docs/url-search-parameters.md +++ b/docs/url-search-parameters.md @@ -65,9 +65,9 @@ Some parameters support multiple values by repeating the parameter: ## Search Mode Behavior -### Direct Download Mode (default) +### Direct Mode -All parameters are used to filter results from the direct download source. +When Search Mode is set to Direct, all parameters are used to filter results from the configured direct source. `content_type` is ignored in Direct mode. ### Universal Mode diff --git a/readme.md b/readme.md index e5b6bc0e..3a558f14 100644 --- a/readme.md +++ b/readme.md @@ -1,12 +1,10 @@ -# 📚 Shelfmark: Book Downloader - -Formerly *Calibre Web Automated Book Downloader (CWABD)* +# 📚 Shelfmark: Book Search & Request Tool Shelfmark -Shelfmark is a self-hosted web interface for searching and downloading books and audiobooks from multiple sources. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own. +Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own. -**Fully standalone** - no external dependencies required. Works great alongside the following library tools, with support for automatic imports: +Works great alongside the following library tools, with support for automatic imports: - [Calibre](https://calibre-ebook.com/) - [Calibre-Web](https://github.com/janeczku/calibre-web) - [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) @@ -15,16 +13,14 @@ Shelfmark is a self-hosted web interface for searching and downloading books and ## ✨ Features -- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place -- **Multiple Sources** - Popular archive websites, Torrent, Usenet, and IRC download support +- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple configured sources in one place +- **Multiple Sources** - Configurable web, torrent, usenet, and IRC source support - **Audiobook Support** - Full audiobook search and download with dedicated processing -- **Two Search Modes**: - - **Direct** - Search popular web sources - - **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads +- **Flexible Search** - Search metadata providers (Hardcover, Open Library, Google Books) for rich book and audiobook discovery, or query configured sources directly - **Multi-User & Requests** - Share your instance with others, let users browse and request books, and manage approvals with configurable notifications - **Authentication** - Built-in login, OIDC single sign-on, proxy auth, and Calibre-Web database support - **Real-Time Progress** - Unified download queue with live status updates across all sources -- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources +- **Network Flexibility** - Configurable proxy support, DNS settings, and optional Cloudflare handling for protected sources ## 🖼️ Screenshots @@ -60,7 +56,7 @@ Shelfmark is a self-hosted web interface for searching and downloading books and 3. Open `http://localhost:8084` -That's it! Configure settings through the web interface as needed. +Open the web interface, then configure the sources and settings you want to use. ### Volume Setup @@ -87,16 +83,13 @@ volumes: ### Search Modes -**Direct** (default) -- Works out of the box, no setup required -- Searches a huge library of books directly -- Returns downloadable releases immediately +**Direct** +- Queries configured sources directly -**Universal** -- Cleaner search results via metadata providers (Hardcover is recommended) +**Universal** (recommended) +- Search via metadata providers (Hardcover, Open Library, Google Books) for richer results - Aggregates releases from multiple configured sources -- Full Audiobook support -- Requires manual setup (API keys, additional sources) +- Full audiobook support ### Environment Variables @@ -108,19 +101,18 @@ Environment variables work for initial setup and Docker deployments. They serve | `INGEST_DIR` | Book download directory | `/books` | | `TZ` | Container timezone | `UTC` | | `PUID` / `PGID` | Runtime user/group for the default root-startup flow (also supports legacy `UID`/`GID`) | `1000` / `1000` | -| `SEARCH_MODE` | `direct` or `universal` | `direct` | +| `SEARCH_MODE` | `direct` or `universal` | `universal` | | `USING_TOR` | Enable Tor routing (requires root startup) | `false` | See the full [Environment Variables Reference](docs/environment-variables.md) for all available options. Some of the additional options available in Settings: -- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads - **Prowlarr** - Configure indexers and download clients to download books and audiobooks -- **AudiobookBay** - Web scraping source for audiobook torrents (audiobooks only) +- **Additional audiobook sources** - Configure additional sources for audiobook discovery - **IRC** - Add details for IRC book sources and download directly from the UI - **Library Link** - Add a link to your Calibre-Web or Grimmory instance in the UI header - **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming -- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing. +- **Network Settings** - Custom proxy support (SOCKS5 + HTTP/S) and configurable DNS - **Format & Language** - Filter downloads by preferred formats, languages and sorting order - **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc. @@ -131,10 +123,10 @@ Some of the additional options available in Settings: docker compose up -d ``` -The full-featured image with built-in Cloudflare bypass. +The full-featured image with all network capabilities included. -#### Enable Tor Routing -Routes all traffic through Tor for enhanced privacy: +#### Tor Routing +Optional Tor support for network privacy: ```bash curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml docker compose -f docker-compose.tor.yml up -d @@ -147,19 +139,18 @@ docker compose -f docker-compose.tor.yml up -d - Custom DNS/proxy settings are ignored when Tor is active ### Lite -A smaller image without the built-in Cloudflare bypasser. Ideal for: +A lighter image without the built-in browser automation. Ideal for: -- **External bypassers** - Already running FlareSolverr or ByParr for other services -- **Fast downloads** - Using fast download sources -- **Alternative sources only** - Exclusively using Prowlarr, AudiobookBay, IRC, or other sources -- **Audiobooks** - Using Shelfmark exclusively for audiobooks +- **External services** - Already running FlareSolverr or similar for other applications +- **Alternative sources** - Using Prowlarr, IRC, or other configured sources +- **Audiobooks** - Using Shelfmark primarily for audiobooks ```bash curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml docker compose -f docker-compose.lite.yml up -d ``` -If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab. +If you need browser-based access with the Lite image, configure an external resolver in Settings. ## 🔐 Authentication @@ -230,16 +221,16 @@ Log level is configurable via Settings or `LOG_LEVEL` environment variable. ## Development ```bash -# Python tooling -make install-python-dev # Sync Python runtime + dev tools with uv +# Quality checks +make checks # Run ALL static analysis (frontend + Python) make python-checks # Run Ruff, BasedPyright, and Vulture -make python-test-checks # Run lightweight lint/type checks for tests +make install-python-dev # Sync Python runtime + dev tools with uv # Frontend development make install # Install dependencies make dev # Start Vite dev server (localhost:5173) make build # Production build -make typecheck # TypeScript checks +make frontend-typecheck # TypeScript checks # Backend (Docker) make up # Start backend via docker-compose.dev.yml diff --git a/scripts/generate_env_docs.py b/scripts/generate_env_docs.py index f88cc37b..d0e6aac3 100755 --- a/scripts/generate_env_docs.py +++ b/scripts/generate_env_docs.py @@ -34,6 +34,7 @@ def get_field_type_name(field: Any) -> str: OrderableListField, PasswordField, SelectField, + TagListField, TextField, ) @@ -45,6 +46,8 @@ def get_field_type_name(field: Any) -> str: return "string (choice)" if isinstance(field, MultiSelectField): return "string (comma-separated)" + if isinstance(field, TagListField): + return "string (comma-separated)" if isinstance(field, OrderableListField): return "JSON array" if isinstance(field, PasswordField): diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index 386dd632..702a01f7 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -218,15 +218,19 @@ def run(self, coro: Any, timeout: float | None = None) -> Any: "ddg_last_challenge", } -# Domains requiring full session cookies (not just protection cookies) -FULL_COOKIE_DOMAINS = {"z-lib.fm", "z-lib.gs", "z-lib.id", "z-library.sk", "zlibrary-global.se"} - def _get_base_domain(domain: str) -> str: """Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com').""" return ".".join(domain.split(".")[-2:]) if "." in domain else domain +def _get_full_cookie_domains() -> set[str]: + """Return mirror domains that need full-session cookie extraction.""" + from shelfmark.core.mirrors import get_zlib_cookie_domains + + return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()} + + def _should_extract_cookie(name: str, *, extract_all: bool) -> bool: """Determine if a cookie should be extracted based on its name.""" if extract_all: @@ -249,7 +253,7 @@ def _store_extracted_cookies( return base_domain = _get_base_domain(domain) - extract_all = base_domain in FULL_COOKIE_DOMAINS + extract_all = base_domain in _get_full_cookie_domains() cookies_found: dict[str, dict[str, Any]] = {} for cookie in cookies: diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index 7c82cc4c..5a2049b0 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -236,14 +236,18 @@ def _get_release_source_options_for_content_type(content_type: str) -> list[dict return [ {"value": source["name"], "label": source["display_name"]} for source in list_available_sources() - if source.get("can_be_default", True) + if source.get("enabled", True) + and source.get("can_be_default", True) and content_type in source.get("supported_content_types", ["ebook", "audiobook"]) ] def _get_book_release_source_options() -> list[dict[str, str]]: """Build default release source options for book searches.""" - return _get_release_source_options_for_content_type("ebook") + return [ + {"value": "", "label": "Use first available source"}, + *_get_release_source_options_for_content_type("ebook"), + ] def _get_audiobook_release_source_options() -> list[dict[str, str]]: @@ -265,18 +269,15 @@ def _string_setting(value: object) -> str: def _get_aa_base_url_options() -> list[dict[str, str]]: - """Build AA URL options dynamically, including additional mirrors from config.""" + """Build AA URL options dynamically from user-supplied mirrors.""" from shelfmark.core.config import config - from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS, get_aa_mirrors + from shelfmark.core.mirrors import get_aa_mirrors from shelfmark.core.utils import normalize_http_url options = [{"value": "auto", "label": "Auto (Recommended)"}] - # Get all mirrors (defaults + custom) all_mirrors = get_aa_mirrors() - # If AA_BASE_URL is configured to a custom mirror that isn't present in the - # defaults/additional list, include it so the UI can display the active value. configured_url = normalize_http_url( _string_setting(config.get("AA_BASE_URL", "auto")), default_scheme="https", @@ -287,63 +288,14 @@ def _get_aa_base_url_options() -> list[dict[str, str]]: for url in all_mirrors: domain = url.replace("https://", "").replace("http://", "") - is_custom = url not in DEFAULT_AA_MIRRORS - label = f"{domain} (custom)" if is_custom else domain - if configured_url and url == configured_url and is_custom: + label = domain + if configured_url and url == configured_url: label = f"{domain} (configured)" options.append({"value": url, "label": label}) return options -def _get_zlib_mirror_options() -> list[dict[str, str]]: - """Build Z-Library mirror options for SelectField.""" - from shelfmark.core.config import config - from shelfmark.core.mirrors import DEFAULT_ZLIB_MIRRORS - - options = [] - - # Add default mirrors - for url in DEFAULT_ZLIB_MIRRORS: - domain = url.replace("https://", "").replace("http://", "") - options.append({"value": url, "label": domain}) - - # Add custom mirrors - additional = _string_setting(config.get("ZLIB_ADDITIONAL_URLS", "")) - if additional: - for raw_url in additional.split(","): - url = raw_url.strip() - if url and url not in DEFAULT_ZLIB_MIRRORS: - domain = url.replace("https://", "").replace("http://", "").split("/")[0] - options.append({"value": url, "label": f"{domain} (custom)"}) - - return options - - -def _get_welib_mirror_options() -> list[dict[str, str]]: - """Build Welib mirror options for SelectField.""" - from shelfmark.core.config import config - from shelfmark.core.mirrors import DEFAULT_WELIB_MIRRORS - - options = [] - - # Add default mirrors - for url in DEFAULT_WELIB_MIRRORS: - domain = url.replace("https://", "").replace("http://", "") - options.append({"value": url, "label": domain}) - - # Add custom mirrors - additional = _string_setting(config.get("WELIB_ADDITIONAL_URLS", "")) - if additional: - for raw_url in additional.split(","): - url = raw_url.strip() - if url and url not in DEFAULT_WELIB_MIRRORS: - domain = url.replace("https://", "").replace("http://", "").split("/")[0] - options.append({"value": url, "label": f"{domain} (custom)"}) - - return options - - def _clear_covers_cache(current_values: dict) -> dict: """Clear the cover image cache.""" try: @@ -393,6 +345,13 @@ def _clear_metadata_cache(current_values: dict) -> dict: def general_settings() -> list[SettingsField]: """Core application settings.""" return [ + TextField( + key="SEARCH_PAGE_TITLE", + label="Search Page Title", + description="Title shown above the main search box on the homepage.", + default="Shelfmark", + placeholder="Shelfmark", + ), TextField( key="CALIBRE_WEB_URL", label="Library URL", @@ -441,7 +400,10 @@ def search_mode_settings() -> list[SettingsField]: HeadingField( key="search_mode_heading", title="Search Mode", - description="Direct mode searches web sources and downloads immediately. Universal mode supports Prowlarr, IRC and audiobooks with metadata-based searching.", + description=( + "Direct mode uses the optional Direct Download source. Universal mode uses " + "metadata search with whichever release sources you have enabled." + ), ), SelectField( key="SEARCH_MODE", @@ -451,7 +413,10 @@ def search_mode_settings() -> list[SettingsField]: { "value": "direct", "label": "Direct", - "description": "Search web sources for books and download directly. Works out of the box.", + "description": ( + "Search with the Direct Download source. Requires enabling the source " + "and adding your own mirror URLs." + ), }, { "value": "universal", @@ -459,7 +424,7 @@ def search_mode_settings() -> list[SettingsField]: "description": "Metadata-based search with downloads from all sources. Book and Audiobook support.", }, ], - default="direct", + default="universal", user_overridable=True, ), SelectField( @@ -523,9 +488,12 @@ def search_mode_settings() -> list[SettingsField]: SelectField( key="DEFAULT_RELEASE_SOURCE", label="Default Book Release Source", - description="The release source tab to open by default in the release modal for books.", + description=( + "The release source tab to open by default in the release modal for books. " + "Leave unset to use the first available source." + ), options=_get_book_release_source_options, # Callable - evaluated lazily to avoid circular imports - default="direct_download", + default="", show_when={"field": "SEARCH_MODE", "value": "universal"}, user_overridable=True, ), @@ -1306,8 +1274,13 @@ def download_settings() -> list[SettingsField]: def _get_fast_source_options() -> list[dict[str, str | bool | int | None]]: """Fast download sources - configurable list shown in settings.""" from shelfmark.core.config import config + from shelfmark.core.mirrors import get_download_source_missing_mirror_reason has_donator_key = bool(config.get("AA_DONATOR_KEY", "")) + aa_fast_reason = get_download_source_missing_mirror_reason("aa-fast") + if not aa_fast_reason and not has_donator_key: + aa_fast_reason = "Requires Donator Key" + libgen_reason = get_download_source_missing_mirror_reason("libgen") return [ { @@ -1315,14 +1288,16 @@ def _get_fast_source_options() -> list[dict[str, str | bool | int | None]]: "label": "AA Fast Downloads", "description": "Fast downloads for donators", "isPinned": True, - "isLocked": not has_donator_key, - "disabledReason": "Requires Donator Key" if not has_donator_key else None, + "isLocked": aa_fast_reason is not None, + "disabledReason": aa_fast_reason, }, { "id": "libgen", "label": "Library Genesis", "description": "Instant downloads, no bypass needed", "isPinned": True, + "isLocked": libgen_reason is not None, + "disabledReason": libgen_reason, }, ] @@ -1338,39 +1313,46 @@ def _get_fast_source_defaults() -> list[dict[str, str | bool]]: def _get_slow_source_options() -> list[dict[str, str | bool | None]]: """Slow download sources - configurable order. All require bypasser.""" from shelfmark.core.config import config + from shelfmark.core.mirrors import get_download_source_missing_mirror_reason bypass_enabled = config.get("USE_CF_BYPASS", True) - locked = not bypass_enabled - disabled_reason = "Requires Cloudflare bypass" if locked else None + + def _get_reason(source_id: str) -> str | None: + mirror_reason = get_download_source_missing_mirror_reason(source_id) + if mirror_reason: + return mirror_reason + if not bypass_enabled: + return "Requires Cloudflare bypass" + return None return [ { "id": "aa-slow-nowait", "label": "AA Slow Downloads (No Waitlist)", "description": "Partner servers", - "isLocked": locked, - "disabledReason": disabled_reason, + "isLocked": _get_reason("aa-slow-nowait") is not None, + "disabledReason": _get_reason("aa-slow-nowait"), }, { "id": "aa-slow-wait", "label": "AA Slow Downloads (Waitlist)", "description": "Partner servers with countdown timer", - "isLocked": locked, - "disabledReason": disabled_reason, + "isLocked": _get_reason("aa-slow-wait") is not None, + "disabledReason": _get_reason("aa-slow-wait"), }, { "id": "welib", "label": "Welib", "description": "Alternative mirror", - "isLocked": locked, - "disabledReason": disabled_reason, + "isLocked": _get_reason("welib") is not None, + "disabledReason": _get_reason("welib"), }, { "id": "zlib", "label": "Zlib", "description": "Alternative mirror", - "isLocked": locked, - "disabledReason": disabled_reason, + "isLocked": _get_reason("zlib") is not None, + "disabledReason": _get_reason("zlib"), }, ] @@ -1393,6 +1375,15 @@ def _get_slow_source_defaults() -> list[dict[str, str | bool]]: def download_source_settings() -> list[SettingsField]: """Return settings for download source behavior.""" return [ + CheckboxField( + key="DIRECT_DOWNLOAD_ENABLED", + label="Enable Direct Download Source", + description=( + "Show Direct Download in release-source lists and allow Direct mode " + "searches. Add your own mirror URLs in the Mirrors tab before using it." + ), + default=False, + ), PasswordField( key="AA_DONATOR_KEY", label="Account Donator Key", @@ -1401,7 +1392,7 @@ def download_source_settings() -> list[SettingsField]: HeadingField( key="source_priority_heading", title="Source Priority", - description="Sources are tried in order until a download succeeds.", + description="Sources are tried in order until a download succeeds. Mirror-backed entries unlock automatically when you configure their mirrors.", ), OrderableListField( key="FAST_SOURCES_DISPLAY", @@ -1548,37 +1539,37 @@ def cloudflare_bypass_settings() -> list[SettingsField]: def _on_save_mirrors(values: dict[str, Any]) -> dict[str, Any]: """Normalize mirror list settings before persisting.""" - from shelfmark.core.logger import setup_logger - from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS from shelfmark.core.utils import normalize_http_url - logger = setup_logger(__name__) + mirror_list_keys = { + "AA_MIRROR_URLS", + "LIBGEN_MIRROR_URLS", + "ZLIB_MIRROR_URLS", + "WELIB_MIRROR_URLS", + } - raw_urls = values.get("AA_MIRROR_URLS") - if raw_urls is None: - return {"error": False, "values": values} + for key in mirror_list_keys: + raw_urls = values.get(key) + if raw_urls is None: + continue - if isinstance(raw_urls, str): - parts = [p.strip() for p in raw_urls.split(",") if p.strip()] - elif isinstance(raw_urls, list): - parts = [str(p).strip() for p in raw_urls if str(p).strip()] - else: - parts = [] + if isinstance(raw_urls, str): + parts = [p.strip() for p in raw_urls.split(",") if p.strip()] + elif isinstance(raw_urls, list): + parts = [str(p).strip() for p in raw_urls if str(p).strip()] + else: + parts = [] - normalized: list[str] = [] - for url in parts: - if url.lower() == "auto": - continue - norm = normalize_http_url(url, default_scheme="https") - if norm and norm not in normalized: - normalized.append(norm) + normalized: list[str] = [] + for url in parts: + if url.lower() == "auto": + continue + norm = normalize_http_url(url, default_scheme="https") + if norm and norm not in normalized: + normalized.append(norm) - if not normalized: - logger.warning("AA_MIRROR_URLS saved empty/invalid; falling back to defaults") - normalized = [normalize_http_url(url, default_scheme="https") for url in DEFAULT_AA_MIRRORS] - normalized = [url for url in normalized if url] + values[key] = normalized - values["AA_MIRROR_URLS"] = normalized return {"error": False, "values": values} @@ -1589,85 +1580,56 @@ def _on_save_mirrors(values: dict[str, Any]) -> dict[str, Any]: @register_settings("mirrors", "Mirrors", icon="globe", order=23, group="direct_download") def mirror_settings() -> list[SettingsField]: """Configure download source mirrors.""" - from shelfmark.core.mirrors import ( - DEFAULT_AA_MIRRORS, - DEFAULT_WELIB_MIRRORS, - DEFAULT_ZLIB_MIRRORS, - ) - return [ # === PRIMARY SOURCE === HeadingField( key="aa_mirrors_heading", title="Anna's Archive", - description="Choose a primary mirror, or use Auto to try mirrors from your list below. The mirror list controls which options appear in the dropdown and the order used in Auto mode.", + description=( + "Add your own Anna's Archive mirror URLs here. Auto mode will try them in the " + "order listed below." + ), ), SelectField( key="AA_BASE_URL", label="Primary Mirror", - description="Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback).", + description=( + "Select Auto to try mirrors from your list on startup and fail over on errors. " + "Choosing a specific mirror pins Shelfmark to that URL." + ), options=_get_aa_base_url_options, default="auto", ), TagListField( key="AA_MIRROR_URLS", label="Mirrors", - description="Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation", - placeholder="https://annas-archive.gl", - default=DEFAULT_AA_MIRRORS, - ), - TextField( - key="AA_ADDITIONAL_URLS", - label="Additional Mirrors (Legacy)", - description="Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables.", - show_when={"field": "AA_ADDITIONAL_URLS", "notEmpty": True}, + description=( + "List the Anna's Archive mirror URLs you want Shelfmark to use. Type a URL and " + "press Enter to add it. Order matters when Auto is selected." + ), + placeholder="https://your-aa-mirror.example", + default=[], ), # === LIBGEN === - HeadingField( - key="libgen_mirrors_heading", - title="LibGen", - description="All mirrors are tried during download until one succeeds. Defaults: libgen.gl, libgen.li, libgen.bz, libgen.la, libgen.vg", - ), - TextField( - key="LIBGEN_ADDITIONAL_URLS", - label="Additional Mirrors", - description="Comma-separated list of custom LibGen mirrors to add to the defaults.", + TagListField( + key="LIBGEN_MIRROR_URLS", + label="LibGen", + description="Mirrors are tried in the order you add them until one works.", + placeholder="https://your-libgen-mirror.example", ), # === Z-LIBRARY === - HeadingField( - key="zlib_mirrors_heading", - title="Z-Library", - description="Z-Library requires Cloudflare bypass. Only the primary mirror is used.", - ), - SelectField( - key="ZLIB_PRIMARY_URL", - label="Primary Mirror", - description="Z-Library mirror to use for downloads.", - options=_get_zlib_mirror_options, - default=DEFAULT_ZLIB_MIRRORS[0], - ), - TextField( - key="ZLIB_ADDITIONAL_URLS", - label="Additional Mirrors", - description="Comma-separated list of custom Z-Library mirror URLs.", + TagListField( + key="ZLIB_MIRROR_URLS", + label="Z-Library", + description="Only the first mirror in the list is used.", + placeholder="https://your-zlibrary-mirror.example", ), # === WELIB === - HeadingField( - key="welib_mirrors_heading", - title="Welib", - description="Welib requires Cloudflare bypass. Only the primary mirror is used.", - ), - SelectField( - key="WELIB_PRIMARY_URL", - label="Primary Mirror", - description="Welib mirror to use for downloads.", - options=_get_welib_mirror_options, - default=DEFAULT_WELIB_MIRRORS[0], - ), - TextField( - key="WELIB_ADDITIONAL_URLS", - label="Additional Mirrors", - description="Comma-separated list of custom Welib mirror URLs.", + TagListField( + key="WELIB_MIRROR_URLS", + label="Welib", + description="Only the first mirror in the list is used.", + placeholder="https://your-welib-mirror.example", ), ] diff --git a/shelfmark/core/mirrors.py b/shelfmark/core/mirrors.py index 55158d01..16ce5788 100644 --- a/shelfmark/core/mirrors.py +++ b/shelfmark/core/mirrors.py @@ -1,4 +1,4 @@ -"""Centralized mirror configuration for all download sources.""" +"""Centralized mirror configuration for direct-download sources.""" from __future__ import annotations @@ -22,33 +22,21 @@ def _get_config() -> Config: return _config_module -# Default mirror lists (hardcoded fallbacks) -DEFAULT_AA_MIRRORS = [ - "https://annas-archive.gl", - "https://annas-archive.pk", - "https://annas-archive.vg", - "https://annas-archive.gd", -] +# Mirror URLs are intentionally user-supplied only. +DEFAULT_AA_MIRRORS: list[str] = [] +DEFAULT_LIBGEN_MIRRORS: list[str] = [] +DEFAULT_ZLIB_MIRRORS: list[str] = [] +DEFAULT_WELIB_MIRRORS: list[str] = [] -DEFAULT_LIBGEN_MIRRORS = [ - "https://libgen.gl", - "https://libgen.li", - "https://libgen.bz", - "https://libgen.la", - "https://libgen.vg", -] - -DEFAULT_ZLIB_MIRRORS = [ - "https://z-lib.fm", - "https://z-lib.gs", - "https://z-lib.id", - "https://z-library.sk", - "https://zlibrary-global.se", -] - -DEFAULT_WELIB_MIRRORS = [ - "https://welib.org", -] +_DOWNLOAD_SOURCE_MIRROR_LABELS = { + "aa-fast": "Anna's Archive", + "aa-slow": "Anna's Archive", + "aa-slow-nowait": "Anna's Archive", + "aa-slow-wait": "Anna's Archive", + "libgen": "LibGen", + "zlib": "Z-Library", + "welib": "Welib", +} def _normalize_mirror_url(url: str) -> str: @@ -60,14 +48,51 @@ def _string_config_value(value: object) -> str: return value if isinstance(value, str) else str(value or "") +def _normalize_configured_urls(value: object) -> list[str]: + """Normalize list or comma-separated mirror config into unique URLs.""" + if isinstance(value, list): + parts = value + elif isinstance(value, str) and value.strip(): + parts = value.split(",") + else: + return [] + + normalized_urls: list[str] = [] + for raw_url in parts: + normalized = _normalize_mirror_url(str(raw_url)) + if normalized and normalized not in normalized_urls: + normalized_urls.append(normalized) + return normalized_urls + + +def _get_primary_mirror_url(key: str) -> str | None: + """Return a configured primary mirror URL, if present.""" + config = _get_config() + primary = _normalize_mirror_url(_string_config_value(config.get(key, ""))) + return primary or None + + +def _build_primary_and_additional_mirrors(primary_key: str, additional_key: str) -> list[str]: + """Build an ordered mirror list from primary + additional config values.""" + config = _get_config() + mirrors: list[str] = [] + + primary = _get_primary_mirror_url(primary_key) + if primary: + mirrors.append(primary) + + for url in _normalize_configured_urls(config.get(additional_key, "")): + if url not in mirrors: + mirrors.append(url) + + return mirrors + + def get_aa_mirrors() -> list[str]: """Get Anna's Archive mirrors. Returns: - Ordered list of AA mirror URLs. - - If AA_MIRROR_URLS is configured, it is treated as the full list. - Otherwise, defaults are used and AA_ADDITIONAL_URLS (legacy) is appended. + Ordered list of user-configured AA mirror URLs. Notes: - The list is used to populate the AA mirror dropdown in Settings. @@ -75,172 +100,149 @@ def get_aa_mirrors() -> list[str]: """ config = _get_config() + configured_list = _normalize_configured_urls(config.get("AA_MIRROR_URLS", None)) + if configured_list: + return configured_list + return _normalize_configured_urls(config.get("AA_ADDITIONAL_URLS", "")) - mirrors: list[str] = [] - configured_list = config.get("AA_MIRROR_URLS", None) - if isinstance(configured_list, list): - for url in configured_list: - normalized = _normalize_mirror_url(str(url)) - if normalized and normalized not in mirrors: - mirrors.append(normalized) - elif isinstance(configured_list, str) and configured_list.strip(): - # Allow comma-separated env/manual configs. - for url in configured_list.split(","): - normalized = _normalize_mirror_url(url) - if normalized and normalized not in mirrors: - mirrors.append(normalized) - - if not mirrors: - mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS] - mirrors = [url for url in mirrors if url] - - # Backwards-compatible append-only behavior for legacy configs/env. - additional = _string_config_value(config.get("AA_ADDITIONAL_URLS", "")) - if additional: - for url in additional.split(","): - normalized = _normalize_mirror_url(url) - if normalized and normalized not in mirrors: - mirrors.append(normalized) +def has_aa_mirror_configuration() -> bool: + """Return True when direct-download search has at least one AA base URL to use.""" + if get_aa_mirrors(): + return True - return mirrors + configured_base_url = normalize_http_url( + _string_config_value(_get_config().get("AA_BASE_URL", "auto")), + default_scheme="https", + allow_special=("auto",), + ) + return bool(configured_base_url and configured_base_url != "auto") def get_libgen_mirrors() -> list[str]: - """Get LibGen mirrors: defaults + any additional from config. + """Get user-configured LibGen mirrors. Returns: - List of LibGen mirror URLs (defaults first, then custom additions). + List of LibGen mirror URLs. """ - mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS] - mirrors = [url for url in mirrors if url] config = _get_config() + configured_list = _normalize_configured_urls(config.get("LIBGEN_MIRROR_URLS", None)) + if configured_list: + return configured_list + return _normalize_configured_urls(config.get("LIBGEN_ADDITIONAL_URLS", "")) - additional = _string_config_value(config.get("LIBGEN_ADDITIONAL_URLS", "")) - if additional: - for url in additional.split(","): - normalized = _normalize_mirror_url(url) - if normalized and normalized not in mirrors: - mirrors.append(normalized) - return mirrors +def has_libgen_mirror_configuration() -> bool: + """Return True when at least one LibGen mirror URL is configured.""" + return bool(get_libgen_mirrors()) def get_zlib_mirrors() -> list[str]: - """Get Z-Library mirrors, with primary first. + """Get user-configured Z-Library mirrors, with primary first. Returns: List of Z-Library mirror URLs, primary first. """ config = _get_config() + configured_list = _normalize_configured_urls(config.get("ZLIB_MIRROR_URLS", None)) + if configured_list: + return configured_list + return _build_primary_and_additional_mirrors("ZLIB_PRIMARY_URL", "ZLIB_ADDITIONAL_URLS") - primary = _normalize_mirror_url( - _string_config_value(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0])) - ) - if not primary: - primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0]) - mirrors = [primary] - - # Add other defaults (excluding primary) - for url in DEFAULT_ZLIB_MIRRORS: - normalized = _normalize_mirror_url(url) - if normalized and normalized != primary: - mirrors.append(normalized) - - # Add custom mirrors - additional = _string_config_value(config.get("ZLIB_ADDITIONAL_URLS", "")) - if additional: - for url in additional.split(","): - normalized = _normalize_mirror_url(url) - if normalized and normalized not in mirrors: - mirrors.append(normalized) - return mirrors +def has_zlib_mirror_configuration() -> bool: + """Return True when at least one Z-Library mirror URL is configured.""" + return bool(get_zlib_mirrors()) -def get_zlib_primary_url() -> str: +def get_zlib_primary_url() -> str | None: """Get the primary Z-Library mirror URL. Returns: - Primary Z-Library mirror URL. + Primary Z-Library mirror URL, if configured. """ - config = _get_config() - primary = _normalize_mirror_url( - _string_config_value(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0])) - ) - return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0]) + mirrors = get_zlib_mirrors() + return mirrors[0] if mirrors else None -def get_zlib_url_template() -> str: +def get_zlib_url_template() -> str | None: """Get Z-Library URL template using configured primary mirror. Returns: - URL template with {md5} placeholder. + URL template with {md5} placeholder, if configured. """ primary = get_zlib_primary_url() - return f"{primary}/md5/{{md5}}" + return f"{primary}/md5/{{md5}}" if primary else None def get_welib_mirrors() -> list[str]: - """Get Welib mirrors, with primary first. + """Get user-configured Welib mirrors, with primary first. Returns: List of Welib mirror URLs, primary first. """ config = _get_config() + configured_list = _normalize_configured_urls(config.get("WELIB_MIRROR_URLS", None)) + if configured_list: + return configured_list + return _build_primary_and_additional_mirrors("WELIB_PRIMARY_URL", "WELIB_ADDITIONAL_URLS") - primary = _normalize_mirror_url( - _string_config_value(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0])) - ) - if not primary: - primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0]) - mirrors = [primary] - - # Add other defaults (excluding primary) - for url in DEFAULT_WELIB_MIRRORS: - normalized = _normalize_mirror_url(url) - if normalized and normalized != primary: - mirrors.append(normalized) - - # Add custom mirrors - additional = _string_config_value(config.get("WELIB_ADDITIONAL_URLS", "")) - if additional: - for url in additional.split(","): - normalized = _normalize_mirror_url(url) - if normalized and normalized not in mirrors: - mirrors.append(normalized) - return mirrors +def has_welib_mirror_configuration() -> bool: + """Return True when at least one Welib mirror URL is configured.""" + return bool(get_welib_mirrors()) + +def has_download_source_mirror_configuration(source_id: str) -> bool: + """Return True when the requested direct-download source has mirror config.""" + if source_id in {"aa-fast", "aa-slow", "aa-slow-nowait", "aa-slow-wait"}: + return has_aa_mirror_configuration() + if source_id == "libgen": + return has_libgen_mirror_configuration() + if source_id == "zlib": + return has_zlib_mirror_configuration() + if source_id == "welib": + return has_welib_mirror_configuration() + return False -def get_welib_primary_url() -> str: + +def get_download_source_missing_mirror_reason(source_id: str) -> str | None: + """Return a user-facing reason when a direct-download source has no mirror config.""" + if has_download_source_mirror_configuration(source_id): + return None + + label = _DOWNLOAD_SOURCE_MIRROR_LABELS.get(source_id) + if not label: + return None + + return f"Add at least one {label} mirror in Mirrors" + + +def get_welib_primary_url() -> str | None: """Get the primary Welib mirror URL. Returns: - Primary Welib mirror URL. + Primary Welib mirror URL, if configured. """ - config = _get_config() - primary = _normalize_mirror_url( - _string_config_value(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0])) - ) - return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0]) + mirrors = get_welib_mirrors() + return mirrors[0] if mirrors else None -def get_welib_url_template() -> str: +def get_welib_url_template() -> str | None: """Get Welib URL template using configured primary mirror. Returns: - URL template with {md5} placeholder. + URL template with {md5} placeholder, if configured. """ primary = get_welib_primary_url() - return f"{primary}/md5/{{md5}}" + return f"{primary}/md5/{{md5}}" if primary else None def get_zlib_cookie_domains() -> set: @@ -254,21 +256,8 @@ def get_zlib_cookie_domains() -> set: """ domains = set() - # Add all default domains - for url in DEFAULT_ZLIB_MIRRORS: - normalized = _normalize_mirror_url(url) - if normalized: - domain = normalized.replace("https://", "").replace("http://", "").split("/")[0] - domains.add(domain) - - # Add custom domains - config = _get_config() - additional = _string_config_value(config.get("ZLIB_ADDITIONAL_URLS", "")) - if additional: - for url in additional.split(","): - normalized = _normalize_mirror_url(url) - if normalized: - domain = normalized.replace("https://", "").replace("http://", "").split("/")[0] - domains.add(domain) + for url in get_zlib_mirrors(): + domain = url.replace("https://", "").replace("http://", "").split("/")[0] + domains.add(domain) return domains diff --git a/shelfmark/core/onboarding.py b/shelfmark/core/onboarding.py index 3b7f5c30..989f1c7e 100644 --- a/shelfmark/core/onboarding.py +++ b/shelfmark/core/onboarding.py @@ -12,8 +12,10 @@ from shelfmark.core.logger import setup_logger from shelfmark.core.settings_registry import ( HeadingField, + MultiSelectField, SettingsField, get_setting_value, + get_settings_field_map, get_settings_tab, save_config_file, serialize_field, @@ -23,6 +25,8 @@ ONBOARDING_STORAGE_KEY = "onboarding_complete" +ONBOARDING_RELEASE_SOURCES_KEY = "ONBOARDING_RELEASE_SOURCES" +_ONBOARDING_VIRTUAL_KEYS = {ONBOARDING_RELEASE_SOURCES_KEY} def _get_config_dir() -> Path: @@ -86,6 +90,20 @@ def _get_field_from_tab(tab_name: str, field_key: str) -> SettingsField | None: return None +def _get_field_tab_name(field: SettingsField, fallback_tab_name: str) -> str: + """Return the owning settings tab for a value field.""" + field_key = getattr(field, "key", None) + if not field_key: + return fallback_tab_name + + field_map = get_settings_field_map() + field_entry = field_map.get(field_key) + if field_entry is None: + return fallback_tab_name + + return field_entry[1] + + def _clone_field_with_overrides(field: SettingsField, **overrides: object) -> SettingsField: """Clone a field with optional attribute overrides. @@ -94,6 +112,100 @@ def _clone_field_with_overrides(field: SettingsField, **overrides: object) -> Se return replace(field, **overrides) +def _get_fields_from_tab( + tab_name: str, + field_keys: list[str], + *, + strip_show_when_keys: set[str] | None = None, +) -> list[SettingsField]: + """Return the requested fields from a settings tab in the supplied order.""" + fields: list[SettingsField] = [] + for field_key in field_keys: + field = _get_field_from_tab(tab_name, field_key) + if field: + show_when = getattr(field, "show_when", None) + stripped_show_when = _strip_show_when_keys(show_when, strip_show_when_keys or set()) + if stripped_show_when != show_when: + field = replace(field, show_when=stripped_show_when) + fields.append(field) + return fields + + +def _strip_show_when_keys( + show_when: dict[str, Any] | list[dict[str, Any]] | None, + field_keys: set[str], +) -> dict[str, Any] | list[dict[str, Any]] | None: + """Remove conditions tied to fields that onboarding handles implicitly.""" + if not show_when or not field_keys: + return show_when + + if isinstance(show_when, list): + remaining = [ + condition for condition in show_when if condition.get("field") not in field_keys + ] + return remaining or None + + if show_when.get("field") in field_keys: + return None + + return show_when + + +def _is_release_source_selected(values: dict[str, Any], source_name: str) -> bool: + """Return True when a release source has been chosen during onboarding.""" + raw_sources = values.get(ONBOARDING_RELEASE_SOURCES_KEY, []) + if not isinstance(raw_sources, list): + return False + return source_name in raw_sources + + +def _evaluate_show_when_condition(condition: dict[str, Any], values: dict[str, Any]) -> bool: + """Evaluate one onboarding show_when condition against submitted values.""" + current_value = values.get(condition["field"]) + expected_value = condition.get("value") + + if condition.get("notEmpty"): + if isinstance(current_value, list): + return len(current_value) > 0 + return current_value not in (None, "") + + if isinstance(current_value, list): + if isinstance(expected_value, list): + return all(item in current_value for item in expected_value) + return expected_value in current_value + + if isinstance(expected_value, list): + return current_value in expected_value + + return current_value == expected_value + + +def _is_step_visible(step_config: dict[str, Any], values: dict[str, Any]) -> bool: + """Return True when a step should be included for the provided values.""" + show_when = step_config.get("show_when") + if not show_when: + return True + return all(_evaluate_show_when_condition(condition, values) for condition in show_when) + + +def _is_field_visible(field: SettingsField, values: dict[str, Any]) -> bool: + """Return True when a field should be included in the onboarding save.""" + if getattr(field, "hidden_in_ui", False): + return False + + if getattr(field, "universal_only", False) and values.get("SEARCH_MODE") != "universal": + return False + + show_when = getattr(field, "show_when", None) + if not show_when: + return True + + if isinstance(show_when, list): + return all(_evaluate_show_when_condition(condition, values) for condition in show_when) + + return _evaluate_show_when_condition(show_when, values) + + # ============================================================================= # Step Definitions # ============================================================================= @@ -217,106 +329,243 @@ def get_googlebooks_setup_fields() -> list[SettingsField]: return fields -def get_prowlarr_fields() -> list[SettingsField]: - """Step 4: Configure Prowlarr connection - uses actual Prowlarr fields.""" +def get_release_source_selection_fields() -> list[SettingsField]: + """Choose which release sources to configure during onboarding.""" fields: list[SettingsField] = [ HeadingField( - key="prowlarr_heading", - title="Prowlarr Integration (Optional)", - description="Connect to Prowlarr to search your indexers for torrents and NZBs. Skip this step if you only want to use Direct Download.", + key="release_sources_heading", + title="Release Sources", + description=( + "Choose the release sources you want to configure now. You can always add or " + "change sources later in Settings." + ), + ), + MultiSelectField( + key=ONBOARDING_RELEASE_SOURCES_KEY, + label="Sources to Set Up", + description="Select one or more release sources to configure now.", + default=[], + variant="dropdown", + env_supported=False, + options=[ + { + "value": "direct_download", + "label": "Direct Download", + "description": "Configure your own Anna's Archive mirror URLs for direct ebook downloads.", + }, + { + "value": "prowlarr", + "label": "Prowlarr", + "description": "Search your torrent and Usenet indexers through Prowlarr.", + }, + { + "value": "audiobookbay", + "label": "AudiobookBay", + "description": "Search AudiobookBay directly for audiobook releases.", + }, + { + "value": "irc", + "label": "IRC", + "description": "Connect to IRC for ebook and audiobook release searches.", + }, + ], ), ] - - # Get actual Prowlarr connection fields - prowlarr_fields = ["PROWLARR_ENABLED", "PROWLARR_URL", "PROWLARR_API_KEY", "test_prowlarr"] - for field_key in prowlarr_fields: - field = _get_field_from_tab("prowlarr_config", field_key) - if field: - fields.append(field) - return fields -def get_prowlarr_indexers_fields() -> list[SettingsField]: - """Step 5: Select Prowlarr indexers to search.""" +def get_direct_download_setup_fields() -> list[SettingsField]: + """Render trimmed direct-download essentials for onboarding.""" fields: list[SettingsField] = [ HeadingField( - key="prowlarr_indexers_heading", - title="Select Indexers", - description="Choose which indexers to search for books. Leave empty to search all available indexers.", - ), + key="direct_download_setup_onboarding_heading", + title="Direct Download Setup", + description=( + "Add at least one Anna's Archive mirror URL to enable Direct Download. If you " + "have an Anna's Archive donator key, you can add it here too. You can configure " + "alternative mirrors later in Settings." + ), + ) ] - - # Get the indexers multi-select field - indexers_field = _get_field_from_tab("prowlarr_config", "PROWLARR_INDEXERS") - if indexers_field: - fields.append(indexers_field) - + fields.extend(_get_fields_from_tab("download_sources", ["AA_DONATOR_KEY"])) + fields.extend(_get_fields_from_tab("mirrors", ["AA_MIRROR_URLS"])) return fields -# ============================================================================= -# Step Configuration -# ============================================================================= +def get_direct_download_bypass_fields() -> list[SettingsField]: + """Render only the core Cloudflare bypass fields for onboarding.""" + return _get_fields_from_tab( + "cloudflare_bypass", + [ + "USE_CF_BYPASS", + "USING_EXTERNAL_BYPASSER", + "EXT_BYPASSER_URL", + "EXT_BYPASSER_PATH", + ], + ) -ONBOARDING_STEPS = [ - { - "id": "search_mode", - "title": "Search Mode", - "tab": "search_mode", - "get_fields": get_search_mode_fields, - }, - { - "id": "metadata_provider", - "title": "Metadata Provider", - "tab": "search_mode", - "get_fields": get_metadata_provider_fields, - "show_when": [{"field": "SEARCH_MODE", "value": "universal"}], - }, - { - "id": "hardcover_setup", - "title": "Hardcover Setup", - "tab": "hardcover", - "get_fields": get_hardcover_setup_fields, - # Must be universal mode AND hardcover selected - "show_when": [ - {"field": "SEARCH_MODE", "value": "universal"}, - {"field": "METADATA_PROVIDER", "value": "hardcover"}, - ], - }, - { - "id": "googlebooks_setup", - "title": "Google Books Setup", - "tab": "googlebooks", - "get_fields": get_googlebooks_setup_fields, - # Must be universal mode AND googlebooks selected - "show_when": [ - {"field": "SEARCH_MODE", "value": "universal"}, - {"field": "METADATA_PROVIDER", "value": "googlebooks"}, +def get_prowlarr_fields() -> list[SettingsField]: + """Render trimmed Prowlarr setup fields for onboarding.""" + return _get_fields_from_tab( + "prowlarr_config", + [ + "prowlarr_heading", + "PROWLARR_URL", + "PROWLARR_API_KEY", + "test_prowlarr", + "PROWLARR_INDEXERS", ], - }, - { - "id": "prowlarr", - "title": "Prowlarr", - "tab": "prowlarr_config", - "get_fields": get_prowlarr_fields, - "show_when": [{"field": "SEARCH_MODE", "value": "universal"}], - "optional": True, - }, - { - "id": "prowlarr_indexers", - "title": "Indexers", - "tab": "prowlarr_config", - "get_fields": get_prowlarr_indexers_fields, - # Only show when Prowlarr is enabled - "show_when": [ - {"field": "SEARCH_MODE", "value": "universal"}, - {"field": "PROWLARR_ENABLED", "value": True}, + strip_show_when_keys={"PROWLARR_ENABLED"}, + ) + + +def get_audiobookbay_fields() -> list[SettingsField]: + """Render trimmed AudiobookBay setup fields for onboarding.""" + return [ + HeadingField( + key="audiobookbay_onboarding_heading", + title="AudiobookBay", + description="Add the AudiobookBay domain you want Shelfmark to search.", + ), + *_get_fields_from_tab( + "audiobookbay_config", + ["ABB_HOSTNAME"], + strip_show_when_keys={"ABB_ENABLED"}, + ), + ] + + +def get_irc_fields() -> list[SettingsField]: + """Render trimmed IRC setup fields for onboarding.""" + return _get_fields_from_tab( + "irc", + [ + "heading", + "IRC_SERVER", + "IRC_PORT", + "IRC_USE_TLS", + "IRC_CHANNEL", + "IRC_NICK", + "IRC_SEARCH_BOT", ], - "optional": True, - }, -] + ) + + +def get_onboarding_steps() -> list[dict[str, Any]]: + """Return the full onboarding step configuration.""" + return [ + { + "id": "search_mode", + "title": "Search Mode", + "tab": "search_mode", + "get_fields": get_search_mode_fields, + }, + { + "id": "metadata_provider", + "title": "Metadata Provider", + "tab": "search_mode", + "get_fields": get_metadata_provider_fields, + "show_when": [{"field": "SEARCH_MODE", "value": "universal"}], + }, + { + "id": "hardcover_setup", + "title": "Hardcover Setup", + "tab": "hardcover", + "get_fields": get_hardcover_setup_fields, + "show_when": [ + {"field": "SEARCH_MODE", "value": "universal"}, + {"field": "METADATA_PROVIDER", "value": "hardcover"}, + ], + }, + { + "id": "googlebooks_setup", + "title": "Google Books Setup", + "tab": "googlebooks", + "get_fields": get_googlebooks_setup_fields, + "show_when": [ + {"field": "SEARCH_MODE", "value": "universal"}, + {"field": "METADATA_PROVIDER", "value": "googlebooks"}, + ], + }, + { + "id": "release_sources", + "title": "Release Sources", + "tab": "search_mode", + "get_fields": get_release_source_selection_fields, + "show_when": [{"field": "SEARCH_MODE", "value": "universal"}], + "optional": True, + }, + { + "id": "direct_download_setup_direct_mode", + "title": "Direct Download Setup", + "tab": "download_sources", + "get_fields": get_direct_download_setup_fields, + "show_when": [{"field": "SEARCH_MODE", "value": "direct"}], + }, + { + "id": "direct_download_cloudflare_bypass_direct_mode", + "title": "Cloudflare Bypass", + "tab": "cloudflare_bypass", + "get_fields": get_direct_download_bypass_fields, + "show_when": [{"field": "SEARCH_MODE", "value": "direct"}], + }, + { + "id": "direct_download_setup", + "title": "Direct Download Setup", + "tab": "download_sources", + "get_fields": get_direct_download_setup_fields, + "show_when": [ + {"field": "SEARCH_MODE", "value": "universal"}, + {"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "direct_download"}, + ], + "optional": True, + }, + { + "id": "direct_download_cloudflare_bypass", + "title": "Cloudflare Bypass", + "tab": "cloudflare_bypass", + "get_fields": get_direct_download_bypass_fields, + "show_when": [ + {"field": "SEARCH_MODE", "value": "universal"}, + {"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "direct_download"}, + ], + "optional": True, + }, + { + "id": "prowlarr", + "title": "Prowlarr", + "tab": "prowlarr_config", + "get_fields": get_prowlarr_fields, + "show_when": [ + {"field": "SEARCH_MODE", "value": "universal"}, + {"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "prowlarr"}, + ], + "optional": True, + }, + { + "id": "audiobookbay", + "title": "AudiobookBay", + "tab": "audiobookbay_config", + "get_fields": get_audiobookbay_fields, + "show_when": [ + {"field": "SEARCH_MODE", "value": "universal"}, + {"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "audiobookbay"}, + ], + "optional": True, + }, + { + "id": "irc", + "title": "IRC", + "tab": "irc", + "get_fields": get_irc_fields, + "show_when": [ + {"field": "SEARCH_MODE", "value": "universal"}, + {"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "irc"}, + ], + "optional": True, + }, + ] def get_onboarding_config() -> dict[str, Any]: @@ -324,19 +573,23 @@ def get_onboarding_config() -> dict[str, Any]: steps = [] all_values = {} - for step_config in ONBOARDING_STEPS: + for step_config in get_onboarding_steps(): fields = step_config["get_fields"]() tab_name = step_config["tab"] # Serialize fields with current values serialized_fields = [] for field in fields: - serialized = serialize_field(field, tab_name, include_value=True) + field_tab_name = _get_field_tab_name(field, tab_name) + serialized = serialize_field(field, field_tab_name, include_value=True) serialized_fields.append(serialized) # Collect values (skip HeadingFields) - if hasattr(field, "key") and field.key and not isinstance(field, HeadingField): - value = get_setting_value(field, tab_name) + if hasattr(field, "env_supported") and getattr(field, "key", None): + if field.key in _ONBOARDING_VIRTUAL_KEYS: + value = getattr(field, "default", "") + else: + value = get_setting_value(field, field_tab_name) all_values[field.key] = ( value if value is not None else getattr(field, "default", "") ) @@ -376,8 +629,10 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]: # Group values by their target tab tab_values: dict[str, dict[str, Any]] = {} - for step_config in ONBOARDING_STEPS: - tab_name = step_config["tab"] + for step_config in get_onboarding_steps(): + if not _is_step_visible(step_config, values): + continue + fields = step_config["get_fields"]() for field in fields: @@ -385,7 +640,12 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]: continue key = field.key + if key in _ONBOARDING_VIRTUAL_KEYS: + continue + if not _is_field_visible(field, values): + continue if key in values: + tab_name = _get_field_tab_name(field, step_config["tab"]) if tab_name not in tab_values: tab_values[tab_name] = {} tab_values[tab_name][key] = values[key] @@ -396,8 +656,7 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]: save_config_file(tab_name, tab_data) logger.info("Saved onboarding settings to %s: %s", tab_name, list(tab_data.keys())) - # Enable the selected metadata provider - search_mode = values.get("SEARCH_MODE", "direct") + search_mode = values.get("SEARCH_MODE", "universal") if search_mode == "universal": provider = values.get("METADATA_PROVIDER", "hardcover") if provider: @@ -425,6 +684,47 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]: list(provider_config.keys()), ) + selected_release_sources = values.get(ONBOARDING_RELEASE_SOURCES_KEY, []) + if not isinstance(selected_release_sources, list): + selected_release_sources = [] + + source_updates: dict[str, dict[str, Any]] = {} + + if search_mode == "direct": + source_updates.setdefault("download_sources", {})["DIRECT_DOWNLOAD_ENABLED"] = True + else: + if _is_release_source_selected(values, "direct_download"): + source_updates.setdefault("download_sources", {})["DIRECT_DOWNLOAD_ENABLED"] = True + if _is_release_source_selected(values, "prowlarr"): + source_updates.setdefault("prowlarr_config", {})["PROWLARR_ENABLED"] = True + if _is_release_source_selected(values, "audiobookbay"): + source_updates.setdefault("audiobookbay_config", {})["ABB_ENABLED"] = True + + if not values.get("DEFAULT_RELEASE_SOURCE"): + for source_name in selected_release_sources: + if source_name in {"direct_download", "prowlarr", "irc"}: + source_updates.setdefault("search_mode", {})["DEFAULT_RELEASE_SOURCE"] = ( + source_name + ) + break + + if not values.get("DEFAULT_RELEASE_SOURCE_AUDIOBOOK"): + for source_name in selected_release_sources: + if source_name in {"prowlarr", "audiobookbay", "irc"}: + source_updates.setdefault("search_mode", {})[ + "DEFAULT_RELEASE_SOURCE_AUDIOBOOK" + ] = source_name + break + + for tab_name, tab_data in source_updates.items(): + if tab_data: + save_config_file(tab_name, tab_data) + logger.info( + "Enabled onboarding release source settings for %s: %s", + tab_name, + list(tab_data.keys()), + ) + # Mark onboarding as complete mark_onboarding_complete() diff --git a/shelfmark/core/settings_registry.py b/shelfmark/core/settings_registry.py index b1011c32..c8b6d3a7 100644 --- a/shelfmark/core/settings_registry.py +++ b/shelfmark/core/settings_registry.py @@ -521,6 +521,15 @@ def initialize_default_configs() -> bool: def sync_env_to_config() -> None: """Sync supported environment-backed settings into config files.""" + config_dir = _get_config_dir() + plugins_dir = config_dir / "plugins" + had_existing_search_page_title = "SEARCH_PAGE_TITLE" in load_config_file("general") + had_existing_install_state = ( + (config_dir / "settings.json").exists() + or (config_dir / "users.db").exists() + or (plugins_dir.exists() and any(plugins_dir.glob("*.json"))) + ) + # Initialize default configs first (for fresh installs) initialize_default_configs() @@ -532,13 +541,8 @@ def sync_env_to_config() -> None: if not getattr(settings_field, "env_supported", True): continue - # Check if ENV var is set - env_var_name = settings_field.get_env_var_name() - env_value = os.environ.get(env_var_name) - - if env_value is not None: - # Parse the ENV value to the appropriate type - parsed_value = _parse_env_value(env_value, settings_field) + has_env_value, parsed_value = _get_env_value_for_field(settings_field) + if has_env_value: values_to_sync[settings_field.key] = parsed_value # Save synced values to config file (merge with existing) @@ -554,21 +558,39 @@ def sync_env_to_config() -> None: migrate_legacy_settings() migrate_download_to_browser_settings() migrate_mirror_settings() + migrate_direct_download_upgrade(existing_install=had_existing_install_state) + migrate_search_page_title( + existing_install=had_existing_install_state, + had_existing_value=had_existing_search_page_title, + ) -def migrate_mirror_settings() -> None: - """Sync AA mirror list when code defaults change between versions. +def migrate_direct_download_upgrade(*, existing_install: bool) -> None: + """Preserve the direct-download enabled flag for existing installs only.""" + if not existing_install: + return - On startup, compares a hash of DEFAULT_AA_MIRRORS against the hash stored - in the config file. If they differ (i.e., an update shipped new defaults), - the config is overwritten with the new defaults. If they match, the user's - customizations are left untouched. + download_sources_config = load_config_file("download_sources") + + download_updates: dict[str, Any] = {} + + if "DIRECT_DOWNLOAD_ENABLED" not in download_sources_config: + download_updates["DIRECT_DOWNLOAD_ENABLED"] = True + + if download_updates: + save_config_file("download_sources", download_updates) + + +def migrate_search_page_title(*, existing_install: bool, had_existing_value: bool) -> None: + """Keep the legacy homepage search title for existing installs only.""" + if not existing_install or had_existing_value or os.getenv("SEARCH_PAGE_TITLE") is not None: + return + + save_config_file("general", {"SEARCH_PAGE_TITLE": "Book Search & Download"}) - Also handles legacy migration from AA_ADDITIONAL_URLS. - """ - import hashlib - from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS +def migrate_mirror_settings() -> None: + """Normalize canonical mirror-list settings and migrate legacy config forward safely.""" from shelfmark.core.utils import normalize_http_url def _normalize_list(values: list[str]) -> list[str]: @@ -581,88 +603,75 @@ def _normalize_list(values: list[str]) -> list[str]: out.append(norm) return out - def _hash_mirrors(mirrors: list[str]) -> str: - return hashlib.sha256(",".join(mirrors).encode()).hexdigest() - - normalized_defaults = _normalize_list(DEFAULT_AA_MIRRORS) - current_defaults_hash = _hash_mirrors(normalized_defaults) - mirrors_config = load_config_file("mirrors") - stored_hash = mirrors_config.get("_AA_MIRRORS_DEFAULTS_HASH") - raw_list = mirrors_config.get("AA_MIRROR_URLS") - raw_additional = mirrors_config.get("AA_ADDITIONAL_URLS", "") - - def _save_mirrors(values: dict[str, Any]) -> None: - merged = dict(mirrors_config) - merged.update(values) - save_config_file("mirrors", merged) - mirrors_config.update(values) - - # Defaults changed since last startup — push new mirrors to config - if stored_hash != current_defaults_hash: - _save_mirrors( - { - "AA_MIRROR_URLS": normalized_defaults, - "_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash, - } - ) - return - - # --- Legacy migration (only runs if hash already matches / first time) --- - - # If already a proper list, just ensure it's non-empty. - if isinstance(raw_list, list): - normalized = _normalize_list([str(v) for v in raw_list]) - if normalized: + mirror_updates: dict[str, Any] = {} + + def _queue_update(key: str, normalized: list[str]) -> None: + current = mirrors_config.get(key) + if current != normalized: + mirror_updates[key] = normalized + + def _resolve_canonical_list( + canonical_key: str, + *, + legacy_list_key: str | None = None, + legacy_primary_key: str | None = None, + legacy_additional_key: str | None = None, + ) -> None: + raw_list = mirrors_config.get(canonical_key) + + if isinstance(raw_list, list): + _queue_update(canonical_key, _normalize_list([str(v) for v in raw_list])) return - _save_mirrors( - { - "AA_MIRROR_URLS": normalized_defaults, - "_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash, - } - ) - return - # If saved as a string, convert to list. - if isinstance(raw_list, str) and raw_list.strip(): - parts = [p.strip() for p in raw_list.split(",") if p.strip()] - normalized = _normalize_list(parts) - if normalized: - _save_mirrors( - { - "AA_MIRROR_URLS": normalized, - "_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash, - } - ) + if isinstance(raw_list, str) and raw_list.strip(): + parts = [p.strip() for p in raw_list.split(",") if p.strip()] + _queue_update(canonical_key, _normalize_list(parts)) return - _save_mirrors( - { - "AA_MIRROR_URLS": normalized_defaults, - "_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash, - } - ) - return - # If there's legacy additional mirrors, seed the full list. - if isinstance(raw_additional, str) and raw_additional.strip(): - additional_parts = [p.strip() for p in raw_additional.split(",") if p.strip()] - combined = _normalize_list(DEFAULT_AA_MIRRORS + additional_parts) - if combined: - _save_mirrors( - { - "AA_MIRROR_URLS": combined, - "_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash, - } - ) - return + combined: list[str] = [] + if legacy_primary_key: + raw_primary = mirrors_config.get(legacy_primary_key, "") + if isinstance(raw_primary, str) and raw_primary.strip(): + combined.append(raw_primary.strip()) + if legacy_list_key: + raw_legacy_list = mirrors_config.get(legacy_list_key, "") + if isinstance(raw_legacy_list, str) and raw_legacy_list.strip(): + combined.extend([p.strip() for p in raw_legacy_list.split(",") if p.strip()]) + elif isinstance(raw_legacy_list, list): + combined.extend([str(p).strip() for p in raw_legacy_list if str(p).strip()]) + if legacy_additional_key: + raw_additional = mirrors_config.get(legacy_additional_key, "") + if isinstance(raw_additional, str) and raw_additional.strip(): + combined.extend([p.strip() for p in raw_additional.split(",") if p.strip()]) + elif isinstance(raw_additional, list): + combined.extend([str(p).strip() for p in raw_additional if str(p).strip()]) + + normalized = _normalize_list(combined) + if normalized: + _queue_update(canonical_key, normalized) - # No config at all yet — write defaults - _save_mirrors( - { - "AA_MIRROR_URLS": normalized_defaults, - "_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash, - } + _resolve_canonical_list( + "AA_MIRROR_URLS", + legacy_list_key="AA_ADDITIONAL_URLS", + ) + _resolve_canonical_list( + "LIBGEN_MIRROR_URLS", + legacy_list_key="LIBGEN_ADDITIONAL_URLS", ) + _resolve_canonical_list( + "ZLIB_MIRROR_URLS", + legacy_primary_key="ZLIB_PRIMARY_URL", + legacy_additional_key="ZLIB_ADDITIONAL_URLS", + ) + _resolve_canonical_list( + "WELIB_MIRROR_URLS", + legacy_primary_key="WELIB_PRIMARY_URL", + legacy_additional_key="WELIB_ADDITIONAL_URLS", + ) + + if mirror_updates: + save_config_file("mirrors", mirror_updates) def migrate_legacy_settings() -> None: @@ -813,11 +822,9 @@ def migrate_download_to_browser_settings() -> None: def get_setting_value(field: FieldBase, tab_name: str) -> object: """Resolve the effective value for a settings field.""" # 1. Check environment variable (if supported for this field) - if field.env_supported: - env_var_name = field.get_env_var_name() - env_value = os.environ.get(env_var_name) - if env_value is not None: - return _parse_env_value(env_value, field) + has_env_value, parsed_env_value = _get_env_value_for_field(field) + if has_env_value: + return parsed_env_value # 2. Check config file config = load_config_file(tab_name) @@ -860,12 +867,78 @@ def _parse_env_value(value: str, field: FieldBase) -> object: return value +def _normalize_mirror_env_urls(values: list[str]) -> list[str]: + """Normalize mirror URL env values to stable https URLs without duplicates.""" + from shelfmark.core.utils import normalize_http_url + + normalized: list[str] = [] + for raw_value in values: + stripped = raw_value.strip() + if not stripped: + continue + normalized_url = normalize_http_url(stripped, default_scheme="https") + if normalized_url and normalized_url not in normalized: + normalized.append(normalized_url) + return normalized + + +def _get_env_value_for_field(field: FieldBase) -> tuple[bool, object | None]: + """Return parsed env-backed value for a field, including legacy mirror aliases.""" + if not field.env_supported: + return False, None + + env_var_name = field.get_env_var_name() + env_value = os.environ.get(env_var_name) + if env_value is not None: + parsed = _parse_env_value(env_value, field) + if field.key in { + "AA_MIRROR_URLS", + "LIBGEN_MIRROR_URLS", + "ZLIB_MIRROR_URLS", + "WELIB_MIRROR_URLS", + } and isinstance(parsed, list): + parsed = _normalize_mirror_env_urls(parsed) + return True, parsed + + if field.key == "AA_MIRROR_URLS": + legacy_additional = os.environ.get("AA_ADDITIONAL_URLS") + if legacy_additional is not None: + return True, _normalize_mirror_env_urls(legacy_additional.split(",")) + + if field.key == "LIBGEN_MIRROR_URLS": + legacy_additional = os.environ.get("LIBGEN_ADDITIONAL_URLS") + if legacy_additional is not None: + return True, _normalize_mirror_env_urls(legacy_additional.split(",")) + + if field.key == "ZLIB_MIRROR_URLS": + legacy_primary = os.environ.get("ZLIB_PRIMARY_URL") + legacy_additional = os.environ.get("ZLIB_ADDITIONAL_URLS") + if legacy_primary is not None or legacy_additional is not None: + combined = [] + if legacy_primary is not None: + combined.append(legacy_primary) + if legacy_additional is not None: + combined.extend(legacy_additional.split(",")) + return True, _normalize_mirror_env_urls(combined) + + if field.key == "WELIB_MIRROR_URLS": + legacy_primary = os.environ.get("WELIB_PRIMARY_URL") + legacy_additional = os.environ.get("WELIB_ADDITIONAL_URLS") + if legacy_primary is not None or legacy_additional is not None: + combined = [] + if legacy_primary is not None: + combined.append(legacy_primary) + if legacy_additional is not None: + combined.extend(legacy_additional.split(",")) + return True, _normalize_mirror_env_urls(combined) + + return False, None + + def is_value_from_env(field: FieldBase) -> bool: """Check if a field's value comes from an environment variable.""" - # UI-only settings never come from ENV (env_supported=False) - if not getattr(field, "env_supported", True): - return False - return field.get_env_var_name() in os.environ + has_env_value, _parsed = _get_env_value_for_field(field) + return has_env_value def serialize_field( @@ -1155,7 +1228,7 @@ def _apply_dns_settings(config: Config) -> None: def _apply_aa_mirror_settings(config: Config) -> None: """Apply AA mirror settings changes to the network module. - This ensures AA_BASE_URL / AA_ADDITIONAL_URLS changes take effect immediately + This ensures AA_BASE_URL / AA_MIRROR_URLS changes take effect immediately without requiring a container restart. """ try: diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index aa12c76c..8460e28e 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -204,7 +204,8 @@ def _try_rotation( original_url: str, current_url: str, selector: network.AAMirrorSelector ) -> str | None: """Try mirror/DNS rotation. Returns new URL or None.""" - if current_url.startswith(network.get_aa_base_url()): + aa_base_url = network.get_aa_base_url() + if aa_base_url and current_url.startswith(aa_base_url): new_base, action = selector.next_mirror_or_rotate_dns() if action in ("mirror", "dns") and new_base: new_url = selector.rewrite(original_url) @@ -538,7 +539,7 @@ def download_url( and not zlib_cookie_refresh_attempted ): parsed = urlparse(current_url) - if parsed.hostname and "z-lib" in parsed.hostname and referer: + if _is_configured_zlib_host(parsed.hostname) and referer: zlib_cookie_refresh_attempted = True logger.info("Z-Library 403 - refreshing cookies via referer: %s", referer) try: @@ -602,6 +603,24 @@ def download_url( return None +def _is_configured_zlib_host(hostname: str | None) -> bool: + """Return True when a hostname matches a configured Z-Library mirror.""" + if not hostname: + return False + + from shelfmark.core.mirrors import get_zlib_cookie_domains + + hostname = hostname.lower() + base_domain = ".".join(hostname.split(".")[-2:]) if "." in hostname else hostname + + for domain in get_zlib_cookie_domains(): + candidate = str(domain).lower() + if hostname == candidate or hostname.endswith(f".{candidate}") or base_domain == candidate: + return True + + return False + + def _try_resume( url: str, buffer: BytesIO, diff --git a/shelfmark/download/network.py b/shelfmark/download/network.py index 6aa1bce1..14fe95b0 100644 --- a/shelfmark/download/network.py +++ b/shelfmark/download/network.py @@ -924,9 +924,13 @@ def rotate_dns_and_reset_aa() -> bool: if configured_url == "auto": # Auto mode always resets to the first mirror to restart the cascade _current_aa_url_index = 0 - _aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.gl" - logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url) - _save_state(aa_url=_aa_base_url) + if _aa_urls: + _aa_base_url = _aa_urls[0] + logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url) + _save_state(aa_url=_aa_base_url) + else: + _aa_base_url = "" + logger.info("After DNS switch, AA URL remains unconfigured") else: # Keep the user's configured primary mirror (if it exists in the list), # otherwise keep the configured URL as-is (custom/env). @@ -1105,6 +1109,12 @@ def _initialize_aa_state() -> None: if configured_url != "auto" and configured_url not in _aa_urls: _aa_urls = [configured_url, *_aa_urls] + if not _aa_urls: + _aa_base_url = "" + _current_aa_url_index = 0 + logger.info("AA_BASE_URL: unconfigured") + return + if configured_url == "auto": if state.get("aa_base_url") and state["aa_base_url"] in _aa_urls: _current_aa_url_index = _aa_urls.index(state["aa_base_url"]) diff --git a/shelfmark/main.py b/shelfmark/main.py index 198bc1e8..66837114 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -1113,10 +1113,10 @@ def api_config() -> Response | tuple[Response, int]: db_user_id = get_session_db_user_id(session) - search_mode = app_config.get("SEARCH_MODE", "direct", user_id=db_user_id) + search_mode = app_config.get("SEARCH_MODE", "universal", user_id=db_user_id) default_release_source = app_config.get( "DEFAULT_RELEASE_SOURCE", - "direct_download", + "", user_id=db_user_id, ) default_release_source_audiobook = app_config.get( @@ -1145,6 +1145,7 @@ def api_config() -> Response | tuple[Response, int]: config = { "calibre_web_url": app_config.get("CALIBRE_WEB_URL", ""), "audiobook_library_url": app_config.get("AUDIOBOOK_LIBRARY_URL", ""), + "search_page_title": app_config.get("SEARCH_PAGE_TITLE", "Shelfmark"), "debug": app_config.get("DEBUG", False), "build_version": BUILD_VERSION, "release_version": RELEASE_VERSION, diff --git a/shelfmark/release_sources/direct_download.py b/shelfmark/release_sources/direct_download.py index 01746620..c919cbdf 100644 --- a/shelfmark/release_sources/direct_download.py +++ b/shelfmark/release_sources/direct_download.py @@ -7,7 +7,7 @@ from dataclasses import replace from http import HTTPStatus from typing import TYPE_CHECKING, ClassVar, NoReturn, TypedDict -from urllib.parse import quote +from urllib.parse import quote, urlparse import requests from bs4 import BeautifulSoup, Tag @@ -204,6 +204,24 @@ def _tag_has_class_containing(tag: Tag, needle: str) -> bool: _AA_PAGE_SOURCES = frozenset({"aa-slow-nowait", "aa-slow-wait"}) +def _is_configured_zlib_link(url: str) -> bool: + """Return True when a URL belongs to a configured Z-Library mirror.""" + from shelfmark.core.mirrors import get_zlib_cookie_domains + + hostname = (urlparse(url).hostname or "").lower() + if not hostname: + return False + + base_domain = ".".join(hostname.split(".")[-2:]) if "." in hostname else hostname + + for domain in get_zlib_cookie_domains(): + candidate = str(domain).lower() + if hostname == candidate or hostname.endswith(f".{candidate}") or base_domain == candidate: + return True + + return False + + def _get_md5_url_template(source_id: str) -> str | None: """Get URL template for MD5-based sources from centralized config.""" from shelfmark.core import mirrors @@ -246,6 +264,8 @@ def _get_source_priority() -> list[SourcePriorityEntry]: Fast sources come from user config (FAST_SOURCES_DISPLAY). Slow sources come from user config. """ + from shelfmark.core import mirrors + fast_sources = _parse_source_priority_entries( config.get("FAST_SOURCES_DISPLAY"), allowed_ids={"aa-fast", "libgen"}, @@ -253,13 +273,18 @@ def _get_source_priority() -> list[SourcePriorityEntry]: has_donator_key = bool(config.get("AA_DONATOR_KEY")) for source in fast_sources: - if source["id"] == "aa-fast" and not has_donator_key: + if (not mirrors.has_download_source_mirror_configuration(source["id"])) or ( + source["id"] == "aa-fast" and not has_donator_key + ): source["enabled"] = False slow_sources = _parse_source_priority_entries( config.get("SOURCE_PRIORITY"), excluded_ids={"aa-fast", "libgen"}, ) + for source in slow_sources: + if not mirrors.has_download_source_mirror_configuration(source["id"]): + source["enabled"] = False return fast_sources + slow_sources @@ -275,6 +300,31 @@ def _is_source_enabled(source_id: str) -> bool: return False +def _get_direct_download_unavailable_reason() -> str | None: + """Return a user-facing reason when Direct Download cannot be used.""" + from shelfmark.core import mirrors + + if not config.get("DIRECT_DOWNLOAD_ENABLED", False): + return ( + "Direct Download is disabled. Enable the source in Settings and add your mirror URLs." + ) + + if not mirrors.has_aa_mirror_configuration(): + return ( + "Direct Download is not configured. Add at least one Anna's Archive mirror URL in " + "Settings." + ) + + return None + + +def _ensure_direct_download_available() -> None: + """Raise a source-unavailable error when Direct Download is disabled or unconfigured.""" + reason = _get_direct_download_unavailable_reason() + if reason: + raise SearchUnavailableError(reason) + + _SIZE_UNIT_PATTERN = re.compile(r"(kb|mb|gb|tb)", re.IGNORECASE) @@ -907,7 +957,10 @@ def _get_download_urls_from_welib( if not _is_source_enabled("welib"): return [] - url = mirrors.get_welib_url_template().format(md5=book_id) + template = mirrors.get_welib_url_template() + if not template: + return [] + url = template.format(md5=book_id) logger.info("Fetching welib download URLs for %s", book_id) try: html = downloader.html_get_page( @@ -1144,7 +1197,7 @@ def _get_download_url( url = "" # Z-Library - if link.startswith("https://z-lib."): + if _is_configured_zlib_link(link): dl = soup.find("a", href=True, class_="addDownloadedBook") if not dl: # Retry after delay if page not fully loaded @@ -1454,6 +1507,7 @@ def get_record( fetch_download_count: bool = True, ) -> BrowseRecord | None: """Resolve a direct-download record for direct-mode info/download flows.""" + _ensure_direct_download_available() return get_book_info(record_id, fetch_download_count=fetch_download_count) def search_results_are_releases(self) -> bool: @@ -1506,6 +1560,7 @@ def search( content_type: Ignored - Direct download uses format filtering instead """ + _ensure_direct_download_available() lang_filter = plan.languages # Reset search type tracking @@ -1595,8 +1650,8 @@ def search( return [_browse_record_to_release(record) for record in all_results] def is_available(self) -> bool: - """Direct download is always available.""" - return True + """Check if Direct Download has been explicitly enabled and configured.""" + return _get_direct_download_unavailable_reason() is None @register_handler("direct_download") diff --git a/src/README.md b/src/README.md index 53f276d2..6189cf1d 100644 --- a/src/README.md +++ b/src/README.md @@ -42,7 +42,7 @@ make build make preview # Run type checking -make typecheck +make frontend-typecheck ``` Alternatively, from `src/frontend`: @@ -95,7 +95,7 @@ Output is generated in `src/frontend/dist/` Run TypeScript checks without building: ```bash -make typecheck +make frontend-typecheck ``` ## Debugging diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index 4daf08c1..f099662e 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -2470,6 +2470,7 @@ function App() { onSearch={handleSearchDispatch} isLoading={isSearching} isInitialState={isInitialState} + searchPageTitle={config?.search_page_title || 'Shelfmark'} bookLanguages={bookLanguages} defaultLanguage={defaultLanguageCodes} logoUrl={logoUrl} diff --git a/src/frontend/src/components/Dropdown.tsx b/src/frontend/src/components/Dropdown.tsx index f9181e3b..ea0a4a26 100644 --- a/src/frontend/src/components/Dropdown.tsx +++ b/src/frontend/src/components/Dropdown.tsx @@ -1,22 +1,9 @@ import type { ReactNode } from 'react'; import { useCallback, useLayoutEffect, useRef, useState } from 'react'; +import { createPortal } from 'react-dom'; import { useDismiss } from '../hooks/useDismiss'; -// Find the closest scrollable ancestor element -function getScrollableAncestor(element: HTMLElement | null): HTMLElement | null { - let current = element?.parentElement; - while (current) { - const style = getComputedStyle(current); - const overflowY = style.overflowY; - if (overflowY === 'auto' || overflowY === 'scroll' || overflowY === 'hidden') { - return current; - } - current = current.parentElement; - } - return null; -} - // Simple throttle function to limit how often a function can be called function throttle( fn: (...args: Args) => void, @@ -75,11 +62,11 @@ export const Dropdown = ({ }: DropdownProps) => { const [isOpen, setIsOpen] = useState(false); const containerRef = useRef(null); + const triggerRef = useRef(null); const panelRef = useRef(null); const [panelDirection, setPanelDirection] = useState<'down' | 'up'>('down'); - const [resolvedAlign, setResolvedAlign] = useState<'left' | 'right'>( - align === 'right' ? 'right' : 'left', - ); + const [panelPos, setPanelPos] = useState({ top: 0, left: 0, width: 0 }); + let triggerBorderRadius = '0.5rem'; if (triggerChrome === 'minimal') { triggerBorderRadius = '0'; @@ -87,13 +74,6 @@ export const Dropdown = ({ triggerBorderRadius = panelDirection === 'down' ? '0.5rem 0.5rem 0 0' : '0 0 0.5rem 0.5rem'; } - let panelOffsetClassName = ''; - if (panelDirection === 'down') { - panelOffsetClassName = renderTrigger ? 'mt-2' : ''; - } else { - panelOffsetClassName = renderTrigger ? 'bottom-full mb-2' : 'bottom-full'; - } - let panelBorderRadius = '0.5rem'; if (!renderTrigger) { panelBorderRadius = panelDirection === 'down' ? '0 0 0.5rem 0.5rem' : '0.5rem 0.5rem 0 0'; @@ -113,47 +93,47 @@ export const Dropdown = ({ onOpenChange?.(false); }, [onOpenChange]); - useDismiss(isOpen, [containerRef], close); + useDismiss(isOpen, [containerRef, panelRef], close); - // Memoize the panel direction calculation - const updatePanelDirection = useCallback(() => { - if (!containerRef.current || !panelRef.current) { - return; - } + // Compute panel direction and fixed position relative to the trigger + const updatePanelPosition = useCallback(() => { + if (!triggerRef.current || !panelRef.current) return; - const rect = containerRef.current.getBoundingClientRect(); + const rect = triggerRef.current.getBoundingClientRect(); const panelHeight = panelRef.current.offsetHeight || panelRef.current.scrollHeight; + const panelWidth = panelRef.current.offsetWidth || panelRef.current.scrollWidth; - // Check if we're inside a scrollable container and use its bounds - const scrollableAncestor = getScrollableAncestor(containerRef.current); - const containerBottom = scrollableAncestor - ? scrollableAncestor.getBoundingClientRect().bottom - : window.innerHeight; - const containerTop = scrollableAncestor ? scrollableAncestor.getBoundingClientRect().top : 0; - - const spaceBelow = containerBottom - rect.bottom - 8; - const spaceAbove = rect.top - containerTop - 8; + // Direction: flip up if not enough space below but enough above + const spaceBelow = window.innerHeight - rect.bottom - 8; + const spaceAbove = rect.top - 8; const shouldOpenUp = spaceBelow < panelHeight && spaceAbove >= panelHeight; - setPanelDirection(shouldOpenUp ? 'up' : 'down'); - // Auto horizontal alignment: check if panel overflows viewport right/left + // Vertical: seamless trigger uses -1px border overlap, custom trigger uses 8px gap + let top: number; + if (shouldOpenUp) { + top = renderTrigger ? rect.top - panelHeight - 8 : rect.top - panelHeight + 1; + } else { + top = renderTrigger ? rect.bottom + 8 : rect.bottom - 1; + } + + // Horizontal alignment + let left: number; if (align === 'auto') { - const panelWidth = panelRef.current.offsetWidth || panelRef.current.scrollWidth; const overflowsRight = rect.left + panelWidth > window.innerWidth - 8; const overflowsLeft = rect.right - panelWidth < 8; - - if (overflowsRight && !overflowsLeft) { - setResolvedAlign('right'); - } else if (overflowsLeft && !overflowsRight) { - setResolvedAlign('left'); - } else { - setResolvedAlign('left'); - } + left = + overflowsRight && !overflowsLeft + ? rect.right - Math.max(panelWidth, rect.width) + : rect.left; + } else if (align === 'right') { + left = rect.right - Math.max(panelWidth, rect.width); } else { - setResolvedAlign(align === 'right' ? 'right' : 'left'); + left = rect.left; } - }, [align]); + + setPanelPos({ top, left, width: rect.width }); + }, [align, renderTrigger]); useLayoutEffect(() => { if (!isOpen) { @@ -161,9 +141,9 @@ export const Dropdown = ({ } // Throttle scroll/resize handlers to reduce layout thrashing - const throttledUpdate = throttle(updatePanelDirection, 100); + const throttledUpdate = throttle(updatePanelPosition, 100); - updatePanelDirection(); + updatePanelPosition(); window.addEventListener('resize', throttledUpdate); window.addEventListener('scroll', throttledUpdate, true); @@ -171,7 +151,7 @@ export const Dropdown = ({ window.removeEventListener('resize', throttledUpdate); window.removeEventListener('scroll', throttledUpdate, true); }; - }, [isOpen, updatePanelDirection]); + }, [isOpen, updatePanelPosition]); return (
@@ -183,7 +163,7 @@ export const Dropdown = ({ {label} )} -
+
{renderTrigger ? ( renderTrigger({ isOpen, toggle: toggleOpen }) ) : ( @@ -214,25 +194,28 @@ export const Dropdown = ({ )} - {isOpen && ( -
-
- {children({ close })} -
-
- )} + {isOpen && + createPortal( +
+
+ {children({ close })} +
+
, + document.body, + )}
); diff --git a/src/frontend/src/components/OnboardingModal.tsx b/src/frontend/src/components/OnboardingModal.tsx index bbc4eea2..eacb0333 100644 --- a/src/frontend/src/components/OnboardingModal.tsx +++ b/src/frontend/src/components/OnboardingModal.tsx @@ -3,7 +3,7 @@ import { useState, useCallback, useMemo } from 'react'; import { useBodyScrollLock } from '../hooks/useBodyScrollLock'; import { useEscapeKey } from '../hooks/useEscapeKey'; import { useMountEffect } from '../hooks/useMountEffect'; -import type { OnboardingStep } from '../services/api'; +import type { OnboardingStep, OnboardingStepCondition } from '../services/api'; import { getOnboarding, saveOnboarding, @@ -32,8 +32,10 @@ interface OnboardingModalProps { onShowToast?: (message: string, type: 'success' | 'error' | 'info') => void; } +type VisibilityCondition = ShowWhenCondition | OnboardingStepCondition; + function evaluateShowWhenCondition( - showWhen: ShowWhenCondition, + showWhen: VisibilityCondition, values: Record, ): boolean { const currentValue = values[showWhen.field]; @@ -45,9 +47,19 @@ function evaluateShowWhenCondition( return currentValue !== undefined && currentValue !== null && currentValue !== ''; } - return Array.isArray(showWhen.value) - ? typeof currentValue === 'string' && showWhen.value.includes(currentValue) - : currentValue === showWhen.value; + if (Array.isArray(currentValue)) { + if (Array.isArray(showWhen.value)) { + return showWhen.value.every((value) => currentValue.includes(value)); + } + return showWhen.value !== undefined && currentValue.includes(showWhen.value); + } + + if (Array.isArray(showWhen.value)) { + const currentStringValue = toStringValue(currentValue); + return currentStringValue !== undefined && showWhen.value.includes(currentStringValue); + } + + return currentValue === showWhen.value; } // Check if a field should be visible based on showWhen condition @@ -70,11 +82,7 @@ function isFieldVisible(field: SettingsField, values: Record): function isStepVisible(step: OnboardingStep, values: Record): boolean { if (!step.showWhen || step.showWhen.length === 0) return true; - // All conditions must be true (AND logic) - return step.showWhen.every((condition) => { - const currentValue = values[condition.field]; - return currentValue === condition.value; - }); + return step.showWhen.every((condition) => evaluateShowWhenCondition(condition, values)); } // Render the appropriate field component based on type @@ -240,8 +248,12 @@ const OnboardingModalSession = ({ return steps.filter((step) => isStepVisible(step, values)); }, [steps, values]); + // Clamp step index to valid range (handles steps becoming hidden) + const clampedStepIndex = + visibleSteps.length === 0 ? 0 : Math.min(currentStepIndex, visibleSteps.length - 1); + // Get current step - const currentStep = visibleSteps[currentStepIndex]; + const currentStep = visibleSteps[clampedStepIndex]; // Get visible fields for current step const visibleFields = useMemo(() => { @@ -256,17 +268,17 @@ const OnboardingModalSession = ({ // Handle next step const handleNext = useCallback(() => { - if (currentStepIndex < visibleSteps.length - 1) { - setCurrentStepIndex(currentStepIndex + 1); + if (clampedStepIndex < visibleSteps.length - 1) { + setCurrentStepIndex(clampedStepIndex + 1); } - }, [currentStepIndex, visibleSteps.length]); + }, [clampedStepIndex, visibleSteps.length]); // Handle previous step const handleBack = useCallback(() => { - if (currentStepIndex > 0) { - setCurrentStepIndex(currentStepIndex - 1); + if (clampedStepIndex > 0) { + setCurrentStepIndex(clampedStepIndex - 1); } - }, [currentStepIndex]); + }, [clampedStepIndex]); // Handle skip const handleSkip = useCallback(async () => { @@ -399,9 +411,9 @@ const OnboardingModalSession = ({ ); } - const isFirstStep = currentStepIndex === 0; - const isLastStep = currentStepIndex === visibleSteps.length - 1; - const progress = ((currentStepIndex + 1) / visibleSteps.length) * 100; + const isFirstStep = clampedStepIndex === 0; + const isLastStep = clampedStepIndex === visibleSteps.length - 1; + const progress = ((clampedStepIndex + 1) / visibleSteps.length) * 100; return (
@@ -412,72 +424,76 @@ const OnboardingModalSession = ({ {/* Modal */}
- {/* Header */} -
-
-
- {currentStepIndex + 1} -
-
-

{currentStep?.title || 'Setup'}

-

- Step {currentStepIndex + 1} of {visibleSteps.length} -

+
+ {/* Header */} +
+
+
+ {clampedStepIndex + 1} +
+
+

{currentStep?.title || 'Setup'}

+

+ Step {clampedStepIndex + 1} of {visibleSteps.length} +

+
-
- -
+ + + + +
- {/* Progress bar */} -
-
+ {/* Progress bar */} +
+
+
{/* Content */} -
- {visibleFields.map((field) => { - const isDisabled = 'fromEnv' in field ? (field.fromEnv ?? false) : false; - return ( - - {renderField( - field, - values[field.key], - (v) => handleChange(field.key, v), - () => handleAction(field.key), - isDisabled, - )} - - ); - })} +
+
+ {visibleFields.map((field) => { + const isDisabled = 'fromEnv' in field ? (field.fromEnv ?? false) : false; + return ( + + {renderField( + field, + values[field.key], + (v) => handleChange(field.key, v), + () => handleAction(field.key), + isDisabled, + )} + + ); + })} +
{/* Footer */} -
+
{ onComplete(); - const parsedSearchMode = config.search_mode || 'direct'; + const parsedSearchMode = config.search_mode || 'universal'; const urlContentTypeOverride = parsedSearchMode === 'universal' ? parsedParams.contentType : undefined; diff --git a/src/frontend/src/components/settings/users/UserSearchPreferencesSection.tsx b/src/frontend/src/components/settings/users/UserSearchPreferencesSection.tsx index df357e54..d82aefe2 100644 --- a/src/frontend/src/components/settings/users/UserSearchPreferencesSection.tsx +++ b/src/frontend/src/components/settings/users/UserSearchPreferencesSection.tsx @@ -153,11 +153,11 @@ export const UserSearchPreferencesSection = ({ }); }; - const searchModeValue = readValue('SEARCH_MODE', 'direct'); + const searchModeValue = readValue('SEARCH_MODE', 'universal'); const effectiveSearchMode = normalizeSearchMode(searchModeValue); const metadataProviderValue = readValue('METADATA_PROVIDER'); const metadataProviderAudiobookValue = readValue('METADATA_PROVIDER_AUDIOBOOK'); - const defaultReleaseSourceValue = readValue('DEFAULT_RELEASE_SOURCE', 'direct_download'); + const defaultReleaseSourceValue = readValue('DEFAULT_RELEASE_SOURCE'); const defaultAudiobookReleaseSourceValue = readValue('DEFAULT_RELEASE_SOURCE_AUDIOBOOK'); const canOverrideSearchMode = diff --git a/src/frontend/src/hooks/useSearch.ts b/src/frontend/src/hooks/useSearch.ts index 778ce8da..537531f3 100644 --- a/src/frontend/src/hooks/useSearch.ts +++ b/src/frontend/src/hooks/useSearch.ts @@ -165,7 +165,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { providerOverride?: string; }) => { const effectiveContentType = contentTypeOverride ?? contentType; - const searchMode = (searchModeOverride ?? config?.search_mode) || 'direct'; + const searchMode = (searchModeOverride ?? config?.search_mode) || 'universal'; // In universal mode, check if we have either a query or field values if (searchMode === 'universal') { @@ -314,7 +314,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { // Load more results (universal mode pagination) const loadMore = useCallback( async (config: AppConfig | null, searchModeOverride?: SearchMode) => { - const searchMode = (searchModeOverride ?? config?.search_mode) || 'direct'; + const searchMode = (searchModeOverride ?? config?.search_mode) || 'universal'; if (searchMode !== 'universal') return; if (!lastSearchParamsRef.current) return; if (isLoadingMore || !hasMore) return; diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index 6c0ace61..fcba6413 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -697,6 +697,7 @@ export const executeSettingsAction = async ( export interface OnboardingStepCondition { field: string; value: unknown; + notEmpty?: boolean; } export interface OnboardingStep { diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts index 36b7c408..24c07b92 100644 --- a/src/frontend/src/types/index.ts +++ b/src/frontend/src/types/index.ts @@ -265,6 +265,7 @@ export type BooksOutputMode = 'folder' | 'booklore' | 'email'; export interface AppConfig { calibre_web_url: string; audiobook_library_url: string; + search_page_title: string; debug: boolean; build_version: string; release_version: string; diff --git a/tests/README.md b/tests/README.md index d3558c62..7f1e22e5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -11,12 +11,9 @@ make install-python-dev # Run all unit tests locally (fast, no external dependencies) uv run pytest tests/ -v -m "not integration and not e2e" -# Run Python static analysis +# Run all Python static analysis (lint, format, typecheck, dead code) make python-checks -# Run lightweight test lint/type checks -make python-test-checks - # Run E2E API tests against a running app stack uv run pytest tests/e2e/ -v -m e2e diff --git a/tests/bypass/test_internal_bypasser.py b/tests/bypass/test_internal_bypasser.py index ebae022a..fd37ae37 100644 --- a/tests/bypass/test_internal_bypasser.py +++ b/tests/bypass/test_internal_bypasser.py @@ -79,6 +79,52 @@ async def evaluate(self, _expr): assert internal_bypasser.get_cf_user_agent_for_domain("example.com") == "TestUA/1.0" +def test_extract_cookies_from_cdp_keeps_full_session_cookies_for_configured_zlib_domains( + monkeypatch, +): + import time + + import shelfmark.bypass.internal_bypasser as internal_bypasser + + class FakeCookie: + def __init__(self, name, value, domain, path, expires, secure=True): + self.name = name + self.value = value + self.domain = domain + self.path = path + self.expires = expires + self.secure = secure + + class FakeCookies: + async def get_all(self, requests_cookie_format=False): + assert requests_cookie_format is True + return [ + FakeCookie("cf_clearance", "abc", "z-lib.fm", "/", int(time.time()) + 3600), + FakeCookie("sessionid", "zzz", "z-lib.fm", "/", int(time.time()) + 3600), + ] + + class FakeDriver: + cookies = FakeCookies() + + class FakePage: + async def evaluate(self, _expr): + return "TestUA/1.0" + + monkeypatch.setattr(internal_bypasser, "_get_full_cookie_domains", lambda: {"z-lib.fm"}) + + internal_bypasser.clear_cf_cookies() + asyncio.run( + internal_bypasser._extract_cookies_from_cdp( + FakeDriver(), + FakePage(), + "https://z-lib.fm/books/example", + ) + ) + + cookies = internal_bypasser.get_cf_cookies_for_domain("z-lib.fm") + assert cookies == {"cf_clearance": "abc", "sessionid": "zzz"} + + def test_extract_cookies_from_cdp_normalizes_session_expiry(): import time diff --git a/tests/config/test_download_settings.py b/tests/config/test_download_settings.py index 8bd0c660..4fd67fc7 100644 --- a/tests/config/test_download_settings.py +++ b/tests/config/test_download_settings.py @@ -277,3 +277,79 @@ def _fake_validate_destination(path, status_callback): assert result["success"] is True assert captured["path"] == destination + + +def test_search_mode_defaults_to_universal_and_direct_mentions_configuration(): + from shelfmark.config.settings import search_mode_settings + + fields = search_mode_settings() + search_mode_field = next( + field for field in fields if getattr(field, "key", None) == "SEARCH_MODE" + ) + direct_option = next( + option for option in search_mode_field.options if option["value"] == "direct" + ) + + assert search_mode_field.default == "universal" + assert "mirror URLs" in direct_option["description"] + + +def test_download_source_settings_include_direct_download_toggle(): + from shelfmark.config.settings import download_source_settings + + fields = download_source_settings() + toggle_field = next( + field for field in fields if getattr(field, "key", None) == "DIRECT_DOWNLOAD_ENABLED" + ) + + assert toggle_field.default is False + assert "Add your own mirror URLs" in toggle_field.description + + +def test_fast_source_options_lock_entries_without_mirror_or_donator_requirements(monkeypatch): + from shelfmark.config.settings import _get_fast_source_options + + def _fake_get(key: str, default=None, user_id=None): + del user_id + values = { + "AA_DONATOR_KEY": "", + } + return values.get(key, default) + + monkeypatch.setattr("shelfmark.core.config.config.get", _fake_get) + monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: False) + monkeypatch.setattr("shelfmark.core.mirrors.has_libgen_mirror_configuration", lambda: True) + + options = {option["id"]: option for option in _get_fast_source_options()} + + assert options["aa-fast"]["isLocked"] is True + assert ( + options["aa-fast"]["disabledReason"] == "Add at least one Anna's Archive mirror in Mirrors" + ) + assert options["libgen"]["isLocked"] is False + assert options["libgen"]["disabledReason"] is None + + +def test_slow_source_options_lock_entries_until_mirror_dependencies_exist(monkeypatch): + from shelfmark.config.settings import _get_slow_source_options + + def _fake_get(key: str, default=None, user_id=None): + del user_id + values = { + "USE_CF_BYPASS": True, + } + return values.get(key, default) + + monkeypatch.setattr("shelfmark.core.config.config.get", _fake_get) + monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True) + monkeypatch.setattr("shelfmark.core.mirrors.has_welib_mirror_configuration", lambda: False) + monkeypatch.setattr("shelfmark.core.mirrors.has_zlib_mirror_configuration", lambda: False) + + options = {option["id"]: option for option in _get_slow_source_options()} + + assert options["aa-slow-nowait"]["isLocked"] is False + assert options["aa-slow-wait"]["isLocked"] is False + assert options["welib"]["isLocked"] is True + assert options["welib"]["disabledReason"] == "Add at least one Welib mirror in Mirrors" + assert options["zlib"]["isLocked"] is True + assert options["zlib"]["disabledReason"] == "Add at least one Z-Library mirror in Mirrors" diff --git a/tests/config/test_generate_env_docs.py b/tests/config/test_generate_env_docs.py new file mode 100644 index 00000000..cbdccda5 --- /dev/null +++ b/tests/config/test_generate_env_docs.py @@ -0,0 +1,40 @@ +from scripts.generate_env_docs import generate_env_docs + + +def test_generated_env_docs_use_canonical_mirror_env_vars() -> None: + docs = generate_env_docs() + + for canonical_var in ( + "AA_BASE_URL", + "AA_MIRROR_URLS", + "LIBGEN_MIRROR_URLS", + "ZLIB_MIRROR_URLS", + "WELIB_MIRROR_URLS", + ): + assert f"`{canonical_var}`" in docs + + for legacy_var in ( + "AA_ADDITIONAL_URLS", + "LIBGEN_ADDITIONAL_URLS", + "ZLIB_PRIMARY_URL", + "ZLIB_ADDITIONAL_URLS", + "WELIB_PRIMARY_URL", + "WELIB_ADDITIONAL_URLS", + ): + assert f"`{legacy_var}`" not in docs + + assert "https://annas-archive.gl" not in docs + + +def test_generated_env_docs_describe_mirror_lists_as_comma_separated_strings() -> None: + docs = generate_env_docs() + + assert ( + "| `AA_MIRROR_URLS` | List the Anna's Archive mirror URLs you want Shelfmark to use. " + "Type a URL and press Enter to add it. Order matters when Auto is selected. | " + "string (comma-separated) | _empty list_ |" + ) in docs + assert ( + "| `LIBGEN_MIRROR_URLS` | Mirrors are tried in the order you add them until one works. | " + "string (comma-separated) | _empty list_ |" + ) in docs diff --git a/tests/config/test_mirror_settings_migration.py b/tests/config/test_mirror_settings_migration.py new file mode 100644 index 00000000..00b8cf46 --- /dev/null +++ b/tests/config/test_mirror_settings_migration.py @@ -0,0 +1,203 @@ +from unittest.mock import MagicMock + + +def test_mirror_settings_use_canonical_tag_lists_for_all_non_base_mirror_sets(): + import shelfmark.config.settings # noqa: F401 + from shelfmark.core.settings_registry import TagListField, get_settings_tab + + tab = get_settings_tab("mirrors") + + assert tab is not None + fields = {field.key: field for field in tab.fields if hasattr(field, "key")} + + assert isinstance(fields["AA_MIRROR_URLS"], TagListField) + assert isinstance(fields["LIBGEN_MIRROR_URLS"], TagListField) + assert isinstance(fields["ZLIB_MIRROR_URLS"], TagListField) + assert isinstance(fields["WELIB_MIRROR_URLS"], TagListField) + + +def test_migrate_mirror_settings_does_not_seed_defaults_on_fresh_install(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.setattr( + registry, "load_config_file", lambda tab_name: {} if tab_name == "mirrors" else {} + ) + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_mirror_settings() + + saves.assert_not_called() + + +def test_migrate_mirror_settings_converts_legacy_additional_urls_to_list(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.setattr( + registry, + "load_config_file", + lambda tab_name: ( + {"AA_ADDITIONAL_URLS": "aa.one, https://aa.two/"} if tab_name == "mirrors" else {} + ), + ) + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_mirror_settings() + + saves.assert_called_once_with( + "mirrors", + { + "AA_MIRROR_URLS": ["https://aa.one", "https://aa.two"], + }, + ) + + +def test_migrate_mirror_settings_preserves_existing_list_without_reseeding(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.setattr( + registry, + "load_config_file", + lambda tab_name: ( + { + "AA_MIRROR_URLS": ["https://annas-archive.existing"], + "_AA_MIRRORS_DEFAULTS_HASH": "old-hash", + } + if tab_name == "mirrors" + else {} + ), + ) + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_mirror_settings() + + saves.assert_not_called() + + +def test_migrate_mirror_settings_converts_legacy_split_fields_to_canonical_lists(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.setattr( + registry, + "load_config_file", + lambda tab_name: ( + { + "LIBGEN_ADDITIONAL_URLS": "libgen.one, https://libgen.two/", + "ZLIB_PRIMARY_URL": "zlib.primary", + "ZLIB_ADDITIONAL_URLS": "https://zlib.primary, zlib.backup", + "WELIB_PRIMARY_URL": "welib.primary", + "WELIB_ADDITIONAL_URLS": "https://welib.backup", + } + if tab_name == "mirrors" + else {} + ), + ) + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_mirror_settings() + + saves.assert_called_once_with( + "mirrors", + { + "LIBGEN_MIRROR_URLS": ["https://libgen.one", "https://libgen.two"], + "ZLIB_MIRROR_URLS": ["https://zlib.primary", "https://zlib.backup"], + "WELIB_MIRROR_URLS": ["https://welib.primary", "https://welib.backup"], + }, + ) + + +def test_canonical_mirror_fields_accept_legacy_env_vars(monkeypatch): + import shelfmark.config.settings # noqa: F401 + from shelfmark.core.settings_registry import ( + get_setting_value, + get_settings_tab, + is_value_from_env, + ) + + monkeypatch.setenv("ZLIB_PRIMARY_URL", "zlib.primary") + monkeypatch.setenv("ZLIB_ADDITIONAL_URLS", "https://zlib.primary, zlib.backup") + + tab = get_settings_tab("mirrors") + + assert tab is not None + fields = {field.key: field for field in tab.fields if hasattr(field, "key")} + zlib_field = fields["ZLIB_MIRROR_URLS"] + + assert is_value_from_env(zlib_field) is True + assert get_setting_value(zlib_field, "mirrors") == [ + "https://zlib.primary", + "https://zlib.backup", + ] + + +def test_migrate_direct_download_upgrade_enables_direct_download_without_seeding_mirrors( + monkeypatch, +): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.setattr(registry, "load_config_file", lambda tab_name: {}) + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_direct_download_upgrade(existing_install=True) + + saves.assert_called_once_with("download_sources", {"DIRECT_DOWNLOAD_ENABLED": True}) + + +def test_migrate_direct_download_upgrade_skips_fresh_install(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_direct_download_upgrade(existing_install=False) + + saves.assert_not_called() + + +def test_migrate_search_page_title_preserves_legacy_title_for_existing_installs(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.delenv("SEARCH_PAGE_TITLE", raising=False) + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_search_page_title(existing_install=True, had_existing_value=False) + + saves.assert_called_once_with("general", {"SEARCH_PAGE_TITLE": "Book Search & Download"}) + + +def test_migrate_search_page_title_skips_when_value_already_exists(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.delenv("SEARCH_PAGE_TITLE", raising=False) + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_search_page_title(existing_install=True, had_existing_value=True) + + saves.assert_not_called() + + +def test_migrate_search_page_title_skips_when_env_var_is_set(monkeypatch): + import shelfmark.core.settings_registry as registry + + saves = MagicMock(return_value=True) + + monkeypatch.setenv("SEARCH_PAGE_TITLE", "Shelfmark") + monkeypatch.setattr(registry, "save_config_file", saves) + + registry.migrate_search_page_title(existing_install=True, had_existing_value=False) + + saves.assert_not_called() diff --git a/tests/core/test_config_api.py b/tests/core/test_config_api.py index 3bb91212..4a1a8fb6 100644 --- a/tests/core/test_config_api.py +++ b/tests/core/test_config_api.py @@ -42,6 +42,7 @@ def fake_get(key, default=None, user_id=None): "SHOW_RELEASE_SOURCE_LINKS": False, "SHOW_COMBINED_SELECTOR": False, "SEARCH_MODE": "universal", + "SEARCH_PAGE_TITLE": "Custom Shelfmark", "METADATA_PROVIDER": "openlibrary", "METADATA_PROVIDER_AUDIOBOOK": "", "DEFAULT_RELEASE_SOURCE": "prowlarr", @@ -68,6 +69,7 @@ def fake_get(key, default=None, user_id=None): assert data["show_release_source_links"] is False assert data["show_combined_selector"] is False assert data["search_mode"] == "universal" + assert data["search_page_title"] == "Custom Shelfmark" assert data["metadata_sort_options"] == ["sort-a"] assert data["metadata_search_fields"] == ["field-a"] assert data["default_release_source"] == "prowlarr" diff --git a/tests/core/test_mirrors_config.py b/tests/core/test_mirrors_config.py index 6d1d2cc5..b618983d 100644 --- a/tests/core/test_mirrors_config.py +++ b/tests/core/test_mirrors_config.py @@ -24,7 +24,7 @@ def test_get_aa_mirrors_prefers_full_configured_list(monkeypatch): ] -def test_get_aa_mirrors_falls_back_to_defaults_and_legacy_additional(monkeypatch): +def test_get_aa_mirrors_uses_legacy_additional_when_no_explicit_list(monkeypatch): dummy = _DummyConfig( { "AA_MIRROR_URLS": [], @@ -33,8 +33,111 @@ def test_get_aa_mirrors_falls_back_to_defaults_and_legacy_additional(monkeypatch ) monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) - aa = mirrors.get_aa_mirrors() - for default_mirror in mirrors.DEFAULT_AA_MIRRORS: - assert default_mirror in aa - assert "https://extra.example" in aa - assert "https://extra2.example" in aa + assert mirrors.get_aa_mirrors() == [ + "https://extra.example", + "https://extra2.example", + ] + + +def test_get_aa_mirrors_returns_empty_when_unconfigured(monkeypatch): + monkeypatch.setattr(mirrors, "_get_config", lambda: _DummyConfig({})) + + assert mirrors.get_aa_mirrors() == [] + + +def test_has_aa_mirror_configuration_accepts_custom_base_url_without_list(monkeypatch): + dummy = _DummyConfig( + { + "AA_BASE_URL": "https://custom-aa.example", + "AA_MIRROR_URLS": [], + } + ) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.has_aa_mirror_configuration() is True + + +def test_get_libgen_mirrors_returns_user_supplied_urls_only(monkeypatch): + dummy = _DummyConfig({"LIBGEN_MIRROR_URLS": ["libgen.one", "https://libgen.two/"]}) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.get_libgen_mirrors() == [ + "https://libgen.one", + "https://libgen.two", + ] + + +def test_get_libgen_mirrors_falls_back_to_legacy_config(monkeypatch): + dummy = _DummyConfig({"LIBGEN_ADDITIONAL_URLS": "libgen.legacy, https://libgen.backup/"}) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.get_libgen_mirrors() == [ + "https://libgen.legacy", + "https://libgen.backup", + ] + + +def test_get_zlib_mirrors_prefers_canonical_list(monkeypatch): + dummy = _DummyConfig({"ZLIB_MIRROR_URLS": ["zlib.primary", "https://zlib.backup"]}) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.get_zlib_mirrors() == [ + "https://zlib.primary", + "https://zlib.backup", + ] + assert mirrors.get_zlib_url_template() == "https://zlib.primary/md5/{md5}" + + +def test_get_zlib_mirrors_falls_back_to_primary_then_additional(monkeypatch): + dummy = _DummyConfig( + { + "ZLIB_PRIMARY_URL": "zlib.primary", + "ZLIB_ADDITIONAL_URLS": "https://zlib.primary, zlib.backup", + } + ) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.get_zlib_mirrors() == [ + "https://zlib.primary", + "https://zlib.backup", + ] + assert mirrors.get_zlib_url_template() == "https://zlib.primary/md5/{md5}" + + +def test_get_zlib_url_template_returns_none_without_config(monkeypatch): + monkeypatch.setattr(mirrors, "_get_config", lambda: _DummyConfig({})) + + assert mirrors.get_zlib_url_template() is None + + +def test_get_welib_mirrors_prefers_primary_then_additional(monkeypatch): + dummy = _DummyConfig({"WELIB_MIRROR_URLS": ["welib.primary", "https://welib.backup"]}) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.get_welib_mirrors() == [ + "https://welib.primary", + "https://welib.backup", + ] + assert mirrors.get_welib_url_template() == "https://welib.primary/md5/{md5}" + + +def test_get_welib_mirrors_falls_back_to_primary_then_additional(monkeypatch): + dummy = _DummyConfig( + { + "WELIB_PRIMARY_URL": "welib.primary", + "WELIB_ADDITIONAL_URLS": "https://welib.backup", + } + ) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.get_welib_mirrors() == [ + "https://welib.primary", + "https://welib.backup", + ] + assert mirrors.get_welib_url_template() == "https://welib.primary/md5/{md5}" + + +def test_get_welib_url_template_returns_none_without_config(monkeypatch): + monkeypatch.setattr(mirrors, "_get_config", lambda: _DummyConfig({})) + + assert mirrors.get_welib_url_template() is None diff --git a/tests/core/test_onboarding.py b/tests/core/test_onboarding.py new file mode 100644 index 00000000..2b14d515 --- /dev/null +++ b/tests/core/test_onboarding.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from collections import defaultdict + +import shelfmark.config.settings as settings_config +import shelfmark.metadata_providers.googlebooks as googlebooks_provider +import shelfmark.metadata_providers.hardcover as hardcover_provider +import shelfmark.metadata_providers.openlibrary as openlibrary_provider +import shelfmark.release_sources.audiobookbay.settings as audiobookbay_settings +import shelfmark.release_sources.irc.settings as irc_settings +import shelfmark.release_sources.prowlarr.settings as prowlarr_settings + +_REGISTERED_SETTINGS_MODULES = ( + settings_config, + googlebooks_provider, + hardcover_provider, + openlibrary_provider, + audiobookbay_settings, + irc_settings, + prowlarr_settings, +) + + +def _group_save_calls( + save_calls: list[tuple[str, dict[str, object]]], +) -> dict[str, list[dict[str, object]]]: + grouped: dict[str, list[dict[str, object]]] = defaultdict(list) + for tab_name, payload in save_calls: + grouped[tab_name].append(payload) + return grouped + + +def test_get_onboarding_config_uses_release_source_settings_pages(): + from shelfmark.core.onboarding import get_onboarding_config + + config = get_onboarding_config() + steps = {step["id"]: step for step in config["steps"]} + + assert "release_sources" in steps + assert "direct_download_setup" in steps + assert "direct_download_cloudflare_bypass" in steps + assert "prowlarr" in steps + assert "audiobookbay" in steps + assert "irc" in steps + + direct_download_fields = { + field["key"] for field in steps["direct_download_setup"]["fields"] if "key" in field + } + direct_download_bypass_fields = { + field["key"] + for field in steps["direct_download_cloudflare_bypass"]["fields"] + if "key" in field + } + prowlarr_fields = { + field["key"]: field for field in steps["prowlarr"]["fields"] if "key" in field + } + audiobookbay_fields = { + field["key"]: field for field in steps["audiobookbay"]["fields"] if "key" in field + } + + assert "DIRECT_DOWNLOAD_ENABLED" not in direct_download_fields + assert "FAST_SOURCES_DISPLAY" not in direct_download_fields + assert "SOURCE_PRIORITY" not in direct_download_fields + assert "AA_DONATOR_KEY" in direct_download_fields + assert "AA_MIRROR_URLS" in direct_download_fields + assert "LIBGEN_ADDITIONAL_URLS" not in direct_download_fields + assert "ZLIB_PRIMARY_URL" not in direct_download_fields + assert "EXT_BYPASSER_TIMEOUT" not in direct_download_bypass_fields + assert "PROWLARR_ENABLED" not in prowlarr_fields + assert "PROWLARR_AUTO_EXPAND" not in prowlarr_fields + assert "ABB_ENABLED" not in audiobookbay_fields + assert "ABB_PAGE_LIMIT" not in audiobookbay_fields + assert "showWhen" not in prowlarr_fields["PROWLARR_URL"] + assert "showWhen" not in audiobookbay_fields["ABB_HOSTNAME"] + + +def test_save_onboarding_settings_enables_selected_release_sources(monkeypatch): + import shelfmark.core.config as app_config + import shelfmark.core.onboarding as onboarding + + save_calls: list[tuple[str, dict[str, object]]] = [] + + def _fake_save(tab_name: str, values: dict[str, object]) -> bool: + save_calls.append((tab_name, dict(values))) + return True + + monkeypatch.setattr(onboarding, "save_config_file", _fake_save) + monkeypatch.setattr(onboarding, "mark_onboarding_complete", lambda: True) + monkeypatch.setattr(app_config.config, "refresh", lambda: None, raising=False) + + result = onboarding.save_onboarding_settings( + { + "SEARCH_MODE": "universal", + "METADATA_PROVIDER": "hardcover", + "HARDCOVER_API_KEY": "hardcover-api-key", + onboarding.ONBOARDING_RELEASE_SOURCES_KEY: [ + "direct_download", + "audiobookbay", + "prowlarr", + ], + "AA_MIRROR_URLS": ["https://annas-archive.example"], + "PROWLARR_URL": "http://prowlarr:9696", + "PROWLARR_API_KEY": "secret-key", + "ABB_HOSTNAME": "audiobookbay.lu", + } + ) + + grouped_calls = _group_save_calls(save_calls) + + assert result == {"success": True, "message": "Onboarding complete!"} + assert any( + payload.get("DIRECT_DOWNLOAD_ENABLED") is True + for payload in grouped_calls["download_sources"] + ) + assert any( + payload.get("AA_MIRROR_URLS") == ["https://annas-archive.example"] + for payload in grouped_calls["mirrors"] + ) + assert any( + payload.get("PROWLARR_ENABLED") is True for payload in grouped_calls["prowlarr_config"] + ) + assert any( + payload.get("ABB_ENABLED") is True for payload in grouped_calls["audiobookbay_config"] + ) + assert any( + payload.get("PROWLARR_URL") == "http://prowlarr:9696" + and payload.get("PROWLARR_API_KEY") == "secret-key" + for payload in grouped_calls["prowlarr_config"] + ) + assert any( + payload.get("HARDCOVER_ENABLED") is True + and payload.get("HARDCOVER_API_KEY") == "hardcover-api-key" + for payload in grouped_calls["hardcover"] + ) + assert any( + payload.get("DEFAULT_RELEASE_SOURCE") == "direct_download" + and payload.get("DEFAULT_RELEASE_SOURCE_AUDIOBOOK") == "audiobookbay" + for payload in grouped_calls["search_mode"] + ) + assert not any( + onboarding.ONBOARDING_RELEASE_SOURCES_KEY in payload for _, payload in save_calls + ) + + +def test_save_onboarding_settings_skips_hidden_fields(monkeypatch): + import shelfmark.core.config as app_config + import shelfmark.core.onboarding as onboarding + + save_calls: list[tuple[str, dict[str, object]]] = [] + + def _fake_save(tab_name: str, values: dict[str, object]) -> bool: + save_calls.append((tab_name, dict(values))) + return True + + monkeypatch.setattr(onboarding, "save_config_file", _fake_save) + monkeypatch.setattr(onboarding, "mark_onboarding_complete", lambda: True) + monkeypatch.setattr(app_config.config, "refresh", lambda: None, raising=False) + + result = onboarding.save_onboarding_settings( + { + "SEARCH_MODE": "direct", + "USE_CF_BYPASS": True, + "USING_EXTERNAL_BYPASSER": False, + "EXT_BYPASSER_URL": "http://should-not-save.example", + "EXT_BYPASSER_PATH": "/v2", + "EXT_BYPASSER_TIMEOUT": 120000, + } + ) + + grouped_calls = _group_save_calls(save_calls) + bypass_payloads = grouped_calls["cloudflare_bypass"] + + assert result == {"success": True, "message": "Onboarding complete!"} + assert any( + payload.get("USE_CF_BYPASS") is True and payload.get("USING_EXTERNAL_BYPASSER") is False + for payload in bypass_payloads + ) + assert all("EXT_BYPASSER_URL" not in payload for payload in bypass_payloads) + assert all("EXT_BYPASSER_PATH" not in payload for payload in bypass_payloads) + assert all("EXT_BYPASSER_TIMEOUT" not in payload for payload in bypass_payloads) diff --git a/tests/direct_download/test_search_queries.py b/tests/direct_download/test_search_queries.py index 72440e6a..8bd36527 100644 --- a/tests/direct_download/test_search_queries.py +++ b/tests/direct_download/test_search_queries.py @@ -9,6 +9,22 @@ def _browse_record(record_id: str, title: str) -> BrowseRecord: return BrowseRecord(id=record_id, title=title, source="direct_download") +def _enable_direct_download(monkeypatch): + import shelfmark.release_sources.direct_download as dd + + original_get = dd.config.get + + def _fake_get(key: str, default=None, user_id=None): + del user_id + if key == "DIRECT_DOWNLOAD_ENABLED": + return True + return original_get(key, default) + + monkeypatch.setattr(dd.config, "get", _fake_get) + monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True) + return dd + + class TestDirectDownloadSearchQueries: def test_uses_search_title_for_english_queries(self, monkeypatch): captured: list[str] = [] @@ -17,7 +33,7 @@ def fake_search_books(query: str, filters): captured.append(query) return [] - import shelfmark.release_sources.direct_download as dd + dd = _enable_direct_download(monkeypatch) monkeypatch.setattr(dd, "search_books", fake_search_books) @@ -59,7 +75,7 @@ def fake_search_books(query: str, filters): captured.append((query, filters.lang)) return records_by_query[query] - import shelfmark.release_sources.direct_download as dd + dd = _enable_direct_download(monkeypatch) monkeypatch.setattr(dd, "search_books", fake_search_books) @@ -103,7 +119,7 @@ def fake_search_books(query: str, filters): return [] return fallback_results[query] - import shelfmark.release_sources.direct_download as dd + dd = _enable_direct_download(monkeypatch) monkeypatch.setattr(dd, "search_books", fake_search_books) @@ -141,7 +157,7 @@ def fake_search_books(query: str, filters): return [] return [_browse_record("manual-1", "Manual result")] - import shelfmark.release_sources.direct_download as dd + dd = _enable_direct_download(monkeypatch) monkeypatch.setattr(dd, "search_books", fake_search_books) diff --git a/tests/direct_download/test_source_availability.py b/tests/direct_download/test_source_availability.py new file mode 100644 index 00000000..16c2dc79 --- /dev/null +++ b/tests/direct_download/test_source_availability.py @@ -0,0 +1,97 @@ +from types import SimpleNamespace + +import pytest + +from shelfmark.release_sources.direct_download import DirectDownloadSource, SearchUnavailableError + + +def _fake_config_get(values: dict[str, object]): + def _get(key: str, default=None, user_id=None): + del user_id + return values.get(key, default) + + return _get + + +def test_direct_download_source_is_unavailable_when_disabled(monkeypatch): + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr(dd.config, "get", _fake_config_get({"DIRECT_DOWNLOAD_ENABLED": False})) + monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True) + + source = DirectDownloadSource() + + assert source.is_available() is False + + with pytest.raises(SearchUnavailableError, match="Direct Download is disabled"): + source.search(SimpleNamespace(), SimpleNamespace()) + + +def test_direct_download_source_is_unavailable_without_aa_mirrors(monkeypatch): + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr(dd.config, "get", _fake_config_get({"DIRECT_DOWNLOAD_ENABLED": True})) + monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: False) + + source = DirectDownloadSource() + + assert source.is_available() is False + + with pytest.raises(SearchUnavailableError, match="not configured"): + source.get_record("md5-abc") + + +def test_direct_download_source_is_available_when_enabled_and_configured(monkeypatch): + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr(dd.config, "get", _fake_config_get({"DIRECT_DOWNLOAD_ENABLED": True})) + monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True) + + source = DirectDownloadSource() + + assert source.is_available() is True + + +def test_get_source_priority_disables_entries_without_required_mirrors(monkeypatch): + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr( + dd.config, + "get", + _fake_config_get( + { + "AA_DONATOR_KEY": "donator-key", + "FAST_SOURCES_DISPLAY": [ + {"id": "aa-fast", "enabled": True}, + {"id": "libgen", "enabled": True}, + ], + "SOURCE_PRIORITY": [ + {"id": "welib", "enabled": True}, + {"id": "zlib", "enabled": True}, + ], + } + ), + ) + monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: False) + monkeypatch.setattr("shelfmark.core.mirrors.has_libgen_mirror_configuration", lambda: True) + monkeypatch.setattr("shelfmark.core.mirrors.has_welib_mirror_configuration", lambda: False) + monkeypatch.setattr("shelfmark.core.mirrors.has_zlib_mirror_configuration", lambda: True) + + priority = {item["id"]: item["enabled"] for item in dd._get_source_priority()} + + assert priority["aa-fast"] is False + assert priority["libgen"] is True + assert priority["welib"] is False + assert priority["zlib"] is True + + +def test_is_configured_zlib_link_uses_configured_mirror_domains(monkeypatch): + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr( + "shelfmark.core.mirrors.get_zlib_cookie_domains", + lambda: {"custom-zlib.example"}, + ) + + assert dd._is_configured_zlib_link("https://custom-zlib.example/books/example") is True + assert dd._is_configured_zlib_link("https://other-zlib.example/books/example") is False diff --git a/tests/download/test_http_bypasser_fallbacks.py b/tests/download/test_http_bypasser_fallbacks.py index b13177dd..b2ea5c9a 100644 --- a/tests/download/test_http_bypasser_fallbacks.py +++ b/tests/download/test_http_bypasser_fallbacks.py @@ -69,6 +69,7 @@ def test_download_url_ignores_zlib_cookie_refresh_failure(monkeypatch): import shelfmark.download.http as http monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + monkeypatch.setattr(http, "_is_configured_zlib_host", lambda hostname: hostname == "z-lib.fm") monkeypatch.setattr(http, "get_proxies", lambda _url: {}) monkeypatch.setattr(http.time, "sleep", lambda _seconds: None) diff --git a/tests/download/test_network_dns_failover.py b/tests/download/test_network_dns_failover.py index aa63d8a6..80a781ea 100644 --- a/tests/download/test_network_dns_failover.py +++ b/tests/download/test_network_dns_failover.py @@ -81,6 +81,23 @@ def test_rotate_dns_provider_cycles_back_to_first_provider(monkeypatch): ] +def test_rotate_dns_and_reset_aa_keeps_aa_unconfigured_without_user_mirrors(monkeypatch): + network = _set_auto_dns_mode(monkeypatch) + events: list[tuple] = [] + + monkeypatch.setattr(network, "rotate_dns_provider", lambda: True) + monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto") + monkeypatch.setattr(network, "_aa_urls", []) + monkeypatch.setattr(network, "_aa_base_url", "https://legacy-aa.example") + monkeypatch.setattr(network, "_current_aa_url_index", 3) + monkeypatch.setattr(network, "_save_state", lambda **kwargs: events.append(("save", kwargs))) + + assert network.rotate_dns_and_reset_aa() is True + assert network._current_aa_url_index == 0 + assert network._aa_base_url == "" + assert events == [] + + def test_system_failover_getaddrinfo_retries_after_dns_switch(monkeypatch): network = _set_auto_dns_mode(monkeypatch) calls: list[tuple] = []