diff --git a/.github/workflows/backstop.yml b/.github/workflows/backstop.yml new file mode 100644 index 000000000..7cd458224 --- /dev/null +++ b/.github/workflows/backstop.yml @@ -0,0 +1,62 @@ +name: Backstop test +on: + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + backstop-tests: + name: Visual regression tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 + with: + node-version: "20" + + - name: Set up Python + uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + npm install + + - name: Install xvfb + run: sudo apt-get install -y xvfb pandoc + + - name: Build documentation + run: | + pip install -e .[doc] + xvfb-run make -C doc html + + - name: Serve documentation + run: npx http-server -u -p 8000 ./doc/_build/html & + + - name: check if server is up + run: curl -I http://localhost:8000 || true + + - name: Run BackstopJS tests + run: | + npm run test + + - name: Upload Backstop report (if failed) + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: backstop-report + path: backstop_data/html_report + + diff --git a/.gitignore b/.gitignore index 98e1418fd..87686dc01 100644 --- a/.gitignore +++ b/.gitignore @@ -167,4 +167,8 @@ doc/source/examples/api/ # node modules and env node_modules/ .nodeenv/ -package-lock.json \ No newline at end of file +package-lock.json + +# BackstopJS outputs (do not commit) +tests/backstop_data/ +test-results/ \ No newline at end of file diff --git a/doc/changelog.d/804.added.md b/doc/changelog.d/804.added.md new file mode 100644 index 000000000..324628207 --- /dev/null +++ b/doc/changelog.d/804.added.md @@ -0,0 +1 @@ +Add backstop js test diff --git a/package.json b/package.json index 36b82cfee..a00c37aa8 100644 --- a/package.json +++ b/package.json @@ -4,16 +4,30 @@ "scripts": { "build:scss": "sass src/ansys_sphinx_theme/assets/styles/ansys-sphinx-theme.scss src/ansys_sphinx_theme/theme/ansys_sphinx_theme/static/styles/ansys-sphinx-theme.css", "build:postcss": "postcss src/ansys_sphinx_theme/theme/ansys_sphinx_theme/static/styles/ansys-sphinx-theme.css -o src/ansys_sphinx_theme/theme/ansys_sphinx_theme/static/styles/ansys-sphinx-theme.css", - "build": "npm run build:scss && npm run build:postcss" + "build": "npm run build:scss && npm run build:postcss", + "reference-tests": "cd tests && backstop reference && backstop test", + "playwright-test": "npx playwright install && cd tests && npx playwright test", + "test": "npm run reference-tests && npm run playwright-test" + }, + "scriptsDescription": { + "build:scss": "Compile SCSS to CSS", + "build:postcss": "Apply PostCSS transformations (autoprefixer, minify, etc.)", + "build": "Build complete CSS from SCSS and apply PostCSS", + "reference-tests": "Run BackstopJS visual regression tests (reference and test)", + "playwright-test": "Run Playwright end-to-end tests", + "test": "Run all tests (visual regression + e2e)" }, "devDependencies": { + "@playwright/test": "^1.56.0", "autoprefixer": "^10.4.21", + "backstopjs": "^6.3.25", "postcss": "^8.5.3", "postcss-cli": "^11.0.0", "postcss-import": "^16.1.0", "postcss-minify": "^1.1.0", "postcss-nested": "^7.0.2", - "sass": "^1.85.1" + "sass": "^1.85.1", + "wait-on": "^9.0.1" }, "dependencies": { "postcss-scss": "^4.0.9" diff --git a/tests/backstop.json b/tests/backstop.json new file mode 100644 index 000000000..37afeac3f --- /dev/null +++ b/tests/backstop.json @@ -0,0 +1,116 @@ +{ + "id": "ansys_sphinx_theme_backstop", + "viewports": [ + { + "label": "tablet", + "width": 1024, + "height": 768 + }, + { + "label": "desktop", + "width": 1920, + "height": 1080 + } + ], + "onBeforeScript": "puppet/onBefore.js", + "onReadyScript": "puppet/onReady.js", + "scenarios": [ + { + "label": "Ansys Sphinx Theme - Home Page", + "cookiePath": "backstop_data/engine_scripts/cookies.json", + "url": "http://localhost:8000", + "referenceUrl": "https://sphinxdocs.ansys.com/version/dev", + "readyEvent": "", + "readySelector": "", + "delay": 20, + "hideSelectors": [], + "removeSelectors": [], + "hoverSelector": "", + "clickSelector": "", + "postInteractionWait": 0, + "selectors": [], + "selectorExpansion": true, + "expect": 0, + "misMatchThreshold" : 0.1, + "requireSameDimensions": true + }, + { + "label": "Ansys Sphinx Theme - sphinx design example page", + "url": "http://localhost:8000/examples/sphinx-design.html", + "referenceUrl": "https://sphinxdocs.ansys.com/version/dev/examples/sphinx-design.html", + "readyEvent": "", + "readySelector": "", + "delay": 20, + "misMatchThreshold" : 0.1, + "requireSameDimensions": false + }, + { + "label": "Ansys Sphinx Theme - api reference example page", + "url": "http://localhost:8000/examples/api/examples/index.html", + "referenceUrl": "https://sphinxdocs.ansys.com/version/dev/examples/api/examples/index.html", + "readyEvent": "", + "readySelector": "", + "delay": 20, + "misMatchThreshold" : 0.2, + "requireSameDimensions": false + }, + { + "label": "Ansys Sphinx Theme - jupyter notebook example page", + "url": "http://localhost:8000/examples/nbsphinx/jupyter-notebook.html", + "referenceUrl": "https://sphinxdocs.ansys.com/version/dev/examples/nbsphinx/jupyter-notebook.html", + "readyEvent": "", + "readySelector": "", + "delay": 20, + "misMatchThreshold" : 0.1, + "requireSameDimensions": false + }, + { + "label": "Ansys Sphinx Theme - examples- table page", + "url": "http://localhost:8000/examples/table.html", + "referenceUrl": "https://sphinxdocs.ansys.com/version/dev/examples/table.html", + "readyEvent": "", + "readySelector": "", + "delay": 20, + "misMatchThreshold" : 0.1, + "requireSameDimensions": false + }, + { + "label": "Ansys Sphinx Theme - admonitions example page", + "url": "http://localhost:8000/examples/admonitions.html", + "referenceUrl": "https://sphinxdocs.ansys.com/version/dev/examples/admonitions.html", + "readyEvent": "", + "readySelector": "", + "delay": 20, + "misMatchThreshold" : 0.1, + "requireSameDimensions": false + }, + { + "label": "Ansys Sphinx Theme - gallery examples page", + "url": "http://localhost:8000/examples/gallery-examples/index.html", + "referenceUrl": "https://sphinxdocs.ansys.com/version/dev/examples/gallery-examples/index.html", + "delay": 20, + "misMatchThreshold" : 0.1, + "requireSameDimensions": false + } + ], + "paths": { + "bitmaps_reference": "backstop_data/bitmaps_reference", + "bitmaps_test": "backstop_data/bitmaps_test", + "engine_scripts": "backstop_data/engine_scripts", + "html_report": "backstop_data/html_report", + "ci_report": "backstop_data/ci_report" + }, + "ci": { + "format": "junit", + "testReportFileName": "myproject-xunit" + }, + "report": ["browser","CI"], + "engine": "puppet", + "engineOptions": { + "args": ["--no-sandbox"] + }, + "asyncCaptureLimit": 5, + "asyncCompareLimit": 50, + "debug": false, + "debugWindow": false +} diff --git a/tests/test_components.spec.js b/tests/test_components.spec.js new file mode 100644 index 000000000..fda704d25 --- /dev/null +++ b/tests/test_components.spec.js @@ -0,0 +1,136 @@ +import { test, expect } from "@playwright/test"; + +// Code blocks and copy button +test("Test code blocks and copy button", async ({ page }) => { + await page.goto("http://localhost:8000/user-guide/configuration.html"); + const codeBlock = await page.$(".highlight"); + if (!codeBlock) test.skip("No code block found on this page"); + expect(codeBlock).not.toBeNull(); + const copyBtn = await page.$("button.copybtn, .copy-button, .btn-copy"); + if (copyBtn) await copyBtn.click(); + expect(copyBtn).not.toBeNull(); +}); + +// Tabs (Sphinx-design) +test("Test sphinx tabs", async ({ page }) => { + await page.goto("http://localhost:8000/user-guide/configuration.html"); + const tabSet = await page.$(".sd-tab-set, .tab-set, .tab-content"); + expect(tabSet).not.toBeNull(); + const tabLabels = await page.$$( + ".sd-tab-label, .tab-label, .nav-tabs .nav-link", + ); + if (tabLabels.length > 1) { + await tabLabels[1].click(); + } +}); + +// Tables +test("Test tables", async ({ page }) => { + await page.goto("http://localhost:8000/examples/table.html"); + const table = await page.$(".pst-scrollable-table-container"); + expect(table).not.toBeNull(); + const headers = await table.$$("th"); + expect(headers.length).toBeGreaterThan(0); + const rows = await table.$$("tbody tr"); + expect(rows.length).toBeGreaterThan(0); +}); + +// Admonitions +test("Test admonitions", async ({ page }) => { + await page.goto("http://localhost:8000/examples/admonitions.html"); + const admonition = await page.$( + ".admonition, .note, .warning, .caution, .tip, .important", + ); + expect(admonition).not.toBeNull(); +}); + +// Sphinx-design components +test("Test example sphinx design page card", async ({ page }) => { + await page.goto("http://localhost:8000/examples/sphinx-design.html"); + const card = await page.$(".sd-card, .card, div.sd-card"); + expect(card).not.toBeNull(); +}); + +// Sphinx-design grid layout +test("Test example sphinx design page grid", async ({ page }) => { + await page.goto("http://localhost:8000/examples/sphinx-design.html"); + const grid = await page.$(".sd-row, .sd-grid, .row, .grid"); + expect(grid).not.toBeNull(); +}); + +// Sphinx-design badge +test("Test example sphinx design page badge", async ({ page }) => { + await page.goto("http://localhost:8000/examples/sphinx-design.html"); + // Sphinx-design badges are usually rendered as span.sd-badge or similar + const badge = await page.$(".sd-badge, span.sd-badge, button.sd-badge"); + expect(badge).not.toBeNull(); + // Only click if it's a button or has a click handler + if (badge) { + const tag = await badge.evaluate((el) => el.tagName.toLowerCase()); + if (tag === "button") { + await badge.click(); + } + // If not a button, just check it exists + } +}); + +// Sphinx-design clickable card +test("Test clickable card", async ({ page }) => { + await page.goto("http://localhost:8000/examples/sphinx-design.html"); + // Find the stretched link inside the card + const link = await page.$( + '.sd-card:has(.sd-card-title:has-text("Clickable Card (external)")) a.sd-stretched-link', + ); + expect(link).not.toBeNull(); + const href = await link.getAttribute("href"); + // Check if link opens in new tab + const target = await link.getAttribute("target"); + if (target === "_blank") { + const [newPage] = await Promise.all([ + page.context().waitForEvent("page"), + link.click(), + ]); + await newPage.waitForLoadState(); + expect(newPage.url()).toContain(href); + } else { + await Promise.all([page.waitForNavigation(), link.click()]); + expect(page.url()).toContain(href); + } +}); + +// Sphinx-design dropdown +test("Test sphinx-design dropdown", async ({ page }) => { + await page.goto("http://localhost:8000/examples/sphinx-design.html"); + // Target the
element with .sd-dropdown + const dropdown = await page.$("details.sd-dropdown"); + expect(dropdown).not.toBeNull(); + // Find the summary inside the dropdown + const summary = await dropdown.$("summary.sd-summary-title"); + expect(summary).not.toBeNull(); + // If the dropdown is open, close it first + let isOpen = await dropdown.getAttribute("open"); + if (isOpen) { + await summary.click(); + await page.waitForTimeout(200); + } + // Now open the dropdown + await summary.click(); + await page.waitForTimeout(200); + isOpen = await dropdown.getAttribute("open"); + expect(isOpen).not.toBeNull(); + // Check for dropdown content + const content = await dropdown.$(".sd-summary-content"); + expect(content).not.toBeNull(); + const text = await content.textContent(); + expect(text).toContain("Dropdown content"); +}); + +test("Test sidebar", async ({ page }) => { + await page.goto("http://localhost:8000/user-guide.html"); + const sidebar = await page.$( + '.bd-sidebar-primary, .bd-sidebar-secondary, .sidebar, nav[role="navigation"]', + ); + expect(sidebar).not.toBeNull(); + const links = await sidebar.$$("a"); + expect(links.length).toBeGreaterThan(0); +}); diff --git a/tests/test_index.spec.js b/tests/test_index.spec.js new file mode 100644 index 000000000..e51e71253 --- /dev/null +++ b/tests/test_index.spec.js @@ -0,0 +1,197 @@ +import { test, expect } from "@playwright/test"; + +test("Test homepage loading", async ({ page }) => { + await page.goto("http://localhost:8000/index.html"); + const header = await page.$("h1"); + expect(header).not.toBeNull(); + expect(await header.textContent()).toMatch(/Ansys Sphinx Theme|Welcome/i); +}); + +test("Test navbar components", async ({ page }) => { + await page.goto("http://localhost:8000"); + const navLinks = [ + { text: "Home", href: "#" }, + { text: "Getting started", href: "getting-started.html" }, + { text: "User guide", href: "user-guide.html" }, + { text: "Examples", href: "examples.html" }, + { text: "Release notes", href: "changelog.html" }, + ]; + for (const { text, href } of navLinks) { + const link = await page.$(`nav a.nav-link:has-text("${text}")`); + expect(link).not.toBeNull(); + if (link && href) { + const actualHref = await link.getAttribute("href"); + expect(actualHref).toContain(href); + } + } + const moreBtn = await page.$('button.dropdown-toggle:has-text("More")'); + expect(moreBtn).not.toBeNull(); + await moreBtn.click(); + const dropdownLinks = [ + { text: "Contribute", href: "contribute.html" }, + { text: "API reference", href: "examples/api/index.html" }, + ]; + for (const { text, href } of dropdownLinks) { + const link = await page.$(`.dropdown-menu a:has-text("${text}")`); + expect(link).not.toBeNull(); + if (link && href) { + const actualHref = await link.getAttribute("href"); + expect(actualHref).toContain(href); + } + } +}); + +test("Test navbar end components", async ({ page }) => { + await page.goto("http://localhost:8000"); + const searchBar = await page.$( + '.search-bar input[type="search"], .search-bar input[placeholder*="Search" i]', + ); + expect(searchBar).not.toBeNull(); + + // Check version switcher + const versionSwitcher = await page.$( + ".version-switcher__button, .version-switcher__container button", + ); + expect(versionSwitcher).not.toBeNull(); + + // Check theme switcher + const themeSwitcher = await page.$( + 'button.theme-switch-button, button[aria-label*="Color mode" i]', + ); + expect(themeSwitcher).not.toBeNull(); + + // Check GitHub icon link + const githubLink = await page.$( + 'a[href*="github.com/ansys/ansys-sphinx-theme"]', + ); + expect(githubLink).not.toBeNull(); +}); + +test("Test navigation bar", async ({ page }) => { + await page.goto("http://localhost:8000"); + const nav = await page.$('nav, [role="navigation"]'); + expect(nav).not.toBeNull(); + // Try to find Home link in nav + const homeLink = await page.$( + 'nav a:has-text("Home"), [role="navigation"] a:has-text("Home")', + ); + if (!homeLink) test.skip("Home link not found in navigation bar"); + expect(homeLink).not.toBeNull(); +}); + +test("Test sidebar", async ({ page }) => { + await page.goto("http://localhost:8000"); + const sidebar = await page.$( + '.bd-sidebar-primary, .bd-sidebar-secondary, .sidebar, nav[role="navigation"]', + ); + expect(sidebar).not.toBeNull(); + const sidebarLinks = await sidebar.$$("a"); + expect(sidebarLinks.length).toBeGreaterThan(0); +}); + +test("Test version switcher", async ({ page }) => { + await page.goto("http://localhost:8000"); + const versionSwitcher = await page.$( + '.version-switcher, [id*="version-switcher"], [class*="version-switcher"]', + ); + expect(versionSwitcher).not.toBeNull(); +}); + +test("Test version switcher dropdown", async ({ page }) => { + await page.goto("http://localhost:8000"); + const versionSwitcher = await page.$( + '.version-switcher, [id*="version-switcher"], [class*="version-switcher"]', + ); + expect(versionSwitcher).not.toBeNull(); + await versionSwitcher.click(); + const dropdown = await page.$( + '.version-switcher__menu, [id*="pst-version-switcher-list-2"], [class*="version-switcher__menu"]', + ); + expect(dropdown).not.toBeNull(); +}); + +test("Test theme switcher", async ({ page }) => { + await page.goto("http://localhost:8000"); + // Use the actual button class from the provided HTML + const themeSwitcher = await page.$( + 'button.theme-switch-button[aria-label="Color mode"]', + ); + if (!themeSwitcher) test.skip("Theme switcher not found"); + expect(themeSwitcher).not.toBeNull(); + // Find the currently active mode (the visible svg or the one with aria-current or similar) + const getActiveMode = async () => { + // Try to get the mode from the html class or data attribute if available + const htmlMode = await page.evaluate(() => { + if (document.documentElement.dataset.theme) + return document.documentElement.dataset.theme; + if (document.body.dataset.theme) return document.body.dataset.theme; + // Fallback: find the first visible svg with data-mode + const svgs = Array.from( + document.querySelectorAll( + "button.theme-switch-button svg.theme-switch[data-mode]", + ), + ); + for (const svg of svgs) { + const style = window.getComputedStyle(svg); + if ( + style.display !== "none" && + style.visibility !== "hidden" && + style.opacity !== "0" + ) { + return svg.getAttribute("data-mode"); + } + } + return svgs.length ? svgs[0].getAttribute("data-mode") : null; + }); + return htmlMode; + }; + const currentMode = await getActiveMode(); + await themeSwitcher.click(); + console.log(`Switched theme from ${currentMode}`); + await page.waitForTimeout(500); + const newMode = await getActiveMode(); + console.log(`New theme mode is ${newMode}`); + expect(newMode).not.toBe(currentMode); +}); + +test("Test breadcrumbs", async ({ page }) => { + await page.goto("http://localhost:8000/user-guide/configuration.html"); + const breadcrumbs = await page.$( + '.bd-breadcrumb, nav[aria-label="Breadcrumb"]', + ); + if (!breadcrumbs) test.skip("Breadcrumbs not found"); + expect(breadcrumbs).not.toBeNull(); +}); + +test("Test logo", async ({ page }) => { + await page.goto("http://localhost:8000"); + const logo = await page.$('img[alt*="logo" i], .navbar-brand img, .logo'); + expect(logo).not.toBeNull(); + const logoLink = await logo.evaluate((el) => + el.closest("a")?.getAttribute("href"), + ); + expect(logoLink).toMatch(/\/?(index\.html)?$/); +}); + +test("Test edit this page button", async ({ page }) => { + await page.goto("http://localhost:8000/user-guide/configuration.html"); + const editBtn = await page.$( + 'tocsection.editthispage, a:has-text("Edit on GitHub")', + ); + if (!editBtn) test.skip("Edit this page button not found"); + expect(editBtn).not.toBeNull(); +}); + +test("Test footer links", async ({ page }) => { + await page.goto("http://localhost:8000"); + const footer = await page.$(".bd-footer"); + expect(footer).not.toBeNull(); + const Footercopyright = await footer.$(".copyright"); + const footerLinks = await footer.$$("a"); + expect(footerLinks.length).toBeGreaterThan(0); + expect(Footercopyright).not.toBeNull(); + const themeVersion = await footer.$(".theme-version"); + expect(themeVersion).not.toBeNull(); + const versionText = await themeVersion.textContent(); + expect(versionText).not.toBeNull(); +}); diff --git a/tests/test_search.spec.js b/tests/test_search.spec.js new file mode 100644 index 000000000..4e1a8ece4 --- /dev/null +++ b/tests/test_search.spec.js @@ -0,0 +1,23 @@ +import { test, expect } from "@playwright/test"; + +test("Test search functionality", async ({ page }) => { + await page.goto("http://localhost:8000"); + const searchBtn = await page.$( + 'button[aria-label*="search" i], .search-bar .fa-magnifying-glass, .search-button, [data-bs-toggle="search"]', + ); + if (searchBtn) { + await searchBtn.click(); + } else { + await page.keyboard.press( + process.platform === "darwin" ? "Meta+K" : "Control+K", + ); + } + console.log("Tried to open search"); + const searchBar = await page.$( + '.search-bar input[type="search"], .search-bar input[placeholder*="Search" i]', + ); + await searchBar.fill("install"); + await page.waitForTimeout(1500); + const results = await page.$$(".search-bar"); + expect(results.length).toBeGreaterThan(0); +}); diff --git a/tox.ini b/tox.ini index 3524367b7..a9af1b945 100644 --- a/tox.ini +++ b/tox.ini @@ -5,18 +5,51 @@ envlist = doc-style doc-{links,html,pdf,clean,serve} dist + tests skip_missing_interpreters = true isolated_build = true isolated_build_env = build [testenv] -description = Checks for project unit tests and coverage (if desired) -basepython = - {code-style,doc-style,doc-links,doc-html}: python3 +description = Run BackstopJS visual regression tests +basepython = python3 +extras = doc setenv = - PYTHONUNBUFFERED = yes - DOCUMENTATION_CNAME = sphinxdocs.ansys.com -passenv = * + SOURCE_DIR = doc/source + BUILD_DIR = doc/_build + PORT = 8000 + ; BUILDER_OPTS = --color -v -j auto -W --keep-going + BUILDER = html +deps = + nodeenv +commands_pre = + # Ensure node environment and dependencies are ready + nodeenv -p +commands = + # 1. Build Sphinx docs + sphinx-build -d "{toxworkdir}/doc_doctree" {env:SOURCE_DIR} "{toxinidir}/{env:BUILD_DIR}/{env:BUILDER}" -b {env:BUILDER} + + # 2. Start HTTP server to serve built docs + python -c "import subprocess, time, os; \ + build_dir = r'{toxinidir}/{env:BUILD_DIR}/{env:BUILDER}'; \ + proc = subprocess.Popen(['python', '-m', 'http.server', '{env:PORT}', '--directory', build_dir], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); \ + open('server.pid','w').write(str(proc.pid)); \ + time.sleep(3); \ + print('Server started at', build_dir, 'PID', proc.pid)" + + # 3. Run BackstopJS tests + npm install + npm run test +commands_post = + # 4. Stop HTTP server + python -c "import os, signal, time; \ + pidfile='server.pid'; \ + pid = open(pidfile).read().strip() if os.path.exists(pidfile) else None; \ + (os.kill(int(pid), signal.SIGTERM) if pid else None); \ + time.sleep(1); \ + os.remove(pidfile) if os.path.exists(pidfile) else None; \ + print('Server stopped')" + [testenv:code-style] description = check project code style @@ -62,3 +95,4 @@ deps = build commands = python -m build {toxinidir} +