fix: Add weekly baseline testing

Author: devknoll

.github/workflows/build_test.yml

@@ -43,16 +43,12 @@ jobs:
           ./wpt/wpt manifest
           ./wpt/wpt serve --inject-script=${{ github.workspace }}/dist/container-query-polyfill.modern.js &
           npm test
-
-      - name: Open Pull Request
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          node --loader ts-node/esm ./tests/diff.ts
-          git config user.name github-actions[bot]
-          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
-          git checkout -b update-wpt-baseline-${{ github.job_id }}
-          git add ./tests/baseline.json
-          git commit -m "Update Web Platform Test baseline"
-          git push -u origin HEAD
-          gh pr create --title "[Automated] Update Web Platform Tests" --body-file ./tests/pr.txt
+          ret=$(node --loader ts-node/esm ./tests/diff.ts)
+          npm run prettier:fix
+          echo -e "\n# Updated Baseline\n\n```json\n" >> ./tests/pr.txt
+          cat ./tests/baseline.json >> ./tests/pr.txt
+          echo -e "\n```" >> ./tests/pr.txt
+          cat ./tests/pr.txt >> $GITHUB_STEP_SUMMARY
+          if [ $ret == "changed" ]; then
+            exit 1
+          fi

.github/workflows/release-please.yml

@@ -3,7 +3,7 @@ name: release-please
 on:
   push:
     branches:
-      - main
+      - x-rework-test-results
 
 jobs:
   release-please:
@@ -22,6 +22,8 @@ jobs:
         if: ${{ steps.release.outputs.release_created }}
       - run: npm install
         if: ${{ steps.release.outputs.release_created }}
+      - run: npm run build
+        if: ${{ steps.release.outputs.release_created }}
 
       - uses: actions/setup-node@v3
         with:
@@ -31,4 +33,4 @@ jobs:
       - run: npm publish
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
-        if: ${{ steps.release.outputs.release_created }}
+        if: ${{ steps.release.outputs.release_created }}

.github/workflows/update_baseline.yml

@@ -0,0 +1,63 @@
+name: Update WPT Baseline
+
+on:
+  workflow_dispatch:
+    branches: [main]
+  schedule:
+    - cron: '0 0 * * 1'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
+      BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
+      WPT_MANIFEST: ${{ github.workspace }}/wpt/MANIFEST.json
+      SCHEDULED_BASELINE_DIFF: true
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
+      - uses: actions/checkout@v3
+        with:
+          repository: devknoll/wpt
+          path: wpt
+          ref: x-polyfill-all-tests
+
+      - name: Build
+        run: |
+          npm install
+          npm run build:wpt
+
+      - name: Setup WPT
+        run: |
+          cd wpt
+          pip install virtualenv
+          ./wpt make-hosts-file | sudo tee -a /etc/hosts
+      - name: Run Tests
+        run: |
+          npm run serve &
+          ./wpt/wpt manifest
+          ./wpt/wpt serve --inject-script=${{ github.workspace }}/dist/container-query-polyfill.modern.js &
+          npm test
+
+      - name: Open Pull Request
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          ret=(node --loader ts-node/esm ./tests/diff.ts)
+          if [ $ret == "unchanged" ]; then
+            exit 0
+          fi
+          npm run prettier:fix
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+          git checkout -b update-wpt-baseline-$(date +"%Y-%m-%d")
+          git add ./tests/baseline.json
+          git commit -m "Update Web Platform Test baseline"
+          git push -u origin HEAD
+          gh pr create --title "[Automated] Update Web Platform Tests" --body-file ./tests/pr.txt --label "update-baseline"

.prettierignore

@@ -1 +1,2 @@
-dist/
+dist/
+wpt/

package.json

@@ -1,8 +1,11 @@
 {
   "name": "container-query-polyfill",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "",
-  "repository": "github:GoogleChromeLabs/container-query-polyfill",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/GoogleChromeLabs/container-query-polyfill.git"
+  },
   "type": "module",
   "module": "dist/container-query-polyfill.modern.js",
   "unpkg": "dist/container-query-polyfill.modern.js",

tests/diff.ts

@@ -187,6 +187,7 @@ const allTestNames = new Set(
 );
 const summaryLines: string[] = [];
 
+let hasRegressions = false;
 for (const test of allTestNames) {
   const currentSubtests = currentTestMap[test];
   const previousSubtests = previousTestMap[test];
@@ -224,7 +225,7 @@ for (const test of allTestNames) {
       let summary: string | null = null;
 
       if (!after) {
-        summary = '(Removed)';
+        summary = '(removed)';
       } else {
         const diffLines: string[] = [];
         for (const browser of allBrowsers) {
@@ -235,6 +236,10 @@ for (const test of allTestNames) {
             continue;
           }
 
+          if (wasPassing && !isPassing) {
+            hasRegressions = true;
+          }
+
           const prefix = isPassing ? '+' : '-';
           diffLines.push(`${prefix} ${browser}`);
         }
@@ -282,32 +287,37 @@ ${testSummary.join('\n')}
   }
 }
 
-if (summaryLines.length > 0) {
-  const commentLines: string[] = [];
+const commentLines: string[] = [];
+if (process.env.SCHEDULED_BASELINE_DIFF) {
   commentLines.push(
-    'The [Web Platform Test](https://web-platform-tests.org/) results have changed from the expected baseline. The baseline may be updated by merging this pull request.'
+    'The [Web Platform Test](https://web-platform-tests.org/) results have changed from the expected baseline. You may accept these changes by merging this pull request.'
   );
+}
 
-  if (missingBrowsers.size > 0) {
-    commentLines.push(
-      '',
-      `
-  > **Warning**
-  > The test run was missing data for the following browsers:
-  >
-  ${Array.from(missingBrowsers)
-    .map(browser => `>   * ${browser}`)
-    .join('\n')}
-  >
-  > Therefore I couldn't confirm the expected tests are still passing.
-      `
-    );
-  }
-
-  commentLines.push('', '# Test Results', ...summaryLines);
-  await writeFile(getPathForFile('pr.txt'), commentLines.join('\n'));
-  await writeFile(
-    getPathForFile('baseline.json'),
-    JSON.stringify(currentTestMap)
+if (missingBrowsers.size > 0) {
+  commentLines.push(
+    '',
+    `
+> **Warning**
+> The test run was missing data for the following browsers:
+>
+${Array.from(missingBrowsers)
+  .map(browser => `>   * ${browser}`)
+  .join('\n')}
+>
+> The test results for these browsers couldn't be confirmed.
+    `
   );
 }
+
+const results =
+  summaryLines.length > 0 ? summaryLines : ['No changes detected.'];
+commentLines.push('', '# Test Results', ...results);
+
+await writeFile(getPathForFile('pr.txt'), commentLines.join('\n'));
+await writeFile(
+  getPathForFile('baseline.json'),
+  JSON.stringify(currentTestMap)
+);
+
+process.stdout.write(hasRegressions ? 'changed' : 'unchanged');

tests/wpt.ts

@@ -54,50 +54,6 @@ export interface BrowserDefinition {
   versions: BrowserVersion[];
 }
 
-const TEST_FILTERS: Array<RegExp> = [
-  /-serialization.html$/,
-  /-computed.html$/,
-  /calc-evaluation.html$/,
-  /auto-scrollbars.html$/,
-
-  /container-inheritance.html$/,
-  /container-for-shadow-dom.html$/,
-  /container-units-shadow.html$/,
-  /container-longhand-animation-type.html$/,
-  /container-name-invalidation.html$/,
-  /container-name-parsing.html$/,
-  /container-parsing.html$/,
-  /container-type-containment.html$/,
-  /container-type-layout-invalidation.html$/,
-  /container-type-parsing.html$/,
-  /container-units-basic.html$/,
-  /container-units-in-at-container-fallback.html$/,
-  /container-units-invalidation.html$/,
-  /container-units-media-queries.html$/,
-  /container-units-selection.html$/,
-  /container-units-small-viewport-fallback.html$/,
-  /container-units-svglength.html$/,
-  /container-units-typed-om.html$/,
-  /deep-nested-inline-size-containers.html$/,
-  /idlharness.html$/,
-  /iframe-in-container-invalidation.html$/,
-  /iframe-invalidation.html$/,
-  /percentage-padding-orthogonal.html$/,
-  /viewport-units-dynamic.html$/,
-  /viewport-units.html$/,
-];
-
-const SUBTEST_FILTERS: Array<RegExp> = [
-  /calc\(.*\)/,
-  /max\(.*\)/,
-  /style\(.*\)/,
-  /#container width 399px after padding is applied. #second is removed from the rendering/,
-  /ex units/,
-  /ch units/,
-  /ex relative/,
-  /ch relative/,
-];
-
 const CHROME_DEFINITION: BrowserDefinition = {
   name: 'Chrome',
   logo: 'https://unpkg.com/@browser-logos/chrome@2.0.0/chrome.svg',
@@ -292,9 +248,9 @@ async function getTests(manifestPath: string): Promise<TestSuite> {
   });
 
   return {
-    js: Object.keys(htmlTests)
-      .filter(name => !TEST_FILTERS.some(filter => filter.test(name)))
-      .map(name => `http://web-platform.test:8000/${prefix}/${name}`),
+    js: Object.keys(htmlTests).map(
+      name => `http://web-platform.test:8000/${prefix}/${name}`
+    ),
     iframe,
   };
 }
@@ -419,9 +375,6 @@ async function main() {
             const testURL = new URL(test[0]);
             if (Array.isArray(test) && Array.isArray(test[1].tests)) {
               for (const subtest of test[1].tests) {
-                if (SUBTEST_FILTERS.some(filter => filter.test(subtest.name))) {
-                  continue;
-                }
                 const result = {test: testURL.pathname, subtest: subtest.name};
                 const destination =
                   subtest.status === subtest.PASS ? passed : failed;
@@ -450,7 +403,6 @@ async function main() {
     join(dirname(fileURLToPath(import.meta.url)), 'results.json'),
     output
   );
-  console.warn(output);
 }
 
 try {