Mitigate mysterious Vitest + Webkit + Vue environment flakiness (#165)

* Fix: dedupe Vite dependency in yarn.lock

It’s unclear why both versions made it through the last round of updates. This is *possibly* related to some of the flakiness we’re seeing, but it’s safe either way.

Presumably `postcss` is also deduped as a transitive dep somewhere? Shrug emoji!

* Mitigation: web-forms/vue/vitest/webkit flakiness (1 of 2)

Adds a bit of delay in global setup if the test run is in webkit, and if there’s not already a `.vite/deps` cache (as a heuristic to differentiate first/subsequent test runs).

* Mitigation: web-forms/vue/vitest/webkit flakiness (2 of 2)

For `web-forms` only (as it seems to consistently be the package showing flakiness), and for `webkit` runs only (same reasoning), attempt test run a second time if first run fails.

.github/workflows/ci.yml

@@ -550,5 +550,11 @@ jobs:
       - if: ${{ matrix.target == 'Web' }}
         run: 'yarn playwright install ${{ matrix.browser }} --with-deps'
 
-      - if: ${{ matrix.target == 'Web' }}
+      # TODO: hopefully temporary! Attempt to mitigate flakiness in webkit by
+      # running twice when first run fails. At least in local testing, this has
+      # been near (if not at) 100% success.
+      - if: ${{ matrix.target == 'Web' && matrix.browser == 'webkit' }}
+        run: 'yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }} || yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }}'
+
+      - if: ${{ matrix.target == 'Web' && matrix.browser != 'webkit' }}
         run: 'yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }}'

packages/web-forms/tests/globalSetup/mitigate-webkit-flakiness.ts

@@ -0,0 +1,7 @@
+const sleep = async (milliseconds: number) => {
+	return new Promise((resolve) => setTimeout(resolve, milliseconds));
+};
+
+export default async function () {
+	await sleep(250);
+}

packages/web-forms/vitest.config.ts

@@ -1,5 +1,6 @@
 import type { CollectionValues } from '@getodk/common/types/collections/CollectionValues.ts';
 import type { VitestTestConfig } from '@getodk/common/types/vitest-config.ts';
+import { existsSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { configDefaults, defineConfig, mergeConfig } from 'vitest/config';
 import viteConfig from './vite.config';
@@ -29,6 +30,22 @@ const BROWSER_ENABLED = BROWSER_NAME != null;
 
 const TEST_ENVIRONMENT = BROWSER_ENABLED ? 'node' : 'jsdom';
 
+const globalSetup: string[] = [];
+
+/**
+ * @todo this is (hopefully!) temporary. Adds a delay when testing in
+ * `webkit`, to help mitigate flakiness that seems to be rooted in
+ * first-run timing issues (where "first" = "no Vite cache"; the issue was
+ * much more consistently reproducible in a state where
+ * `node_modules/.vite` is not present).
+ */
+const webkitFlakinessMitigations =
+	BROWSER_NAME === 'webkit' && !existsSync('./node_modules/.vite/deps');
+
+if (webkitFlakinessMitigations) {
+	globalSetup.push('./tests/globalSetup/mitigate-webkit-flakiness.ts');
+}
+
 export default mergeConfig(
 	viteConfig,
 	defineConfig({
@@ -37,12 +54,17 @@ export default mergeConfig(
 				enabled: BROWSER_ENABLED,
 				name: BROWSER_NAME!,
 				provider: 'playwright',
+				fileParallelism: false,
 				headless: true,
 				screenshotFailures: false,
 			},
 			environment: TEST_ENVIRONMENT,
 			exclude: [...configDefaults.exclude, 'e2e/**'],
 			root: fileURLToPath(new URL('./', import.meta.url)),
+
+			/** @see {@link webkitFlakinessMitigations} */
+			globalSetup,
+
 			// Suppress the console error log about parsing CSS stylesheet
 			// This is an open issue of jsdom
 			// see primefaces/primevue#4512 and jsdom/jsdom#2177

yarn.lock

@@ -5225,7 +5225,7 @@ postcss-selector-parser@^6.0.15:
     cssesc "^3.0.0"
     util-deprecate "^1.0.2"
 
-postcss@^8.1.10, postcss@^8.4.38:
+postcss@^8.1.10:
   version "8.4.38"
   resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.38.tgz#b387d533baf2054288e337066d81c6bee9db9e0e"
   integrity sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==
@@ -6363,18 +6363,7 @@ vite-plugin-solid@^2.10.2:
     solid-refresh "^0.6.3"
     vitefu "^0.2.5"
 
-vite@^5.0.0:
-  version "5.3.1"
-  resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.1.tgz#bb2ca6b5fd7483249d3e86b25026e27ba8a663e6"
-  integrity sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ==
-  dependencies:
-    esbuild "^0.21.3"
-    postcss "^8.4.38"
-    rollup "^4.13.0"
-  optionalDependencies:
-    fsevents "~2.3.3"
-
-vite@^5.3.3:
+vite@^5.0.0, vite@^5.3.3:
   version "5.3.3"
   resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.3.tgz#5265b1f0a825b3b6564c2d07524777c83e3c04c2"
   integrity sha512-NPQdeCU0Dv2z5fu+ULotpuq5yfCS1BzKUIPhNbP3YBfAMGJXbt2nS+sbTFu+qchaqWTD+H3JK++nRwr6XIcp6A==