refactor: simplify bazel saucelabs targets using karma pre-test wrapp…

…er and shared saucelabs connection between tests (angular#34769) * Added a /tools/saucelabs/sauce-service.sh script that manages the sauce-connect as a service which is used by the karma-saucelabs.js wrapper to start the service. * Added /tools/saucelabs/README.md that covers the details of SauceLabs karma testing with Bazel. PR Close angular#34769
mikoloism · Jan 28, 2020 · dff4e1e · dff4e1e
1 parent a5a5981
commit dff4e1e
Show file tree

Hide file tree

Showing 13 changed files with 672 additions and 374 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -112,6 +112,16 @@ build:remote --project_id=internal-200822
 build:remote --remote_cache=remotebuildexecution.googleapis.com
 build:remote --remote_executor=remotebuildexecution.googleapis.com
 
+##################################
+# Saucelabs tests settings       #
+# Turn on these settings with    #
+#  --config=saucelabs            #
+##################################
+
+# For saucelabs tests we don't want to enable flaky test attempts. Karma has its own integrated
+# retry mechanism and we do not want to retry unnecessarily if Karma already tried multiple times.
+test:saucelabs --flaky_test_attempts=1
+
 ###############################
 # NodeJS rules settings
 # These settings are required for rules_nodejs

diff --git a/.circleci/bazel.common.rc b/.circleci/bazel.common.rc
@@ -13,7 +13,3 @@ test --flaky_test_attempts=2
 
 # More details on failures
 build --verbose_failures=true
-
-# For saucelabs tests we don't want to enable flaky test attempts. Karma has its own integrated
-# retry mechanism and we do not want to retry unnecessarily if Karma already tried multiple times.
-test:saucelabs --flaky_test_attempts=1
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -314,19 +314,15 @@ jobs:
       - custom_attach_workspace
       - init_environment
       - run:
-          name: Run Bazel tests in saucelabs
-          # All web tests are contained within a single //:saucelabs_unit_tests_poc target
-          # for Saucelabs as running each set of tests as a separate target will attempt to acquire
-          # too many browsers on Saucelabs (7 per target currently) and some tests will always
-          # fail to acquire browsers. For example:
-          # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing.
-          # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s
-          command:  |
-            ./scripts/saucelabs/run-bazel-via-tunnel.sh \
-              --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \
-              --username $SAUCE_USERNAME \
-              --key $(echo $SAUCE_ACCESS_KEY | rev) \
-              -- yarn bazel test //:saucelabs_unit_tests_poc --config=saucelabs
+          name: Preparing environment for running tests on Saucelabs.
+          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - run:
+          name: Run Bazel tests on Saucelabs
+          # See /tools/saucelabs/README.md for more info
+          command: |
+            yarn bazel run //tools/saucelabs:sauce_service_setup
+            yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
+            yarn bazel run //tools/saucelabs:sauce_service_stop
           no_output_timeout: 20m
       - notify_webhook_on_fail:
           webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
@@ -341,15 +337,16 @@ jobs:
     steps:
       - custom_attach_workspace
       - init_environment
+      - run:
+          name: Preparing environment for running tests on Saucelabs.
+          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
       - run:
           name: Run Bazel tests on Saucelabs
-          # Runs the //:saucelabs_tests target with Saucelabs and Ivy.
-          command:  |
-            ./scripts/saucelabs/run-bazel-via-tunnel.sh \
-              --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \
-              --username $SAUCE_USERNAME \
-              --key $(echo $SAUCE_ACCESS_KEY | rev) \
-              -- yarn bazel test //:saucelabs_unit_tests --config=ivy --config=saucelabs
+          # See /tools/saucelabs/README.md for more info
+          command: |
+            yarn bazel run //tools/saucelabs:sauce_service_setup
+            yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
+            yarn bazel run //tools/saucelabs:sauce_service_stop
           no_output_timeout: 20m
 
   test_aio:
@@ -665,17 +662,20 @@ jobs:
             setPublicVar KARMA_JS_BROWSERS $(node -e 'console.log(require("./browser-providers.conf").sauceAliases.CI_REQUIRED.join(","))')
             setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
       - run:
-          name: Starting Saucelabs tunnel
-          command: ./scripts/saucelabs/start-tunnel.sh
+          name: Starting Saucelabs tunnel service
+          command: ./tools/saucelabs/sauce-service.sh run
           background: true
       - run: yarn tsc -p packages
       - run: yarn tsc -p modules
       - run: yarn bazel build //packages/zone.js:npm_package
-        # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
-        # too early without Saucelabs not being ready.
-      - run: ./scripts/saucelabs/wait-for-tunnel.sh
+      - run:
+          # Waiting on ready ensures that we don't run tests too early without Saucelabs not being ready.
+          name: Waiting for Saucelabs tunnel to connect
+          command: ./tools/saucelabs/sauce-service.sh ready-wait
       - run: yarn karma start ./karma-js.conf.js --single-run --browsers=${KARMA_JS_BROWSERS}
-      - run: ./scripts/saucelabs/stop-tunnel.sh
+      - run:
+          name: Stop Saucelabs tunnel service
+          command: ./tools/saucelabs/sauce-service.sh stop
 
   # Job that runs all unit tests of the `angular/components` repository.
   components-repo-unit-tests:

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -47,68 +47,75 @@ filegroup(
     ],
 )
 
-# To run a karma_web_test target manually, run the "./scripts/saucelabs/run-bazel-via-tunnel.sh"
-# script. Note: If you are on MacOS or Windows, you need to manually start the Saucelabs tunnel
-# because the script only supports the linux Saucelabs tunnel binary. We combine all tests into
-# a single "karma_web_test" target because running them as separate targets in parallel can result
-# in to too many browsers being acquired at the same time. This will then result in tests being
-# flaky. This target runs in CI with Saucelabs and Ivy.
+# To run manually:
+# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
+# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
+# environment variables and run:
+# ```
+# yarn bazel run //tools/saucelabs:sauce_service_setup
+# yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
+# ```
+# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
 karma_web_test(
     name = "saucelabs_unit_tests",
     # Default timeout is moderate (5min). This causes the test to be terminated while
     # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
     # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
     # Bazel's long timeout (15min). This ensures that Karma can shut down properly.
     timeout = "long",
+    karma = "//tools/saucelabs:karma-saucelabs",
     tags = [
-        "local",
         "manual",
+        "no-remote-exec",
         "saucelabs",
     ],
     deps = [
         "//packages/core/test/acceptance:acceptance_lib",
     ],
 )
 
-karma_web_test(
-    # This target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
+SAUCE_TEST_SUITE_TARGETS = [
+    "packages/common/http/test:test_lib",
+    "packages/common/http/testing/test:test_lib",
+    "packages/common/test:test_lib",
+    "packages/core/test:test_lib",
+    "packages/forms/test:test_lib",
+    "packages/http/test:test_lib",
+]
+
+[
+    # These target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
     # subset of the legacy saucelabs tests.
-    name = "saucelabs_unit_tests_poc",
-    # Default timeout is moderate (5min). This causes the test to be terminated while
-    # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
-    # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
-    # Bazel's long timeout (15min). This ensures that Karma can shut down properly.
-    timeout = "long",
-    tags = [
-        "local",
-        "manual",
-        "saucelabs",
-    ],
-    deps = [
-        # We combine all tests into a single karma_web_test target
-        # as running them as separate targets in parallel leads to too many
-        # browsers being acquired at once in SauceLabs and the tests flake out
-        # TODO: this is an example subset of tests below, add all remaining angular tests
-        "//packages/common/http/test:test_lib",
-        "//packages/common/http/testing/test:test_lib",
-        "//packages/common/test:test_lib",
-        "//packages/core/test:test_lib",
-        "//packages/forms/test:test_lib",
-        "//packages/http/test:test_lib",
-        "//packages/zone.js/test:karma_jasmine_test_ci",
-        # "//packages/router/test:test_lib",
-        # //packages/router/test:test_lib fails with:
-        # IE 11.0.0 (Windows 8.1.0.0) bootstrap should restore the scrolling position FAILED
-        # 	Expected undefined to equal 5000.
-        # 	   at stack (eval code:2338:11)
-        # 	   at buildExpectationResult (eval code:2305:5)
-        # 	   at expectationResultFactory (eval code:858:11)
-        # 	   at Spec.prototype.addExpectationResult (eval code:487:5)
-        # 	   at addExpectationResult (eval code:802:9)
-        # 	   at Anonymous function (eval code:2252:7)
-        # 	   at Anonymous function (eval code:339:25)
-        # 	   at step (eval code:133:17)
-        # 	   at Anonymous function (eval code:114:50)
-        # 	   at fulfilled (eval code:104:47)
-    ],
+    karma_web_test(
+        name = "saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_"),
+        # Default timeout is moderate (5min). This causes the test to be terminated while
+        # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
+        # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
+        # Bazel's long timeout (15min). This ensures that Karma can shut down properly.
+        timeout = "long",
+        karma = "//tools/saucelabs:karma-saucelabs",
+        tags = [
+            "exclusive",
+            "manual",
+            "no-remote-exec",
+            "saucelabs",
+        ],
+        deps = ["//%s" % test],
+    )
+    for test in SAUCE_TEST_SUITE_TARGETS
+]
+
+# To run manually:
+# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
+# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
+# environment variables and run:
+# ```
+# yarn bazel run //tools/saucelabs:sauce_service_setup
+# yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
+# ```
+# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
+test_suite(
+    name = "saucelabs_unit_tests_poc_suite",
+    tags = ["manual"],
+    tests = ["//:saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_") for test in SAUCE_TEST_SUITE_TARGETS],
 )
diff --git a/karma-js.conf.js b/karma-js.conf.js
@@ -152,9 +152,6 @@ module.exports = function(config) {
     set: () => {},
   });
 
-  // When running under Bazel with karma_web_test, SAUCE_TUNNEL_IDENTIFIER and KARMA_WEB_TEST_MODE
-  // will only be available if they are part of the Bazel action environment. More details in the
-  // "scripts/saucelabs/run-bazel-via-tunnel.sh" script.
   if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
     console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);