Skip to content

Commit

Permalink
chore: Migrate to rules_bazel_integration_test (#1598)
Browse files Browse the repository at this point in the history
This patch deletes the old copy of `bazel_integration_test` that we
had vendored into the repo. It's replaced with the maintained
`rules_bazel_integration_test`.

Summary of changes:
* Increases minimum tested bazel version to 6.4.0.
  rules_bazel_integration_test depends on some features in 6.4
* Moves several CI jobs to be BIT tests. This free ups
  about 10 CI slots.
* Runs these tests under a separate CI job. This is so that feedback
  from the regular test jobs is faster.

Notes about these bazel-in-bazel integration tests:
* The tests are very heavy and easily overwhelm a system. Unfortunately,
  there doesn't appear to be a way to cap their parallelism; only
  disable it entirely using the `exclusive` tag. Some light
  testing shows there is some speedup to be gained on CI
  if we can, in the future, limit them to 2 or 4 concurrent
  executions.
* A special version named "self" is created that re-uses
  whatever the outer Bazel program is. This is mainly so that
  Bazel's "at head" testing pipelines (the one that runs tests
  with Bazel built from head) are able to affect the integration tests.
  It also comes in handy when locally testing a custom Bazel build.
* The globbing of child workspace files can be somewhat prone to
  following `bazel-*` symlinks, so its important the `.bazelignore`
  and deleted packages configs are up-to-date. Otherwise the
  globbing can turn into 30,000+ files and consume a system-freezing
  level of memory and CPU.

Fixes #1209

---------

Co-authored-by: Richard Levasseur <[email protected]>
  • Loading branch information
philsc and rickeylev authored Jan 6, 2024
1 parent 4fee7ef commit 8aaaf93
Show file tree
Hide file tree
Showing 16 changed files with 295 additions and 60 deletions.
59 changes: 21 additions & 38 deletions .bazelci/presubmit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ buildifier:
.minimum_supported_version: &minimum_supported_version
# For testing minimum supported version.
# NOTE: Keep in sync with //:version.bzl
bazel: 6.2.0
bazel: 6.4.0
skip_in_bazel_downstream_pipeline: "Bazel 6 required"
.reusable_config: &reusable_config
build_targets:
Expand All @@ -32,6 +32,7 @@ buildifier:
- "@rules_python//examples/wheel/..."
build_flags:
- "--keep_going"
- "--build_tag_filters=-integration-test"
test_targets:
- "--"
- "..."
Expand All @@ -42,6 +43,18 @@ buildifier:
- "--noenable_bzlmod"
build_flags:
- "--noenable_bzlmod"
.common_bazelinbazel_config: &common_bazelinbazel_config
build_flags:
- "--build_tag_filters=integration-test"
test_flags:
- "--test_tag_filters=integration-test"
- "--jobs=2"
# The integration tests are so expensive that only a few can be run concurrently
# without harming overall reliability and runtime.
- "--local_test_jobs=2"
build_targets: ["..."]
test_targets: ["..."]

.reusable_build_test_all: &reusable_build_test_all
build_targets: ["..."]
test_targets: ["..."]
Expand Down Expand Up @@ -420,19 +433,14 @@ tasks:
working_directory: examples/pip_repository_annotations
platform: windows

integration_test_compile_pip_requirements_ubuntu_min_workspace:
<<: *minimum_supported_version
<<: *reusable_build_test_all
<<: *common_workspace_flags
name: "compile_pip_requirements: Ubuntu using minimum Bazel"
working_directory: tests/integration/compile_pip_requirements
platform: ubuntu2004
integration_test_compile_pip_requirements_ubuntu_min_bzlmod:
<<: *minimum_supported_version
<<: *reusable_build_test_all
name: "compile_pip_requirements: Ubuntu using minimum Bazel"
working_directory: tests/integration/compile_pip_requirements
integration_test_bazelinbazel_ubuntu:
<<: *common_bazelinbazel_config
name: "tests/integration bazel-in-bazel: Ubuntu"
platform: ubuntu2004
integration_test_bazelinbazel_debian:
<<: *common_bazelinbazel_config
name: "tests/integration bazel-in-bazel: Debian"
platform: debian11

integration_test_compile_pip_requirements_ubuntu:
<<: *reusable_build_test_all
Expand Down Expand Up @@ -507,18 +515,6 @@ tasks:
- "bazel run //:os_specific_requirements.update"
- "git diff --exit-code"

integration_test_pip_repository_entry_points_ubuntu_workspace:
<<: *reusable_build_test_all
<<: *common_workspace_flags
name: "pip_repository_entry_points: Ubuntu, workspace"
working_directory: tests/integration/pip_repository_entry_points
platform: ubuntu2004
integration_test_pip_repository_entry_points_debian_workspace:
<<: *reusable_build_test_all
<<: *common_workspace_flags
name: "pip_repository_entry_points: Debian, workspace"
working_directory: tests/integration/pip_repository_entry_points
platform: debian11
integration_test_pip_repository_entry_points_macos_workspace:
<<: *reusable_build_test_all
<<: *common_workspace_flags
Expand All @@ -532,18 +528,6 @@ tasks:
working_directory: tests/integration/pip_repository_entry_points
platform: windows

integration_test_ignore_root_user_error_ubuntu_workspace:
<<: *reusable_build_test_all
<<: *common_workspace_flags
name: "ignore_root_user_error: Ubuntu, workspace"
working_directory: tests/integration/ignore_root_user_error
platform: ubuntu2004
integration_test_ignore_root_user_error_debian_workspace:
<<: *reusable_build_test_all
<<: *common_workspace_flags
name: "ignore_root_user_error: Debian, workspace"
working_directory: tests/integration/ignore_root_user_error
platform: debian11
integration_test_ignore_root_user_error_macos_workspace:
<<: *reusable_build_test_all
<<: *common_workspace_flags
Expand Down Expand Up @@ -602,4 +586,3 @@ tasks:
shell_commands:
# Assert that @compile_pip_requirements//:requirements_test does the right thing.
- "bazel test @compile_pip_requirements//..."

12 changes: 9 additions & 3 deletions .bazelignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,19 @@ bazel-testlogs
# treated as directories with valid BUILD files for the main repo.
# Any directory with a WORKSPACE in it should be added here, with
# an entry like `bazel-{workspacename}`
examples/bzlmod/bazel-bin
examples/bzlmod/bazel-bzlmod
examples/bzlmod/bazel-out
examples/bzlmod/bazel-testlogs
examples/bzlmod/other_module/bazel-bin
examples/bzlmod/other_module/bazel-other_module
examples/bzlmod/other_module/bazel-out
examples/bzlmod/other_module/bazel-testlogs
examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation
examples/multi_python_versions/bazel-multi_python_versions
examples/pip_parse/bazel-pip_parse
examples/pip_parse_vendored/bazel-pip_parse_vendored
examples/py_proto_library/bazel-py_proto_library
tests/compile_pip_requirements/bazel-compile_pip_requirements
tests/ignore_root_user_error/bazel-ignore_root_user_error
tests/pip_repository_entry_points/bazel-pip_repository_entry_points
tests/integration/compile_pip_requirements/bazel-compile_pip_requirements
tests/integration/ignore_root_user_error/bazel-ignore_root_user_error
tests/integration/pip_repository_entry_points/bazel-pip_repository_entry_points
3 changes: 2 additions & 1 deletion .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
# Trick bazel into treating BUILD files under examples/* as being regular files
# This lets us glob() up all the files inside the examples to make them inputs to tests
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
# To update these lines, run tools/update_deleted_packages.sh
# To update these lines, execute
# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/pip_repository_entry_points
query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/pip_repository_entry_points

Expand Down
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ repos:
rev: 6.1.0
hooks:
- id: buildifier
args: &args
args: &args
# Keep this argument in sync with .bazelci/presubmit.yaml
- --warnings=all
- id: buildifier-lint
Expand All @@ -30,7 +30,7 @@ repos:
hooks:
- id: isort
name: isort (python)
args:
args:
- --profile
- black
- repo: https://github.com/psf/black
Expand All @@ -42,6 +42,6 @@ repos:
- id: update-deleted-packages
name: Update deleted packages
language: script
entry: ./tools/update_deleted_packages.sh
files: ^((examples|tests)/*/(MODULE.bazel|WORKSPACE|WORKSPACE.bzlmod|BUILD.bazel)|.bazelrc|tools/update_deleted_packages.sh)$
entry: bazel run @rules_bazel_integration_test//tools:update_deleted_packages
files: ^((examples|tests)/.*/(MODULE.bazel|WORKSPACE|WORKSPACE.bzlmod|BUILD.bazel)|.bazelrc)$
pass_filenames: false
4 changes: 1 addition & 3 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,7 @@ filegroup(
"@rules_python_gazelle_plugin//:distribution",
],
visibility = [
"//examples:__pkg__",
"//tests:__pkg__",
"//tests/toolchains:__pkg__",
"//:__subpackages__",
],
)

Expand Down
31 changes: 28 additions & 3 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ bazel_dep(name = "platforms", version = "0.0.4")
bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")

bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")

internal_deps = use_extension("@rules_python//python/private/bzlmod:internal_deps.bzl", "internal_deps")
internal_deps.install()
use_repo(
Expand Down Expand Up @@ -55,7 +53,10 @@ use_repo(python, "pythons_hub")
# This call registers the Python toolchains.
register_toolchains("@pythons_hub//:all")

# ===== DEV ONLY SETUP =====
# ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)

dev_pip = use_extension(
"//python/extensions:pip.bzl",
"pip",
Expand All @@ -76,3 +77,27 @@ dev_pip.parse(
python_version = "3.11",
requirements_lock = "//docs/sphinx:requirements.txt",
)

bazel_binaries = use_extension(
"@rules_bazel_integration_test//:extensions.bzl",
"bazel_binaries",
dev_dependency = True,
)

# Keep in sync with //:version.bzl
bazel_binaries.local(
name = "self",
path = "tests/integration/bazel_from_env",
)
bazel_binaries.download(version = "6.4.0")
bazel_binaries.download(version = "rolling")
use_repo(
bazel_binaries,
"bazel_binaries",
# These don't appear necessary, but are reported as direct dependencies
# that should be use_repo()'d, so we add them as requested
"bazel_binaries_bazelisk",
"build_bazel_bazel_6_4_0",
"build_bazel_bazel_rolling",
"build_bazel_bazel_self",
)
2 changes: 0 additions & 2 deletions examples/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.

package(default_visibility = ["//visibility:public"])

licenses(["notice"]) # Apache 2.0
1 change: 1 addition & 0 deletions gazelle/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ filegroup(
name = "distribution",
srcs = [
":BUILD.bazel",
":MODULE.bazel",
":README.md",
":WORKSPACE",
":def.bzl",
Expand Down
17 changes: 17 additions & 0 deletions internal_deps.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,23 @@ def rules_python_internal_deps():
],
)

http_archive(
name = "rules_bazel_integration_test",
sha256 = "6e65d497c68f5794349bfa004369e144063686ce1ebd0227717cd23285be45ef",
urls = [
"https://github.com/bazel-contrib/rules_bazel_integration_test/releases/download/v0.20.0/rules_bazel_integration_test.v0.20.0.tar.gz",
],
)

# Dependency of rules_bazel_integration_test.
http_archive(
name = "cgrindel_bazel_starlib",
sha256 = "9090280a9cff7322e7c22062506b3273a2e880ca464e520b5c77fdfbed4e8805",
urls = [
"https://github.com/cgrindel/bazel-starlib/releases/download/v0.18.1/bazel-starlib.v0.18.1.tar.gz",
],
)

http_archive(
name = "rules_proto",
sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
Expand Down
8 changes: 8 additions & 0 deletions internal_setup.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,12 @@
"""Setup for rules_python tests and tools."""

load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies")
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies")
load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries")
load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=bzl-visibility

Expand All @@ -34,3 +38,7 @@ def rules_python_internal_setup():
rules_proto_toolchains()

protobuf_deps()

bazel_integration_test_rules_dependencies()
bazel_starlib_dependencies()
bazel_binaries(versions = SUPPORTED_BAZEL_VERSIONS)
4 changes: 3 additions & 1 deletion python/entry_points/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ filegroup(
name = "distribution",
srcs = glob([
"*.bzl",
]),
]) + [
"BUILD.bazel",
],
visibility = ["//python:__subpackages__"],
)
1 change: 1 addition & 0 deletions python/pip_install/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ filegroup(
name = "distribution",
srcs = glob(["*.bzl"]) + [
"BUILD.bazel",
"pip_repository_requirements.bzl.tmpl",
"//python/pip_install/private:distribution",
"//python/pip_install/tools/dependency_resolver:distribution",
"//python/pip_install/tools/wheel_installer:distribution",
Expand Down
93 changes: 93 additions & 0 deletions tests/integration/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# Copyright 2023 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

load("@rules_bazel_integration_test//bazel_integration_test:defs.bzl", "default_test_runner")
load(":integration_test.bzl", "rules_python_integration_test")

licenses(["notice"])

_WORKSPACE_FLAGS = [
"--noenable_bzlmod",
]

_WORKSPACE_GAZELLE_PLUGIN_FLAGS = [
"--override_repository=rules_python_gazelle_plugin=../../../rules_python_gazelle_plugin",
]

_GAZELLE_PLUGIN_FLAGS = [
"--override_module=rules_python_gazelle_plugin=../../../rules_python_gazelle_plugin",
]

default_test_runner(
name = "workspace_test_runner",
bazel_cmds = [
"info {}".format(" ".join(_WORKSPACE_FLAGS)),
"test {} //...".format(" ".join(_WORKSPACE_FLAGS)),
],
visibility = ["//visibility:public"],
)

default_test_runner(
name = "workspace_test_runner_gazelle_plugin",
bazel_cmds = [
"info {}".format(" ".join(_WORKSPACE_FLAGS + _WORKSPACE_GAZELLE_PLUGIN_FLAGS)),
"test {} //...".format(" ".join(_WORKSPACE_FLAGS + _WORKSPACE_GAZELLE_PLUGIN_FLAGS)),
],
visibility = ["//visibility:public"],
)

default_test_runner(
name = "test_runner",
visibility = ["//visibility:public"],
)

default_test_runner(
name = "test_runner_gazelle_plugin",
bazel_cmds = [
"info {}".format(" ".join(_GAZELLE_PLUGIN_FLAGS)),
"test {} //...".format(" ".join(_GAZELLE_PLUGIN_FLAGS)),
],
visibility = ["//visibility:public"],
)

# TODO: add compile_pip_requirements_test_from_external_repo

rules_python_integration_test(
name = "pip_repository_entry_points_workspace_test",
timeout = "long",
bzlmod = False,
# The dependencies needed for this test are not cross-platform: https://github.com/bazelbuild/rules_python/issues/260
tags = ["fix-windows"],
workspace_path = "pip_repository_entry_points",
)

rules_python_integration_test(
name = "compile_pip_requirements_test",
)

rules_python_integration_test(
name = "compile_pip_requirements_workspace_test",
bzlmod = False,
workspace_path = "compile_pip_requirements",
)

rules_python_integration_test(
name = "ignore_root_user_error_test",
)

rules_python_integration_test(
name = "ignore_root_user_error_workspace_test",
bzlmod = False,
workspace_path = "ignore_root_user_error",
)
Loading

0 comments on commit 8aaaf93

Please sign in to comment.