Update dependencies, buildbarn ecosystem to latest (#29)

This change brings bb-remote-asset up-to-date with bb-storage/bb-remote-execution, so that nominally they may be able to use compatible configs, and likely inherit the other benefits (more recent toolchain, better bazel integration, etc.) This is done largely by copying `WORKSPACE` from bb-storage, making sure to match versions and orderings to ensure that the same versions of dependencies are loaded. Other transforms/migrations that proved necessary:0000 * `HTTPClient` was eliminated from bb-storage; mocking had to be done on a `http.RoundTripper` instead, in the same manner as buildbarn/bb-storage@b9951ac * Addition of structured concurrency to main, similar to buildbarn/bb-storage@4f243ea * Addition of `http.ClientConfiguration` proto options to `HttpFetcherConfiguration` * `bazel.canonical_id` added to the list of supported qualifiers, which prevents errors in some (most?) WORKSPACE configurations Tested: * `bazel test //...` with buildbarn/bb-storage#170 works on my system * Seems to work as expected when integrated with the bb-deployments docker-compose setup as in buildbarn/bb-deployments#91
buildbarn · Sep 10, 2023 · 52b25db · 52b25db
1 parent 6a6e6f9
commit 52b25db
Show file tree

Hide file tree

Showing 34 changed files with 2,498 additions and 1,172 deletions.
diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml
@@ -1,32 +1,47 @@
 {
    "jobs": {
       "build_and_test": {
-         "container": "docker://l.gcr.io/google/bazel:3.3.1",
          "runs-on": "ubuntu-latest",
          "steps": [
+            {
+               "name": "Installing Bazel",
+               "run": "curl -L https://github.com/bazelbuild/bazel/releases/download/6.0.0/bazel-6.0.0-linux-x86_64 > ~/bazel && chmod +x ~/bazel && echo ~ >> ${GITHUB_PATH}"
+            },
             {
                "name": "Check out source code",
                "uses": "actions/checkout@v1"
             },
             {
-               "name": "Buildifier",
-               "run": "bazel run @com_github_bazelbuild_buildtools//:buildifier"
+               "name": "Restore Bazel cache",
+               "uses": "actions/cache@v1",
+               "with": {
+                  "key": "bazel",
+                  "path": "~/.cache/bazel"
+               }
             },
             {
                "name": "Gazelle",
                "run": "bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro go_dependencies.bzl%go_dependencies -prune && bazel run //:gazelle"
             },
+            {
+               "name": "Buildifier",
+               "run": "sed '/^$/d' go_dependencies.bzl > go_dependencies.bzl.new && mv go_dependencies.bzl.new go_dependencies.bzl && bazel run @com_github_bazelbuild_buildtools//:buildifier"
+            },
             {
                "name": "Gofmt",
-               "run": "bazel run @cc_mvdan_gofumpt//:gofumpt -- -s -w -extra $(pwd)"
+               "run": "bazel run @cc_mvdan_gofumpt//:gofumpt -- -lang 1.19 -w -extra $(pwd)"
             },
             {
                "name": "Clang format",
-               "run": "find . -name '*.proto' -exec bazel run @llvm_toolchain//:bin/clang-format -- -i {} +"
+               "run": "find . -name '*.proto' -exec bazel run @llvm_toolchain_llvm//:bin/clang-format -- -i {} +"
             },
             {
                "name": "GitHub workflows",
-               "run": "bazel build //tools/github_workflows && cp bazel-bin/tools/github_workflows/* .github/workflows"
+               "run": "bazel build //tools/github_workflows && cp bazel-bin/tools/github_workflows/*.yaml .github/workflows"
+            },
+            {
+               "name": "Protobuf generation",
+               "run": "find . bazel-bin/pkg/proto -name '*.pb.go' -delete || true\nbazel build $(bazel query 'kind(\"go_proto_library\", //...)')\nfind bazel-bin/pkg/proto -name '*.pb.go' | while read f; do\n  cat $f > $(echo $f | sed -e 's|.*/pkg/proto/|pkg/proto/|')\ndone\n"
             },
             {
                "name": "Test style conformance",
@@ -36,17 +51,13 @@
                "name": "Golint",
                "run": "bazel run @org_golang_x_lint//golint -- -set_exit_status $(pwd)/..."
             },
-            {
-               "name": "Check for ineffective assignments",
-               "run": "bazel run @com_github_gordonklaus_ineffassign//:ineffassign $(pwd)"
-            },
             {
                "name": "linux_amd64: build and test",
-               "run": "bazel test --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."
+               "run": "bazel test --test_output=errors --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."
             },
             {
                "name": "linux_amd64: copy bb_remote_asset",
-               "run": "bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
+               "run": "rm -f bb_remote_asset && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
             },
             {
                "name": "linux_amd64: upload bb_remote_asset",
@@ -56,13 +67,61 @@
                   "path": "bb_remote_asset"
                }
             },
+            {
+               "name": "linux_386: build and test",
+               "run": "bazel test --test_output=errors --platforms=@io_bazel_rules_go//go/toolchain:linux_386 //..."
+            },
+            {
+               "name": "linux_386: copy bb_remote_asset",
+               "run": "rm -f bb_remote_asset && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_386 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
+            },
+            {
+               "name": "linux_386: upload bb_remote_asset",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_remote_asset.linux_386",
+                  "path": "bb_remote_asset"
+               }
+            },
+            {
+               "name": "linux_arm: build and test",
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_arm //..."
+            },
+            {
+               "name": "linux_arm: copy bb_remote_asset",
+               "run": "rm -f bb_remote_asset && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_arm //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
+            },
+            {
+               "name": "linux_arm: upload bb_remote_asset",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_remote_asset.linux_arm",
+                  "path": "bb_remote_asset"
+               }
+            },
+            {
+               "name": "linux_arm64: build and test",
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64 //..."
+            },
+            {
+               "name": "linux_arm64: copy bb_remote_asset",
+               "run": "rm -f bb_remote_asset && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
+            },
+            {
+               "name": "linux_arm64: upload bb_remote_asset",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_remote_asset.linux_arm64",
+                  "path": "bb_remote_asset"
+               }
+            },
             {
                "name": "darwin_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64 //..."
             },
             {
                "name": "darwin_amd64: copy bb_remote_asset",
-               "run": "bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
+               "run": "rm -f bb_remote_asset && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
             },
             {
                "name": "darwin_amd64: upload bb_remote_asset",
@@ -72,13 +131,29 @@
                   "path": "bb_remote_asset"
                }
             },
+            {
+               "name": "darwin_arm64: build and test",
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64 //..."
+            },
+            {
+               "name": "darwin_arm64: copy bb_remote_asset",
+               "run": "rm -f bb_remote_asset && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
+            },
+            {
+               "name": "darwin_arm64: upload bb_remote_asset",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_remote_asset.darwin_arm64",
+                  "path": "bb_remote_asset"
+               }
+            },
             {
                "name": "freebsd_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_remote_asset"
             },
             {
                "name": "freebsd_amd64: copy bb_remote_asset",
-               "run": "bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
+               "run": "rm -f bb_remote_asset && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset"
             },
             {
                "name": "freebsd_amd64: upload bb_remote_asset",
@@ -94,7 +169,7 @@
             },
             {
                "name": "windows_amd64: copy bb_remote_asset",
-               "run": "bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset.exe"
+               "run": "rm -f bb_remote_asset.exe && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64 //cmd/bb_remote_asset $(pwd)/bb_remote_asset.exe"
             },
             {
                "name": "windows_amd64: upload bb_remote_asset",
@@ -106,14 +181,14 @@
             },
             {
                "env": {
-                  "DOCKER_CONFIG_JSON": "${{ secrets.DOCKER_CONFIG_JSON }}"
+                  "GITHUB_TOKEN": "${{ secrets.GITHUB_TOKEN }}"
                },
                "name": "Install Docker credentials",
-               "run": "mkdir ~/.docker && echo \"${DOCKER_CONFIG_JSON}\" > ~/.docker/config.json"
+               "run": "echo \"${GITHUB_TOKEN}\" | docker login ghcr.io -u $ --password-stdin"
             },
             {
                "name": "Push container bb_remote_asset:bb_remote_asset",
-               "run": "bazel run //cmd/bb_remote_asset:bb_remote_asset_container_push"
+               "run": "bazel run --stamp //cmd/bb_remote_asset:bb_remote_asset_container_push"
             }
          ]
       }

diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml
@@ -1,32 +1,47 @@
 {
    "jobs": {
       "build_and_test": {
-         "container": "docker://l.gcr.io/google/bazel:3.3.1",
          "runs-on": "ubuntu-latest",
          "steps": [
+            {
+               "name": "Installing Bazel",
+               "run": "curl -L https://github.com/bazelbuild/bazel/releases/download/6.0.0/bazel-6.0.0-linux-x86_64 > ~/bazel && chmod +x ~/bazel && echo ~ >> ${GITHUB_PATH}"
+            },
             {
                "name": "Check out source code",
                "uses": "actions/checkout@v1"
             },
             {
-               "name": "Buildifier",
-               "run": "bazel run @com_github_bazelbuild_buildtools//:buildifier"
+               "name": "Restore Bazel cache",
+               "uses": "actions/cache@v1",
+               "with": {
+                  "key": "bazel",
+                  "path": "~/.cache/bazel"
+               }
             },
             {
                "name": "Gazelle",
                "run": "bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro go_dependencies.bzl%go_dependencies -prune && bazel run //:gazelle"
             },
+            {
+               "name": "Buildifier",
+               "run": "sed '/^$/d' go_dependencies.bzl > go_dependencies.bzl.new && mv go_dependencies.bzl.new go_dependencies.bzl && bazel run @com_github_bazelbuild_buildtools//:buildifier"
+            },
             {
                "name": "Gofmt",
-               "run": "bazel run @cc_mvdan_gofumpt//:gofumpt -- -s -w -extra $(pwd)"
+               "run": "bazel run @cc_mvdan_gofumpt//:gofumpt -- -lang 1.19 -w -extra $(pwd)"
             },
             {
                "name": "Clang format",
-               "run": "find . -name '*.proto' -exec bazel run @llvm_toolchain//:bin/clang-format -- -i {} +"
+               "run": "find . -name '*.proto' -exec bazel run @llvm_toolchain_llvm//:bin/clang-format -- -i {} +"
             },
             {
                "name": "GitHub workflows",
-               "run": "bazel build //tools/github_workflows && cp bazel-bin/tools/github_workflows/* .github/workflows"
+               "run": "bazel build //tools/github_workflows && cp bazel-bin/tools/github_workflows/*.yaml .github/workflows"
+            },
+            {
+               "name": "Protobuf generation",
+               "run": "find . bazel-bin/pkg/proto -name '*.pb.go' -delete || true\nbazel build $(bazel query 'kind(\"go_proto_library\", //...)')\nfind bazel-bin/pkg/proto -name '*.pb.go' | while read f; do\n  cat $f > $(echo $f | sed -e 's|.*/pkg/proto/|pkg/proto/|')\ndone\n"
             },
             {
                "name": "Test style conformance",
@@ -37,17 +52,29 @@
                "run": "bazel run @org_golang_x_lint//golint -- -set_exit_status $(pwd)/..."
             },
             {
-               "name": "Check for ineffective assignments",
-               "run": "bazel run @com_github_gordonklaus_ineffassign//:ineffassign $(pwd)"
+               "name": "linux_amd64: build and test",
+               "run": "bazel test --test_output=errors --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."
             },
             {
-               "name": "linux_amd64: build and test",
-               "run": "bazel test --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."
+               "name": "linux_386: build and test",
+               "run": "bazel test --test_output=errors --platforms=@io_bazel_rules_go//go/toolchain:linux_386 //..."
+            },
+            {
+               "name": "linux_arm: build and test",
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_arm //..."
+            },
+            {
+               "name": "linux_arm64: build and test",
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64 //..."
             },
             {
                "name": "darwin_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64 //..."
             },
+            {
+               "name": "darwin_arm64: build and test",
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64 //..."
+            },
             {
                "name": "freebsd_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_remote_asset"

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -1,21 +1,30 @@
 load("@bazel_gazelle//:def.bzl", "gazelle")
 load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier")
+load("@npm//:defs.bzl", "npm_link_all_packages")
 
 # gazelle:prefix github.com/buildbarn/bb-remote-asset
 # gazelle:resolve proto build/bazel/remote/execution/v2/remote_execution.proto @com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto
-# gazelle:resolve proto go build/bazel/remote/execution/v2/remote_execution.proto @com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:go_default_library
+# gazelle:resolve proto go build/bazel/remote/execution/v2/remote_execution.proto @com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution
 # gazelle:resolve proto build/bazel/remote/asset/v1/remote_asset.proto @com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:remote_asset_proto
-# gazelle:resolve proto go build/bazel/remote/asset/v1/remote_asset.proto @com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:go_default_library
+# gazelle:resolve proto go build/bazel/remote/asset/v1/remote_asset.proto @com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:asset
 # gazelle:resolve proto pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto
-# gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:go_default_library
+# gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore
 # gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto
-# gazelle:resolve proto go pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:go_default_library
+# gazelle:resolve proto go pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global
 # gazelle:resolve proto pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc:grpc_proto
-# gazelle:resolve proto go pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc:go_default_library
+# gazelle:resolve proto go pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc
+# gazelle:resolve proto pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http:http_proto
+# gazelle:resolve proto go pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http
+# gazelle:resolve proto opentelemetry/proto/common/v1/common.proto @io_opentelemetry_proto//:common_proto
+# gazelle:resolve proto go opentelemetry/proto/common/v1/common.proto @io_opentelemetry_go_proto_otlp//common/v1:common
 gazelle(
     name = "gazelle",
 )
 
 buildifier(
     name = "buildifier",
 )
+
+npm_link_all_packages(
+    name = "node_modules",
+)