Windows build: Enable RBE build in Windows CI (#11107)

Commit Message:
Enable RBE build in Windows CI

- Bumps envoy-build-tools and bazel-toolchains
- Bumps envoy-build-ubuntu image tag
  - Running Windows CI via docker uses the same mechanism as Linux to find the image tag, the tags are the same for Windows and Linux images
- Temporarily adds a patch to fix rules_go for Windows RBE issue executing batch scripts, see #11657
- Tags/fixes some additional tests that fail in CI

Additional Description: N/A
Risk Level: Low, but we may observe differences in performance of Windows CI running remotely, we may need to adjust the worker pool accordingly
Testing: N/A
Docs Changes: N/A
Release Notes: N/A
Fixes #10619

Signed-off-by: Sunjay Bhatia <[email protected]>
Co-authored-by: William A Rowe Jr <[email protected]>

.azure-pipelines/pipelines.yml

@@ -168,6 +168,8 @@ jobs:
       - bash: ci/run_envoy_docker_windows.sh ci/windows_ci_steps.sh
         displayName: "Run Windows CI"
         env:
+          ENVOY_RBE: "true"
+          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)"
           BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
           BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
           GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

.bazelrc

@@ -160,6 +160,11 @@ build:rbe-toolchain-gcc --config=rbe-toolchain
 build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain
 build:rbe-toolchain-gcc --extra_toolchains=@rbe_ubuntu_gcc//config:cc-toolchain
 
+build:rbe-toolchain-msvc-cl --host_platform=@rbe_windows_msvc_cl//config:platform
+build:rbe-toolchain-msvc-cl --platforms=@rbe_windows_msvc_cl//config:platform
+build:rbe-toolchain-msvc-cl --crosstool_top=@rbe_windows_msvc_cl//cc:toolchain
+build:rbe-toolchain-msvc-cl --extra_toolchains=@rbe_windows_msvc_cl//config:cc-toolchain
+
 build:remote --spawn_strategy=remote,sandboxed,local
 build:remote --strategy=Javac=remote,sandboxed,local
 build:remote --strategy=Closure=remote,sandboxed,local
@@ -168,6 +173,15 @@ build:remote --remote_timeout=7200
 build:remote --auth_enabled=true
 build:remote --remote_download_toplevel
 
+# Windows bazel does not allow sandboxed as a spawn strategy
+build:remote-windows --spawn_strategy=remote,local
+build:remote-windows --strategy=Javac=remote,local
+build:remote-windows --strategy=Closure=remote,local
+build:remote-windows --strategy=Genrule=remote,local
+build:remote-windows --remote_timeout=7200
+build:remote-windows --auth_enabled=true
+build:remote-windows --remote_download_toplevel
+
 build:remote-clang --config=remote
 build:remote-clang --config=rbe-toolchain-clang
 
@@ -181,9 +195,12 @@ build:remote-msan --config=remote
 build:remote-msan --config=rbe-toolchain-clang-libc++
 build:remote-msan --config=rbe-toolchain-msan
 
+build:remote-msvc-cl --config=remote-windows
+build:remote-msvc-cl --config=rbe-toolchain-msvc-cl
+
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:12b3d2c2ffa582507e5d6dd34632b2b990f1b195
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:f21773ab398a879f976936f72c78c9dd3718ca1e
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker

.circleci/config.yml

@@ -5,7 +5,7 @@ executors:
     description: "A regular build executor based on ubuntu image"
     docker:
       # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
-      - image: envoyproxy/envoy-build-ubuntu:12b3d2c2ffa582507e5d6dd34632b2b990f1b195
+      - image: envoyproxy/envoy-build-ubuntu:f21773ab398a879f976936f72c78c9dd3718ca1e
     resource_class: xlarge
     working_directory: /source
 

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM gcr.io/envoy-ci/envoy-build:12b3d2c2ffa582507e5d6dd34632b2b990f1b195
+FROM gcr.io/envoy-ci/envoy-build:f21773ab398a879f976936f72c78c9dd3718ca1e
 
 ARG USERNAME=vscode
 ARG USER_UID=501

bazel/repositories.bzl

@@ -125,7 +125,12 @@ def _go_deps(skip_targets):
     # Keep the skip_targets check around until Istio Proxy has stopped using
     # it to exclude the Go rules.
     if "io_bazel_rules_go" not in skip_targets:
-        _repository_impl("io_bazel_rules_go")
+        _repository_impl(
+            name = "io_bazel_rules_go",
+            # TODO(wrowe, sunjayBhatia): remove when Windows RBE supports batch file invocation
+            patch_args = ["-p1"],
+            patches = ["@envoy//bazel:rules_go.patch"],
+        )
         _repository_impl("bazel_gazelle")
 
 def envoy_dependencies(skip_targets = []):

bazel/repository_locations.bzl

@@ -53,11 +53,11 @@ DEPENDENCY_REPOSITORIES = dict(
         use_category = ["build"],
     ),
     bazel_toolchains = dict(
-        sha256 = "144290c4166bd67e76a54f96cd504ed86416ca3ca82030282760f0823c10be48",
-        strip_prefix = "bazel-toolchains-3.1.1",
+        sha256 = "db48eed61552e25d36fe051a65d2a329cc0fb08442627e8f13960c5ab087a44e",
+        strip_prefix = "bazel-toolchains-3.2.0",
         urls = [
-            "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.1.1/bazel-toolchains-3.1.1.tar.gz",
-            "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/3.1.1.tar.gz",
+            "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.2.0/bazel-toolchains-3.2.0.tar.gz",
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/3.2.0.tar.gz",
         ],
         use_category = ["build"],
     ),
@@ -67,10 +67,10 @@ DEPENDENCY_REPOSITORIES = dict(
         use_category = ["build"],
     ),
     envoy_build_tools = dict(
-        sha256 = "b0efe70a1d122fffb89570771f4ec3b912aa0a8a0ce56218223918d7737d01e2",
-        strip_prefix = "envoy-build-tools-3cbc1d66b9e9ead42daf69e01597cacf4fb52151",
-        # 2020-05-15
-        urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/3cbc1d66b9e9ead42daf69e01597cacf4fb52151.tar.gz"],
+        sha256 = "c4d27c0a5db918e861b7164d69cdffe920daafbe3f597ffdda5a1d10c1abc992",
+        strip_prefix = "envoy-build-tools-557ee9b44a3d08cf38d9ce6f08adb872c385d6a5",
+        # 2020-06-16
+        urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/557ee9b44a3d08cf38d9ce6f08adb872c385d6a5.tar.gz"],
         use_category = ["build"],
     ),
     boringssl = dict(

bazel/rules_go.patch

@@ -0,0 +1,30 @@
+#
+# Bazel RBE on Windows GCP workers currently will not invoke cmd.exe batch files correctly
+#
+# Symptom is program not found 'bazel-out', because of the way that the CreateProcess command
+# is constructed by bazel with actions.run with forward slashes, e.g. the command
+#   cmd.exe /c "bazel-out/host/bin/external/go_sdk/builder.exe.bat"
+# where cmd.exe on GCP is treating 'bazel-out' as the target, and /host as a command line switch.
+# This problem was not observed on Azure CI pipelines or locally by the developers. The eventual
+# fix is not specific to rules_go; this patch simply addresses immediate breakage and can be removed
+# once the underlying issue within Bazel/RBE is fixed.
+# See:
+# - https://github.com/bazelbuild/rules_go/pull/2542
+# - https://github.com/envoyproxy/envoy/issues/11657
+#
+diff --git a/go/private/rules/binary.bzl b/go/private/rules/binary.bzl
+index b88dfd96..e68b5ece 100644
+--- a/go/private/rules/binary.bzl
++++ b/go/private/rules/binary.bzl
+@@ -128,8 +128,9 @@ def _go_tool_binary_impl(ctx):
+             content = cmd,
+         )
+         ctx.actions.run(
+-            executable = bat,
+-            inputs = sdk.libs + sdk.headers + sdk.tools + ctx.files.srcs + [sdk.go],
++            executable = "cmd.exe",
++            arguments = ["/S", "/C", bat.path.replace("/", "\\")],
++            inputs = sdk.libs + sdk.headers + sdk.tools + ctx.files.srcs + [sdk.go, bat],
+             outputs = [cout],
+             env = {"GOROOT": sdk.root_file.dirname},  # NOTE(#2005): avoid realpath in sandbox
+             mnemonic = "GoToolchainBinaryCompile",

ci/run_envoy_docker_windows.sh

@@ -2,9 +2,8 @@
 
 set -e
 
-# TODO(sunjayBhatia, wrowe): update this note once we have an RBE toolchain generated for Windows
-# NOTE: Update this from the latest pushed image here: https://hub.docker.com/r/envoyproxy/envoy-build-windows2019/tags
-ENVOY_BUILD_SHA="9b7dc527351b9888805377a05e5975d6ef8d6ae1"
+# The image tag for the Windows image is the same as the Linux one so we use the same mechanism to find it
+. $(dirname $0)/envoy_build_sha.sh
 
 [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019"
 # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker

ci/windows_ci_steps.sh

@@ -26,18 +26,12 @@ fi
 
 BAZEL_STARTUP_OPTIONS="--output_base=c:/_eb"
 BAZEL_BUILD_OPTIONS="-c opt --config=msvc-cl --show_task_finish --verbose_failures \
-  --test_output=all ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}"
-
-# With all envoy-static and //test/ tree building, no need to test compile externals
-# bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //bazel/... --build_tag_filters=-skip_on_windows
+  --test_output=errors ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}"
 
 bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static --build_tag_filters=-skip_on_windows
 
-# TODO(sunjayBhatia, wrowe): We are disabling building/running tests for now as the AZP pipelines
-# workers do not provide enough resources for us to produce fast enough or reliable enough builds.
-# Test compilation of known MSVC-compatible test sources
-# bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows --build_tests_only
-
 # Test invocations of known-working tests on Windows
-# bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,-fails_on_windows --build_tests_only --test_summary=terse --test_output=errors
+bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,-fails_on_windows --build_tests_only
 
+# Build tests that are failing to ensure no regressions
+bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,fails_on_windows --build_tests_only

test/common/event/BUILD

@@ -11,6 +11,7 @@ envoy_package()
 envoy_cc_test(
     name = "dispatcher_impl_test",
     srcs = ["dispatcher_impl_test.cc"],
+    tags = ["fails_on_windows"],
     deps = [
         "//source/common/api:api_lib",
         "//source/common/event:deferred_task",

test/common/network/utility_test.cc

@@ -171,7 +171,9 @@ TEST_P(NetworkUtilityGetLocalAddress, GetLocalAddress) {
 
 TEST(NetworkUtility, GetOriginalDst) {
   testing::NiceMock<Network::MockConnectionSocket> socket;
+#ifdef SOL_IP
   EXPECT_CALL(socket, ipVersion()).WillOnce(testing::Return(absl::nullopt));
+#endif
   EXPECT_EQ(nullptr, Utility::getOriginalDst(socket));
 }
 

test/extensions/filters/http/admission_control/BUILD

@@ -60,6 +60,7 @@ envoy_extension_cc_test(
     name = "admission_control_integration_test",
     srcs = ["admission_control_integration_test.cc"],
     extension_name = "envoy.filters.http.admission_control",
+    tags = ["fails_on_windows"],
     deps = [
         "//source/extensions/filters/http/admission_control:config",
         "//test/integration:http_integration_lib",

test/extensions/filters/udp/dns_filter/BUILD

@@ -28,6 +28,7 @@ envoy_extension_cc_test(
     name = "dns_filter_test",
     srcs = ["dns_filter_test.cc"],
     extension_name = "envoy.filters.udp_listener.dns_filter",
+    tags = ["fails_on_windows"],
     deps = [
         ":dns_filter_test_lib",
         "//source/extensions/filters/udp/dns_filter:dns_filter_lib",

test/integration/BUILD

@@ -237,6 +237,7 @@ envoy_cc_test(
     srcs = [
         "drain_close_integration_test.cc",
     ],
+    tags = ["fails_on_windows"],
     deps = [
         ":http_protocol_integration_lib",
         "//source/extensions/filters/http/health_check:config",
@@ -293,6 +294,7 @@ envoy_cc_test(
     data = [
         "//test/config/integration/certs",
     ],
+    tags = ["fails_on_windows"],
     deps = [
         ":http_integration_lib",
         "//test/test_common:utility_lib",