From 6c5da91fc461e02a5b7d35486f826d138d5cd3c0 Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Mon, 3 Mar 2025 20:07:42 -0700 Subject: [PATCH 1/2] XENG-8909 Create docker network for step containers --- README.rst | 3 +- buildrunner/docker/daemon.py | 9 +++++- buildrunner/docker/runner.py | 25 ++++++++++++++-- buildrunner/sshagent/__init__.py | 19 ++++++++++-- buildrunner/steprunner/__init__.py | 2 ++ buildrunner/steprunner/tasks/run.py | 45 +++++++++++++++++++++++++---- 6 files changed, 89 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index f133b96a..88d76e7f 100755 --- a/README.rst +++ b/README.rst @@ -690,8 +690,7 @@ Service Containers Service containers allow you to create and start additional containers that are linked to the primary build container. This is useful, for instance, if your unit or integration tests require an outside service, such as a database -service. Service containers are instantiated in the order they are listed, and -service containers can rely on previously instantiated service containers. +service. Service containers are instantiated in the order they are listed. Service containers have the same injected environment variables and volume mounts as build containers do, but the /source mount is read-only. diff --git a/buildrunner/docker/daemon.py b/buildrunner/docker/daemon.py index 7765baaa..ff9cf4c1 100644 --- a/buildrunner/docker/daemon.py +++ b/buildrunner/docker/daemon.py @@ -19,12 +19,13 @@ class DockerDaemonProxy: Class used to encapsulate Docker daemon information within a container. """ - def __init__(self, docker_client, log, docker_registry, container_labels): + def __init__(self, docker_client, log, docker_registry, container_labels, network): """ """ self.docker_client = docker_client self.docker_registry = docker_registry self.log = log self.container_labels = container_labels + self.network = network self._daemon_container = None self._env = { "DOCKER_HOST": DOCKER_DEFAULT_DOCKERD_URL, @@ -83,6 +84,12 @@ def start(self): command="/bin/sh", volumes=_volumes, host_config=self.docker_client.create_host_config(binds=_binds), + labels=self.container_labels, + networking_config=self.docker_client.create_networking_config( + {self.network: self.docker_client.create_endpoint_config()} + ) + if self.network + else None, )["Id"] self.docker_client.start(self._daemon_container) self.log.write( diff --git a/buildrunner/docker/runner.py b/buildrunner/docker/runner.py index 39921dc6..9e9c8132 100644 --- a/buildrunner/docker/runner.py +++ b/buildrunner/docker/runner.py @@ -155,6 +155,7 @@ def start( systemd_cgroup2: bool = False, cap_add=None, privileged=False, + network=None, ): # pylint: disable=too-many-arguments,too-many-locals """ Kwargs: @@ -250,6 +251,18 @@ def start( if entrypoint: kwargs["entrypoint"] = entrypoint del kwargs["command"] + if network: + kwargs["networking_config"] = ( + self.docker_client.create_networking_config( + { + network: self.docker_client.create_endpoint_config( + aliases=[hostname] if hostname else None + ) + } + ) + if network + else None + ) if compare_version("1.10", self.docker_client.api_version) < 0: kwargs["dns"] = dns @@ -654,7 +667,7 @@ def _get_status(self): pass return status - def get_ip(self): + def get_ip(self, network=None): """ Return the ip address of the running container """ @@ -664,7 +677,15 @@ def get_ip(self): inspection = self.docker_client.inspect_container( self.container["Id"], ) - ipaddr = inspection.get("NetworkSettings", {}).get("IPAddress", None) + network_settings = inspection.get("NetworkSettings", {}) + if network: + ipaddr = ( + network_settings.get("Networks", {}) + .get(network, {}) + .get("IPAddress", None) + ) + else: + ipaddr = network_settings.get("IPAddress", None) except docker.errors.APIError: pass return ipaddr diff --git a/buildrunner/sshagent/__init__.py b/buildrunner/sshagent/__init__.py index 283fed7e..eb698e16 100644 --- a/buildrunner/sshagent/__init__.py +++ b/buildrunner/sshagent/__init__.py @@ -95,6 +95,7 @@ def __init__( docker_registry, multiplatform_image_builder, container_labels, + network, ): """ """ self.docker_client = docker_client @@ -106,6 +107,7 @@ def __init__( self._ssh_channel = None self._multiplatform_image_builder = multiplatform_image_builder self._container_labels = container_labels + self._network = network def get_info(self): """ @@ -138,6 +140,11 @@ def start(self, keys): f"{keys[0].get_name()} {keys[0].get_base64()}", ], labels=self._container_labels, + networking_config=self.docker_client.create_networking_config( + {self._network: self.docker_client.create_endpoint_config()} + ) + if self._network + else None, host_config=self.docker_client.create_host_config( publish_all_ports=True, ), @@ -154,9 +161,15 @@ def start(self, keys): _ssh_container = self.docker_client.inspect_container( self._ssh_agent_container ) - _ssh_host = _ssh_container.get("NetworkSettings", {}).get( - "IPAddress", _ssh_host - ) + network_settings = _ssh_container.get("NetworkSettings", {}) + if self._network: + _ssh_host = ( + network_settings.get("Networks", {}) + .get(self._network, {}) + .get("IPAddress", _ssh_host) + ) + else: + _ssh_host = network_settings.get("IPAddress", _ssh_host) _ssh_port = 22 else: # get the Docker server ip address and ssh port exposed by this diff --git a/buildrunner/steprunner/__init__.py b/buildrunner/steprunner/__init__.py index a5510f13..8674aa25 100644 --- a/buildrunner/steprunner/__init__.py +++ b/buildrunner/steprunner/__init__.py @@ -80,6 +80,8 @@ def __init__( self.id = str(uuid.uuid4()) # pylint: disable=invalid-name self.multi_platform = multi_platform self.container_labels = container_labels + # network name is used to identify the network that the build step is running in + self.network_name = f"{build_runner.build_id}-{step_name}" def run(self): """ diff --git a/buildrunner/steprunner/tasks/run.py b/buildrunner/steprunner/tasks/run.py index c909e092..06421fcc 100644 --- a/buildrunner/steprunner/tasks/run.py +++ b/buildrunner/steprunner/tasks/run.py @@ -14,6 +14,7 @@ import threading import time import uuid + import python_on_whales import buildrunner.docker @@ -66,6 +67,10 @@ def __init__(self, step_runner, step: StepRun): self._docker_client = buildrunner.docker.new_client( timeout=step_runner.build_runner.docker_timeout, ) + if self.step_runner.network_name and not self._docker_client.networks( + names=[self.step_runner.network_name] + ): + self._docker_client.create_network(self.step_runner.network_name) self._source_container = None self._service_runners = OrderedDict() self._service_links = {} @@ -74,6 +79,12 @@ def __init__(self, step_runner, step: StepRun): self.runner = None self.images_to_remove = [] + def __del__(self): + if self.step_runner.network_name and self._docker_client.networks( + names=[self.step_runner.network_name] + ): + self._docker_client.remove_network(self.step_runner.network_name) + def _get_source_container(self): """ Get (creating the container if necessary) the container id of the @@ -84,6 +95,13 @@ def _get_source_container(self): self.step_runner.build_runner.get_source_image(), command="/bin/sh", labels=self.step_runner.container_labels, + networking_config=self._docker_client.create_networking_config( + { + self.step_runner.network_name: self._docker_client.create_endpoint_config() + } + ) + if self.step_runner.network_name + else None, )["Id"] self._docker_client.start( self._source_container, @@ -504,7 +522,7 @@ def _start_service_container(self, name, service: Service): _user = service.user # determine if a hostname is specified - _hostname = service.hostname + _hostname = service.hostname or name # determine if a dns host is specified _dns = None @@ -618,6 +636,7 @@ def _start_service_container(self, name, service: Service): containers=_containers, systemd=systemd, systemd_cgroup2=self.is_systemd_cgroup2(systemd, service, _image), + network=self.step_runner.network_name, ) self._service_links[cont_name] = name @@ -658,9 +677,17 @@ def wait(self, name, wait_for_data): """ Wait for listening port on named container """ - ipaddr = self._docker_client.inspect_container(name)["NetworkSettings"][ - "IPAddress" - ] + network_settings = self._docker_client.inspect_container(name).get( + "NetworkSettings", {} + ) + if self.step_runner.network_name: + ipaddr = ( + network_settings.get("Networks", {}) + .get(self.step_runner.network_name, {}) + .get("IPAddress", None) + ) + else: + ipaddr = network_settings.get("IPAddress", None) socket_open = False if isinstance(wait_for_data, dict): @@ -710,6 +737,7 @@ def wait(self, name, wait_for_data): nc_tester.start( # The shell is the command shell=f"-n -z {ipaddr} {port}", + network=self.step_runner.network_name, ) nc_tester.attach_until_finished() @@ -734,12 +762,14 @@ def wait(self, name, wait_for_data): def _resolve_service_ip(self, service_name): """ - If service_name represents a running service, return it's IP address. + If service_name represents a running service, return its IP address. Otherwise, return the service_name """ rval = service_name if isinstance(service_name, str) and service_name in self._service_runners: - ipaddr = self._service_runners[service_name].get_ip() + ipaddr = self._service_runners[service_name].get_ip( + self.step_runner.network_name + ) if ipaddr is not None: rval = ipaddr return rval @@ -830,6 +860,7 @@ def run(self, context: dict): # pylint: disable=too-many-statements,too-many-br buildrunner_config.global_config.docker_registry, self.step_runner.multi_platform, self.step_runner.container_labels, + self.step_runner.network_name, ) self._sshagent.start( buildrunner_config.get_ssh_keys_from_aliases( @@ -843,6 +874,7 @@ def run(self, context: dict): # pylint: disable=too-many-statements,too-many-br self.step_runner.log, buildrunner_config.global_config.docker_registry, self.step_runner.container_labels, + self.step_runner.network_name, ) self._dockerdaemonproxy.start() @@ -1031,6 +1063,7 @@ def run(self, context: dict): # pylint: disable=too-many-statements,too-many-br container_args["systemd_cgroup2"] = self.is_systemd_cgroup2( container_args["systemd"], self.step, _run_image ) + container_args["network"] = self.step_runner.network_name container_id = self.runner.start( links=self._service_links, **container_args From 5850e76b7196eeecc0e8c86010410ca2d7448faa Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Thu, 6 Mar 2025 14:24:44 -0700 Subject: [PATCH 2/2] XENG-8909 Add test for service container communication --- tests/runservicecontainer/Dockerfile | 3 ++- tests/test-files/test-general-buildx.yaml | 2 -- tests/test-files/test-general.yaml | 2 -- tests/test-files/test-services.yaml | 22 ++++++++++++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 tests/test-files/test-services.yaml diff --git a/tests/runservicecontainer/Dockerfile b/tests/runservicecontainer/Dockerfile index bf7323bc..d0310b88 100644 --- a/tests/runservicecontainer/Dockerfile +++ b/tests/runservicecontainer/Dockerfile @@ -1,3 +1,4 @@ ARG DOCKER_REGISTRY FROM $DOCKER_REGISTRY/rockylinux:8.5 -RUN yum -y install curl +RUN mkdir /results +VOLUME /results diff --git a/tests/test-files/test-general-buildx.yaml b/tests/test-files/test-general-buildx.yaml index 9cfd8f3e..d561b798 100644 --- a/tests/test-files/test-general-buildx.yaml +++ b/tests/test-files/test-general-buildx.yaml @@ -104,8 +104,6 @@ steps: image: {{ DOCKER_REGISTRY }}/rockylinux:8.5 cmd: 'echo "hello" > /hello.txt' post-build: - # use a different container here to make sure inject overrides - #path: runservicecontainer inject: 'tests/postbuildpath/Dockerfile': '/' push: diff --git a/tests/test-files/test-general.yaml b/tests/test-files/test-general.yaml index cb9d2b7e..e66d7c9a 100644 --- a/tests/test-files/test-general.yaml +++ b/tests/test-files/test-general.yaml @@ -105,8 +105,6 @@ steps: image: {{ DOCKER_REGISTRY }}/rockylinux:8.5 cmd: 'echo "hello" > /hello.txt' post-build: - # use a different container here to make sure inject overrides - #path: runservicecontainer inject: 'tests/postbuildpath/Dockerfile': '/' push: diff --git a/tests/test-files/test-services.yaml b/tests/test-files/test-services.yaml new file mode 100644 index 00000000..f4bb34d7 --- /dev/null +++ b/tests/test-files/test-services.yaml @@ -0,0 +1,22 @@ +steps: + my-services-step: + run: + image: rockylinux:8.5 + volumes_from: + - stats1 + cmds: + - timeout 20 sh -c 'while [ ! -e /results/stats1 ] || [ ! -e /results/stats2 ] || [ ! -e /results/stats3 ]; do sleep 5; done' + services: + stats1: + build: tests/runservicecontainer + cmd: until ping -c1 stats2 >/dev/null 2>&1 && ping -c1 stats3 >/dev/null 2>&1; do sleep 5; done && touch /results/stats1 + stats2: + build: tests/runservicecontainer + volumes_from: + - stats1 + cmd: until ping -c1 stats1 >/dev/null 2>&1 && ping -c1 stats3 >/dev/null 2>&1; do sleep 5; done && touch /results/stats2 + stats3: + build: tests/runservicecontainer + volumes_from: + - stats1 + cmd: until ping -c1 stats1 >/dev/null 2>&1 && ping -c1 stats2 >/dev/null 2>&1; do sleep 5; done && touch /results/stats3 \ No newline at end of file