chore(ci): add e2e tests for cloud distros

.github/bundles/rke2/uds-bundle.yaml

@@ -44,6 +44,20 @@ packages:
       - istio-ambient
       - metrics-server
     overrides:
+      istio-admin-gateway:
+        gateway:
+          values:
+            - path: service.annotations
+              value:
+                service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+                service.beta.kubernetes.io/aws-load-balancer-target-node-labels: "kubernetes.io/os=linux"
+      istio-tenant-gateway:
+        gateway:
+          values:
+            - path: service.annotations
+              value:
+                service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+                service.beta.kubernetes.io/aws-load-balancer-target-node-labels: "kubernetes.io/os=linux"
       velero:
         velero:
           variables:

.github/test-infra/aws/rke2/data.tf

@@ -10,7 +10,7 @@ data "aws_vpc" "vpc" {
 
 data "aws_subnet" "rke2_ci_subnet" {
   vpc_id            = data.aws_vpc.vpc.id
-  availability_zone = "${var.region}c"
+  availability_zone = "${var.region}a"
 
   filter {
     name   = "tag:Name"

.github/test-infra/aws/rke2/iam.tf

@@ -77,6 +77,16 @@ data "aws_iam_policy_document" "aws_ccm" {
   }
 }
 
+data "http" "aws-lb-controller-iam" {
+  url = "https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy_us-gov.json"
+}
+
+resource "aws_iam_role_policy" "aws-lb-controller" {
+  name = "${local.cluster_name}-lb-controller"
+  role = aws_iam_role.rke2_server.id
+  policy = data.http.aws-lb-controller-iam.response_body
+}
+
 resource "aws_iam_role_policy" "s3_token" {
   name   = "${local.cluster_name}-server-token"
   role   = aws_iam_role.rke2_server.id

.github/test-infra/aws/rke2/main.tf

@@ -26,6 +26,7 @@ locals {
     ccm_external                = true,
     token_bucket                = module.statestore.bucket,
     token_object                = module.statestore.token_object
+    cluster_name                = local.tags.cluster_name
   }
 }
 
@@ -95,7 +96,7 @@ resource "aws_instance" "rke2_ci_control_plane_node" {
   associate_public_ip_address = true
 
   root_block_device {
-    volume_size = 100
+    volume_size = 250
   }
 
   tags = merge(local.tags, { "kubernetes.io/cluster/${local.cluster_name}" = "owned" })
@@ -107,15 +108,16 @@ resource "aws_instance" "rke2_ci_agent_node" {
   ami                         = data.aws_ami.rhel_rke2.image_id
   instance_type               = var.agent_instance_type
   key_name                    = aws_key_pair.control_plane_key_pair.key_name
-  user_data                   = templatefile("${path.module}/scripts/user_data.sh", merge(local.userdata, { BOOTSTRAP_IP = aws_instance.rke2_ci_bootstrap_node.private_ip }))
+  user_data                   = templatefile("${path.module}/scripts/user_data.sh", merge(local.userdata, { BOOTSTRAP_IP = aws_instance.rke2_ci_bootstrap_node.private_ip, AGENT_NODE = true }))
   subnet_id                   = data.aws_subnet.rke2_ci_subnet.id
   user_data_replace_on_change = true
   iam_instance_profile        = aws_iam_instance_profile.rke2_server.name
   vpc_security_group_ids      = [aws_security_group.rke2_ci_node_sg.id]
   associate_public_ip_address = true
+  availability_zone           = "${var.region}a"
 
   root_block_device {
-    volume_size = 100
+    volume_size = 250
   }
 
   tags = merge(local.tags, { "kubernetes.io/cluster/${local.cluster_name}" = "owned" })

.github/test-infra/aws/rke2/scripts/get-kubeconfig.sh

@@ -27,7 +27,7 @@ done
 mkdir -p ~/.kube
 
 # Copy kubectl from cluster node
-ssh -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip} "mkdir -p /home/${node_user}/.kube && sudo cp /etc/rancher/rke2/rke2.yaml /home/${node_user}/.kube/config && sudo chown ${node_user} /home/${node_user}/.kube/config"  > /dev/null
+ssh -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip} "mkdir -p /home/${node_user}/.kube && sudo cp /etc/rancher/rke2/rke2.yaml /home/${node_user}/.kube/config && sudo chown ${node_user} /home/${node_user}/.kube/config" > /dev/null
 scp -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip}:/home/${node_user}/.kube/config ./rke2-config > /dev/null
 
 # Replace the loopback address with the cluster hostname

.github/test-infra/aws/rke2/scripts/user_data.sh

@@ -2,8 +2,6 @@
 # Copyright 2024 Defense Unicorns
 # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
 
-
-
 info() {
     echo "[INFO] " "$@"
 }
@@ -42,32 +40,39 @@ spec:
       - --cloud-provider=aws
 EOM
 
-#longhorn helm values: https://github.com/longhorn/longhorn/tree/master/chart
-cat > /var/lib/rancher/rke2/server/manifests/01-longhorn.yaml << EOM
+# aws lb controller helm values: https://github.com/kubernetes-sigs/aws-load-balancer-controller/tree/main/helm/aws-load-balancer-controller#configuration
+cat > /var/lib/rancher/rke2/server/manifests/01-lb-controller.yaml << EOM
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
-  name: longhorn
+  name: aws-load-balancer-controller
   namespace: kube-system
 spec:
-  chart: longhorn
-  repo: https://charts.longhorn.io
-  version: 1.7.1
-  targetNamespace: kube-system 
+  chart: aws-load-balancer-controller
+  repo: https://aws.github.io/eks-charts
+  version: 1.11.0
+  targetNamespace: kube-system
+  valuesContent: |-
+    clusterName: ${cluster_name}
 EOM
 
-#metallb helm values: https://github.com/metallb/metallb/tree/main/charts/metallb
-cat > /var/lib/rancher/rke2/server/manifests/02-metallb.yaml << EOM
+#longhorn helm values: https://github.com/longhorn/longhorn/tree/master/chart
+cat > /var/lib/rancher/rke2/server/manifests/02-longhorn.yaml << EOM
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
-  name: metallb
+  name: longhorn
   namespace: kube-system
 spec:
-  chart: metallb
-  repo: https://metallb.github.io/metallb
-  version: 0.14.8
+  chart: longhorn
+  repo: https://charts.longhorn.io
+  version: 1.8.1
   targetNamespace: kube-system
+  valuesContent: |-
+    defaultSettings:
+      deletingConfirmationFlag: true
+    longhornUI:
+      replicas: 0
 EOM
 
 info "Installing awscli"
@@ -89,14 +94,15 @@ chmod +x yq
 ./yq -i '.cloud-provider-name += "external"' /etc/rancher/rke2/config.yaml
 ./yq -i '.disable-cloud-controller += "true"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "service-account-key-file=/irsa/signer.key.pub"' /etc/rancher/rke2/config.yaml
-./yq -i '.kube-apiserver-arg += "service-account-key-file=/irsa/signer.key.pub"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "service-account-signing-key-file=/irsa/signer.key"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "api-audiences=kubernetes.svc.default"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "service-account-issuer=https://${BUCKET_REGIONAL_DOMAIN_NAME}"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "audit-log-path=/var/log/kubernetes/audit/audit.log"' /etc/rancher/rke2/config.yaml
+#Fix for metrics server scraping of kubernetes api server components
+./yq -i '.kube-controller-manager-arg[2] = "bind-address=0.0.0.0"' /etc/rancher/rke2/config.yaml
+./yq -i '.kube-scheduler-arg += "bind-address=0.0.0.0"' /etc/rancher/rke2/config.yaml
+./yq -i '.etcd-arg += "listen-metrics-urls=http://0.0.0.0:2381"|.etcd-arg style="double"' /etc/rancher/rke2/config.yaml
 rm -rf ./yq
-
-
 }
 
 pre_userdata

.github/workflows/test-aks.yaml

@@ -88,12 +88,18 @@ jobs:
       - name: Create IAC
         run: uds run -f tasks/iac.yaml apply-tofu --no-progress --set K8S_DISTRO=aks --set CLOUD=azure
 
+      - name: Configure Cluster DNS
+        run: uds run -f tasks/utils.yaml aks-coredns-setup --no-progress
+
       - name: Deploy Core Bundle
         env:
           UDS_CONFIG: .github/bundles/aks/uds-config.yaml
         run: uds deploy .github/bundles/aks/uds-bundle-uds-core-aks-nightly-*.tar.zst --confirm
         timeout-minutes: 30
 
+      - name: Test UDS Core
+        run: uds run -f tasks/test.yaml uds-core-non-k3d --set EXCLUDED_PACKAGES="metrics-server"
+
       - name: Debug Output
         if: ${{ always() }}
         uses: ./.github/actions/debug-output

.github/workflows/test-eks.yaml

@@ -92,12 +92,19 @@ jobs:
         run: uds run -f tasks/iac.yaml create-iac --no-progress --set K8S_DISTRO=eks --set CLOUD=aws
         timeout-minutes: 20
 
+      - name: Configure Cluster DNS
+        run: uds run -f tasks/utils.yaml eks-coredns-setup --no-progress
+
       - name: Deploy Core Bundle
         env:
           UDS_CONFIG: .github/bundles/eks/uds-config.yaml
         run: uds deploy .github/bundles/eks/uds-bundle-uds-core-eks-nightly-*.tar.zst --confirm
         timeout-minutes: 30
 
+      - name: Test UDS Core
+        run: uds run -f tasks/test.yaml uds-core-non-k3d --set EXCLUDED_PACKAGES="metrics-server"
+        continue-on-error: true
+
       - name: Debug Output
         if: ${{ always() }}
         uses: ./.github/actions/debug-output

.github/workflows/test-rke2.yaml

@@ -95,12 +95,18 @@ jobs:
         run: uds run -f tasks/iac.yaml rke2-cluster-ready --no-progress
         timeout-minutes: 20
 
+      - name: Configure Cluster DNS
+        run: uds run -f tasks/utils.yaml rke2-coredns-setup --no-progress
+
       - name: Deploy Core Bundle
         env:
           UDS_CONFIG: .github/bundles/rke2/uds-config.yaml
         run: uds deploy .github/bundles/rke2/uds-bundle-uds-core-rke2-nightly-*.tar.zst --confirm
         timeout-minutes: 30
 
+      - name: Test UDS Core
+        run: uds run -f tasks/test.yaml uds-core-non-k3d
+
       - name: Debug Output
         if: ${{ always() }}
         uses: ./.github/actions/debug-output

src/istio/tasks.yaml

@@ -6,7 +6,7 @@ tasks:
     inputs:
       validate_passthrough:
         description: Whether to validate the passthrough gateway
-        default: "true"
+        default: "false"
     actions:
       - description: Validate the Istio Admin Gateway
         wait:

src/test/tasks.yaml

@@ -1,6 +1,9 @@
 # Copyright 2024 Defense Unicorns
 # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
 
+includes:
+  - utils: ../../tasks/utils.yaml
+
 tasks:
   - name: validate
     actions:
@@ -20,9 +23,14 @@ tasks:
 
   - name: create-deploy
     description: Test app used for UDS Core validation
+    inputs:
+      architecture:
+        description: "System architecture that the test-apps package should be built for."
+        default: ${UDS_ARCH}
+
     actions:
       - description: Create zarf package for the test resources
-        cmd: "uds zarf package create src/test --confirm --no-progress --skip-sbom"
+        cmd: uds zarf package create src/test --confirm --no-progress --skip-sbom -a ${{ index .inputs "architecture" }}
 
       - description: Deploy the test resources
         cmd: "uds zarf package deploy build/zarf-package-uds-core-test-apps-*.zst --confirm --no-progress"
@@ -116,9 +124,10 @@ tasks:
 
       - description: Verify the authservice tenant app is protected by checking redirect
         maxRetries: 3
+        task: utils:tenant-gw-ip
         cmd: |
           set -e
-          SSO_REDIRECT=$(uds zarf tools kubectl run curl-test --image=cgr.dev/chainguard/curl:latest -q --restart=Never --rm -i -- -Ls -o /dev/null -w %{url_effective} "https://protected.uds.dev")
+          SSO_REDIRECT=$(uds zarf tools kubectl run curl-test --image=cgr.dev/chainguard/curl:latest -q --restart=Never --rm -i -- --resolve 'protected.uds.dev:$TENANT_GW_IP:443' -Ls -o /dev/null -w %{url_effective} "https://protected.uds.dev")
 
           case "${SSO_REDIRECT}" in
           "https://sso.uds.dev"*)

src/velero/tasks.yaml

@@ -47,26 +47,26 @@ tasks:
       - description: wait for the backup object
         wait:
           cluster:
-            kind: Backup
+            kind: backup.velero.io
             name: ${BACKUP_NAME}
             namespace: velero
       - description: check the status of the backup object
         cmd: |-
-          STATUS=$(uds zarf tools kubectl get backups -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}')
+          STATUS=$(uds zarf tools kubectl get backup.velero.io -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}')
           if [ ${STATUS} != "Completed" ]; then
             echo "Status is '$STATUS'... waiting to see if it changes"
 
             # local testing indicates the status is "Finalizing" for a few seconds after completion
             sleep 30
 
             # check again...
-            STATUS=$(uds zarf tools kubectl get backups -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}')
+            STATUS=$(uds zarf tools kubectl get backup.velero.io -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}')
             if [ ${STATUS} != "Completed" ]; then
               echo "Status is $STATUS... something isn't right.."
 
               # get backup object
-              uds zarf tools kubectl get backups -n velero ${BACKUP_NAME} -o yaml
-              uds zarf tools kubectl get backups -A -o yaml
+              uds zarf tools kubectl get backup.velero.io -n velero ${BACKUP_NAME} -o yaml
+              uds zarf tools kubectl get backup.velero.io -A -o yaml
               echo "::endgroup::"
 
               # get backupstoragelocations

tasks/iac.yaml

@@ -1,6 +1,8 @@
 # Copyright 2024 Defense Unicorns
 # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
 
+includes:
+  - util: ./utils.yaml
 
 variables:
   - name: CLUSTER_NAME
@@ -118,18 +120,19 @@ tasks:
               break
             fi
           done
-
           # wait for cluster components
           while true; do
-            if [ $(uds zarf tools kubectl get po,job -A --no-headers=true | egrep -v 'Running|Complete' | wc -l) -gt 0 ]; then
+            if [ $(uds zarf tools kubectl get po,job -A --no-headers=true | egrep -v 'helm-install|Running|Complete' | wc -l) -gt 0 ]; then
               echo "Waiting for cluster components to be ready...";
               sleep 5;
             else
               echo "Cluster is ready"
               break
             fi
           done
-          uds zarf tools kubectl apply -f ./metallb.yaml
+          #uds zarf tools kubectl apply -f ./metallb.yaml
+      - task: util:rke2-coredns-setup
+      - task: util:rke2-allow-prom-kube-dns
         dir: .github/test-infra/aws/rke2/
         maxTotalSeconds: 600