From f2789fa25217dcd5fa66d7e210f4ad88861c8961 Mon Sep 17 00:00:00 2001
From: Marco Braga <mrbraga@redhat.com>
Date: Fri, 8 Nov 2024 13:54:54 -0300
Subject: [PATCH] splat-1844: add step to report iam used by job

---
 ...penshift-release-master__nightly-4.18.yaml |  11 +
 .../openshift-release-master-periodics.yaml   |  77 +++++++
 .../aws-provision-iam-user-commands.sh        |  35 ++-
 .../iam-user/aws-provision-iam-user-ref.yaml  |  10 +
 .../gather/cloud-iam-access/OWNERS            |   4 +
 .../gather/cloud-iam-access/aws/OWNERS        |   4 +
 .../gather-cloud-iam-access-aws-commands.sh   | 206 ++++++++++++++++++
 ...her-cloud-iam-access-aws-ref.metadata.json |  10 +
 .../aws/gather-cloud-iam-access-aws-ref.yaml  |  31 +++
 .../ipi/aws/post/ipi-aws-post-chain.yaml      |   1 +
 ...-conf-aws-user-min-permissions-commands.sh |   9 +
 ...ipi-conf-aws-user-min-permissions-ref.yaml |   9 +
 .../install/ipi-install-install-commands.sh   |   5 +
 13 files changed, 400 insertions(+), 12 deletions(-)
 create mode 100644 ci-operator/step-registry/gather/cloud-iam-access/OWNERS
 create mode 100644 ci-operator/step-registry/gather/cloud-iam-access/aws/OWNERS
 create mode 100755 ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-commands.sh
 create mode 100644 ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.metadata.json
 create mode 100644 ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.yaml

diff --git a/ci-operator/config/openshift/release/openshift-release-master__nightly-4.18.yaml b/ci-operator/config/openshift/release/openshift-release-master__nightly-4.18.yaml
index 7ceb36ffcc111..96ef8f2c7f661 100644
--- a/ci-operator/config/openshift/release/openshift-release-master__nightly-4.18.yaml
+++ b/ci-operator/config/openshift/release/openshift-release-master__nightly-4.18.yaml
@@ -1858,6 +1858,17 @@ tests:
     cluster_profile: aws-perfscale-qe
     workflow: rosa-aws-sts-hcp-conformance
   timeout: 6h0m0s
+- as: e2e-aws-ovn-perms-discovery
+  cron: '@weekly'
+  steps:
+    cluster_profile: aws
+    env:
+      AWS_INSTALL_USE_MINIMAL_PERMISSIONS: "yes"
+      AWS_INSTALL_USE_CUSTOM_IDENTITY: "yes"
+      SKIP_CREATE_POLICY: "yes"
+      POLICY_ARN: "arn:aws:iam::aws:policy/AdministratorAccess"
+    workflow: openshift-e2e-aws
+  timeout: 6h0m0s
 zz_generated_metadata:
   branch: master
   org: openshift
diff --git a/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml
index a7baae013e1e4..266a9669903a2 100644
--- a/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml
+++ b/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml
@@ -105462,6 +105462,83 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build10
+  cron: '@weekly'
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+    timeout: 6h0m0s
+  extra_refs:
+  - base_ref: master
+    org: openshift
+    repo: release
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws
+    ci-operator.openshift.io/variant: nightly-4.18
+    ci.openshift.io/generator: prowgen
+    ci.openshift.io/no-builds: "true"
+    job-release: "4.18"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-release-master-nightly-4.18-e2e-aws-ovn-perms-discovery
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=e2e-aws-ovn-perms-discovery
+      - --variant=nightly-4.18
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build10
   cron: 0 0 */2 * *
diff --git a/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-commands.sh b/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-commands.sh
index 3c4e996e9e59d..e452071d58323 100644
--- a/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-commands.sh
+++ b/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-commands.sh
@@ -54,26 +54,32 @@ PERMISSIONS_POLICY_FILENAME="aws-permissions-policy-creds.json"
 USER_POLICY_FILE="${SHARED_DIR}/${PERMISSIONS_POLICY_FILENAME}"
 USER_CREDENTIALS_OUTPUT_FILENAME="aws_minimal_permission"
 
-if [ ! -f ${USER_POLICY_FILE} ]; then
+if [[ "${SKIP_CREATE_POLICY-}" == "yes" ]]; then
+	if [ -z "${POLICY_ARN-}" ]; then
+		echo "POLICY_ARN must be set when SKIP_CREATE_POLICY is set"
+		exit 1
+	fi
+elif [ ! -f "${USER_POLICY_FILE}" ]; then
 	echo "User permission policy file not found. Skipping user creation"
 	exit 0
+else
+	echo "Policy file:"
+	jq . "${USER_POLICY_FILE}"
+
+	POLICY_NAME="${CLUSTER_NAME}-required-policy"
+	POLICY_DOC=$(cat "${USER_POLICY_FILE}" | jq -c .)
+	POLICY_OUTOUT=/tmp/aws_policy_output
+	echo "Creating policy ${POLICY_NAME}"
+	aws_create_policy "${REGION}" "${POLICY_NAME}" "${POLICY_DOC}" "${POLICY_OUTOUT}"
 fi
 
-echo "Policy file:"
-jq . $USER_POLICY_FILE
-
-POLICY_NAME="${CLUSTER_NAME}-required-policy"
-POLICY_DOC=$(cat "${USER_POLICY_FILE}" | jq -c .)
-POLICY_OUTOUT=/tmp/aws_policy_output
-
-echo "Creating policy ${POLICY_NAME}"
-aws_create_policy $REGION "${POLICY_NAME}" "${POLICY_DOC}" "${POLICY_OUTOUT}"
-
 USER_NAME="${CLUSTER_NAME}-minimal-perm"
-POLICY_ARN=$(jq -r '.Policy.Arn' ${POLICY_OUTOUT})
 USER_OUTOUT=/tmp/aws_user_output
 CRED_OUTOUT=/tmp/aws_cred_output
 
+# Registering the creation time to collect audit logs since it
+USER_CREATED_TIMESTAMP="$(date -u "+%Y-%m-%dT%H:%M:%S+00:00")"
+
 echo "Creating user ${USER_NAME}"
 aws_create_user $REGION "${USER_NAME}" "${POLICY_ARN}" "${USER_OUTOUT}" "${CRED_OUTOUT}"
 
@@ -95,3 +101,8 @@ EOF
 # for destroy
 echo ${POLICY_ARN} >"${SHARED_DIR}/aws_policy_arns"
 echo ${USER_NAME} >"${SHARED_DIR}/aws_user_names"
+
+# used by IAM event parser
+echo "${USER_CREATED_TIMESTAMP}" > "${SHARED_DIR}"/time_iam_created
+echo "${CLUSTER_NAME}" > "${SHARED_DIR}"/CLUSTER_NAME
+echo "${LEASED_RESOURCE}" > "${SHARED_DIR}"/LEASED_RESOURCE
diff --git a/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-ref.yaml b/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-ref.yaml
index c5e696858ac2f..254209d54b4c2 100644
--- a/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-ref.yaml
+++ b/ci-operator/step-registry/aws/provision/iam-user/aws-provision-iam-user-ref.yaml
@@ -11,5 +11,15 @@ ref:
     requests:
       cpu: 10m
       memory: 100Mi
+  env:
+    - name: SKIP_CREATE_POLICY
+      default: ""
+      documentation: |-
+        Set to "yes" to skip the policy creation. It requires the variable POLICY_ARN to be set.
+    - name: POLICY_ARN
+      default: ""
+      documentation: |-
+        Defines the Policy ARN when the SKIP_CREATE_POLICY is set to "yes", preventing the
+        step to create an Policy, using a custom provided by the job level.
   documentation: |-
     Create AWS IAM user with the permissions from aws-permissions-policy-creds.json file .
diff --git a/ci-operator/step-registry/gather/cloud-iam-access/OWNERS b/ci-operator/step-registry/gather/cloud-iam-access/OWNERS
new file mode 100644
index 0000000000000..5c348746b4262
--- /dev/null
+++ b/ci-operator/step-registry/gather/cloud-iam-access/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- r4f4
+- mtulio
+- wking
diff --git a/ci-operator/step-registry/gather/cloud-iam-access/aws/OWNERS b/ci-operator/step-registry/gather/cloud-iam-access/aws/OWNERS
new file mode 100644
index 0000000000000..5c348746b4262
--- /dev/null
+++ b/ci-operator/step-registry/gather/cloud-iam-access/aws/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- r4f4
+- mtulio
+- wking
diff --git a/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-commands.sh b/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-commands.sh
new file mode 100755
index 0000000000000..a046bdcd6a420
--- /dev/null
+++ b/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-commands.sh
@@ -0,0 +1,206 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM
+
+# attemps to collect events only if job is running with custom IAM User
+if [ "${AWS_INSTALL_USE_CUSTOM_IDENTITY-}" != "yes" ];
+then
+	echo "Skipping step as custom IAM identity is not enabled. AWS_INSTALL_USE_CUSTOM_IDENTITY=[${AWS_INSTALL_USE_CUSTOM_IDENTITY-}]"
+	exit 0
+fi
+
+# Check if custom user name has been set, otherwise fail.
+if [ ! -f "${SHARED_DIR}/aws_user_names" ];
+then
+	echo "Flag AWS_INSTALL_USE_CUSTOM_IDENTITY is enabled but no custom user name has been found. Check if step aws-provision-iam-user has been succeeded."
+	exit 1
+fi
+
+# Check if the control file with timestamp of user creation has been created, otherwise fail.
+if [ ! -f "${SHARED_DIR}/time_iam_created" ];
+then
+	echo "Unable to find timestamp that custom IAM user has been created. Check if step aws-provision-iam-user has been succeeded."
+	exit 1
+fi
+
+if [ -z "${CLUSTER_NAME-}" ];
+then
+	export CLUSTER_NAME=$(<"${SHARED_DIR}/cluster_name")
+	if [ -z "${CLUSTER_NAME}" ];
+	then
+		echo "Unable to locate CLUSTER_NAME"
+		exit 1
+	fi
+fi
+
+if [ -z "${LEASED_RESOURCE-}" ];
+then
+	export LEASED_RESOURCE=$(<"${SHARED_DIR}"/LEASED_RESOURCE || true)
+	if [ -z "${LEASED_RESOURCE}" ];
+	then
+		echo "Unable to locate LEASED_RESOURCE"
+		exit 1
+	fi
+fi
+
+
+function log_msg() {
+	echo -e "$(date -u --rfc-3339=seconds)> $*"
+}
+
+#
+# Globals
+#
+EVENT_WORKDIR=/tmp/iam-events-"${CLUSTER_NAME}"
+EVENTS_PATH_RAW=${EVENT_WORKDIR}/objects
+EVENTS_PATH_PARSED=${EVENT_WORKDIR}/parsed
+CREDS_REQ_PATH=${EVENT_WORKDIR}/credrequests
+CREDS_REQ_PATH_RAW=${EVENT_WORKDIR}/credrequests-raw
+
+export AWS_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/.awscred"
+
+export GATHER_EVENT_START_TIME="$(<"${SHARED_DIR}"/time_iam_created)"
+export GATHER_EVENT_END_TIME="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+
+ACCOUNT_ID="$(aws sts get-caller-identity | jq -r .Account)"
+
+#FILE_PREFIX="${ACCOUNT_ID}_CloudTrail_${JOB_REGION}"
+OBJECTS_PREFIX="AWSLogs/${ACCOUNT_ID}/CloudTrail/${LEASED_RESOURCE}"
+OBJECTS_PREFIX_START="${OBJECTS_PREFIX}/$(date -d "${GATHER_EVENT_START_TIME}" +%Y/%m)"
+
+#
+# Init event discovery
+#
+log_msg "Starting event gathering with timestamps: "
+echo "start=[${GATHER_EVENT_START_TIME}] end=[${GATHER_EVENT_END_TIME}]"
+
+mkdir -pv "${EVENTS_PATH_RAW}" || true
+GATHER_THRESHOLD=0
+RETRY_LIMIT=10
+while true; do
+	GATHER_THRESHOLD=$((GATHER_THRESHOLD+1))
+	if [ "${GATHER_THRESHOLD}" -ge ${RETRY_LIMIT} ]; then
+		log_msg "ERROR Timeout waiting for newer events, trying the best effort parsing existing logs"
+		break
+	fi
+	log_msg "Collecting objects with started since ${GATHER_EVENT_START_TIME}"
+	aws s3api list-objects-v2 \
+		--bucket "${AWS_TRAIL_BUCKET_NAME}" \
+		--prefix "${OBJECTS_PREFIX_START}" \
+		--query 'Contents[?LastModified > `'"${GATHER_EVENT_START_TIME}"'`]' \
+		> "${EVENTS_PATH_RAW}"-metadata
+
+	found=$(jq -r '.|length' "${EVENTS_PATH_RAW}"-metadata)
+	if [[ ${found} -eq 0 ]]; then
+		log_msg "Found 0 event, waiting 30s for next iteration [${GATHER_THRESHOLD}/${RETRY_LIMIT}]";
+		sleep 30;
+		continue
+	fi
+	log_msg "Found [$(jq -r '.|length' "${EVENTS_PATH_RAW}"-metadata)] archive files with events"
+	for obKey in $(jq -r .[].Key "${EVENTS_PATH_RAW}"-metadata); do
+		objName=$(basename "${obKey}")
+		# syncronize only if event archive wasn't downloaded yet
+		if [[ ! -f ${EVENTS_PATH_RAW}/${objName} ]]; then
+			echo "Downloading archive ${objName//${ACCOUNT_ID}/XXXXXXXXXXXX}"
+			aws s3 cp s3://"${AWS_TRAIL_BUCKET_NAME}"/"${obKey}" "${EVENTS_PATH_RAW}"/ >/dev/null
+		fi
+	done
+
+	log_msg "Checking timestamp of the first record"
+	LOG_INITIAL_EVENT=$(zcat "${EVENTS_PATH_RAW}"/* | jq -r '.Records[].eventTime'| sort -n | head -n1 || true)
+
+	log_msg "Checking timestamp of the last record"
+	LOG_LATEST_EVENT=$(zcat "${EVENTS_PATH_RAW}"/* | jq -r '.Records[].eventTime'| sort -n | tail -n1 || true)
+
+	log_msg "Found event timestamps: initial=[${LOG_INITIAL_EVENT}] final=[${LOG_LATEST_EVENT}]"
+
+	# Check if the latest event is greater than desired stop time.
+	if [ "$(date -d "${LOG_LATEST_EVENT}" +%s)" -gt "$(date -d "${GATHER_EVENT_END_TIME}" +%s)" ];
+	then
+		log_msg "Found event with timestamp newer than desired: [${LOG_LATEST_EVENT}]"
+		break
+	fi
+
+	log_msg "Latest[${LOG_LATEST_EVENT}] event is older than the desired[${GATHER_EVENT_END_TIME}]"
+	log_msg "Pausing 30 seconds before checking latest events. [${GATHER_THRESHOLD}/${RETRY_LIMIT}]"
+	sleep 30
+done
+
+#
+# Download cci (cloud credentials insights)
+#
+# TODO(mtulio): define where to save that cross-component tool to parse IAM events.
+# This script must not be savend in component repo as it is intented to be used by
+# CI.
+CCI=/tmp/cci
+log_msg "Downloading cci (cloud credential insights) utility"
+curl -s https://raw.githubusercontent.com/mtulio/mtulio.labs/refs/heads/exp-ocp-cred-informer/labs/ocp-identity/cloud-credentials-insights/cci.py > ${CCI}
+chmod +x ${CCI}
+
+#
+# Extracting events
+#
+log_msg "\nChecking the size of discovered events / raw data"
+du -sh "${EVENTS_PATH_RAW}"
+
+log_msg "Extracting insights from events"
+
+#INSTALLER_USER_NAME="${CLUSTER_NAME}"-installer 
+#INSTALLER_USER_NAME="${CLUSTER_NAME}"-minimal-perm 
+#INSTALLER_USER_NAME=origin-ci-robot-provision
+INSTALLER_USER_NAME=$(head -n1 "${SHARED_DIR}/aws_user_names")
+
+mkdir -v "${EVENTS_PATH_PARSED}" || true
+${CCI} --command extract \
+	--events-path "${EVENTS_PATH_RAW}" \
+	--output "${EVENTS_PATH_PARSED}" \
+    --filters principal-prefix="${CLUSTER_NAME}",principal-name="${INSTALLER_USER_NAME}"
+
+#
+# Extract credentials requests
+#
+log_msg "Attempting to extract credential requests from RELEASE_IMAGE_LATEST=${RELEASE_IMAGE_LATEST:-}"
+
+mkdir -v "${CREDS_REQ_PATH}" || true
+mkdir -v "${CREDS_REQ_PATH_RAW}" || true
+
+function extract_credrequests() {
+	pushd "${CREDS_REQ_PATH_RAW}"
+	cp "${CLUSTER_PROFILE_DIR}"/pull-secret pull-secret
+	#cp $PULL_SECRET_FILE pull-secret
+	oc registry login --to pull-secret
+	oc adm release extract --registry-config pull-secret \
+		--credentials-requests --cloud=aws \
+		--to="${CREDS_REQ_PATH}" \
+		--from="${RELEASE_IMAGE_LATEST}"
+	popd
+	CCI_EXTRA_ARGS+="--credentials-requests-path=${CREDS_REQ_PATH} "
+}
+extract_credrequests || true
+
+#
+# Parse events considering requested permissions by CredentialsRequests manifests
+#
+log_msg "Creating report based in events and credential requests..."
+
+# TODO: check if we can cover this file more generically.
+INSTALLER_REQUEST_FILE=${SHARED_DIR}/4.18-mint-creds.json
+if [[ ! -f "${INSTALLER_REQUEST_FILE}" ]];
+then
+	echo "{}" | jq . > "${INSTALLER_REQUEST_FILE}"
+fi
+
+${CCI} --command compare \
+	--events-path "${EVENTS_PATH_PARSED}"/events.json \
+	--output "${EVENTS_PATH_PARSED}" \
+	--installer-user-name "${INSTALLER_USER_NAME}" \
+	--installer-requests-file "${INSTALLER_REQUEST_FILE}" \
+	--filters cluster-name="${CLUSTER_NAME}" \
+	${CCI_EXTRA_ARGS-}
+
+log_msg "Copying results to artifacts directory"
+cp -v "${EVENTS_PATH_PARSED}"/* "${ARTIFACT_DIR}"/
diff --git a/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.metadata.json b/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.metadata.json
new file mode 100644
index 0000000000000..d9b7a3771d14e
--- /dev/null
+++ b/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.metadata.json
@@ -0,0 +1,10 @@
+{
+	"path": "gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.yaml",
+	"owners": {
+		"approvers": [
+			"r4f4",
+			"mtulio",
+			"wking"
+		]
+	}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.yaml b/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.yaml
new file mode 100644
index 0000000000000..93ee4b10422d3
--- /dev/null
+++ b/ci-operator/step-registry/gather/cloud-iam-access/aws/gather-cloud-iam-access-aws-ref.yaml
@@ -0,0 +1,31 @@
+ref:
+  as: gather-cloud-iam-access-aws
+  optional_on_success: true
+  from_image:
+    namespace: ocp
+    name: "4.14"
+    tag: upi-installer
+  grace_period: 10m
+  commands: gather-cloud-iam-access-aws-commands.sh
+  env:
+  - name: AWS_INSTALL_USE_CUSTOM_IDENTITY
+    default: ""
+    documentation: |-
+      Set to "yes" to force to collect IAM events to discover used permissions.
+  - name: AWS_TRAIL_BUCKET_NAME
+    default: "cloud-trail-test-clayton"
+    documentation: |-
+      Bucket name with CloudTrail events.
+  resources:
+    requests:
+      cpu: 300m
+      memory: 300Mi
+  documentation: |-
+    The gather-cloud-iam-access gather collets IAM events through cloud services to try to get
+    inference of permissions used when installing a cluster on CI job.
+    The inference will be more precise when the custom IAM identity is created to execute the job.
+    This step is only enabled when AWS_INSTALL_USE_CUSTOM_IDENTITY flag is set to 'yes'.
+    For AWS clusters, the logs will be collected from the CloudTrail, scraping logs from
+    a pre-provisioned Bucket used as a backend to store the Trail events.
+    The inference and report will consider only IAM credentials prefixed with CLUSTER_NAME,
+    all other resources must be ignored.
diff --git a/ci-operator/step-registry/ipi/aws/post/ipi-aws-post-chain.yaml b/ci-operator/step-registry/ipi/aws/post/ipi-aws-post-chain.yaml
index 2ca11937bc625..7732e9ff857a7 100644
--- a/ci-operator/step-registry/ipi/aws/post/ipi-aws-post-chain.yaml
+++ b/ci-operator/step-registry/ipi/aws/post/ipi-aws-post-chain.yaml
@@ -4,5 +4,6 @@ chain:
   - ref: gather-aws-console
   - chain: ipi-deprovision
   - ref: aws-deprovision-users-and-policies
+  - ref: gather-cloud-iam-access-aws
   documentation: |-
     The IPI cleanup step contains all steps that gather and deprovision an OpenShift cluster on AWS, provisioned by the `ipi-aws-pre` chain.
diff --git a/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-commands.sh b/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-commands.sh
index f192cdcde31f3..235cac908c69d 100644
--- a/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-commands.sh
+++ b/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-commands.sh
@@ -15,6 +15,15 @@ if [[ "${AWS_INSTALL_USE_MINIMAL_PERMISSIONS}" != "yes" ]]; then
 	exit 0
 fi
 
+if [[ "${SKIP_CREATE_POLICY-}" == "yes" ]]; then
+	if [[ -z "${POLICY_ARN-}" ]]; then
+		echo "POLICY_ARN must be defined with the Policy ARN when SKIP_CREATE_POLICY is set."
+		exit 1
+	fi
+	echo "Detected to skip policy creation by SKIP_CREATE_POLICY=yes. The existing POLICY_ARN must have required permissions to install a cluster."
+	exit 0
+fi
+
 RELEASE_IMAGE_INSTALL="${RELEASE_IMAGE_INITIAL:-}"
 if [[ -z "${RELEASE_IMAGE_INSTALL}" ]]; then
 	# If there is no initial release, we will be installing latest.
diff --git a/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-ref.yaml b/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-ref.yaml
index e48f5a403d3f9..9b419ef5ba4b8 100644
--- a/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-ref.yaml
+++ b/ci-operator/step-registry/ipi/conf/aws/user-min-permissions/ipi-conf-aws-user-min-permissions-ref.yaml
@@ -14,6 +14,15 @@ ref:
       documentation: |-
         "no", default option, if the install should be done with the standard user from the cluster profile.
         "yes" if the step should create a new user and attach a policy with only the install-needed permissions.
+    - name: SKIP_CREATE_POLICY
+      default: ""
+      documentation: |-
+        Set to "yes" to skip the policy creation. It requires the variable POLICY_ARN to be set.
+    - name: POLICY_ARN
+      default: ""
+      documentation: |-
+        Defines the Policy ARN when the SKIP_CREATE_POLICY is set to "yes", preventing the
+        step to create an Policy, using a custom provided by the job level.
   documentation: |-
     Generate a permissions policy for the given cluster configuration. This has to run as the last step in the conf
     chain, since the cluster configuration affects the permissions required.
diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh
index 187226d69dabb..baf31de8bcce3 100755
--- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh
+++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh
@@ -579,6 +579,11 @@ aws|aws-arm64|aws-usgov)
     if [[ -f "${SHARED_DIR}/aws_minimal_permission" ]]; then
         echo "Setting AWS credential with minimal permision for installer"
         export AWS_SHARED_CREDENTIALS_FILE=${SHARED_DIR}/aws_minimal_permission
+        # TODO: running the workflow locally (release-devenv) the 'create manifests' was failing
+        # for invalid credentials, testing manually it is working. Looks like there are some
+        # async operation from AWS side preventing the APIs to validate the recent created user.
+        echo "Waiting 30s to AWS populate recently created credentials"
+        sleep 30
     else
         export AWS_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/.awscred
     fi