From 689f430e89aab3ab92d1d058bbaffdd71b98d14f Mon Sep 17 00:00:00 2001 From: jooho lee Date: Wed, 4 Dec 2024 14:35:11 -0500 Subject: [PATCH 1/8] seprate creating dsc/dsci objects part from kustomization for e2e test to avoid timing issue Signed-off-by: jooho lee --- config/overlays/test/kustomization.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/overlays/test/kustomization.yaml b/config/overlays/test/kustomization.yaml index 6cb3ee97c39..982ac998abc 100644 --- a/config/overlays/test/kustomization.yaml +++ b/config/overlays/test/kustomization.yaml @@ -5,8 +5,6 @@ resources: - ../../crd/external - ../../default - minio -- dsci.yaml -- dsc.yaml patches: - path: configmap/inferenceservice.yaml From ec98fe629ae08d93674d821d589f403b9836ae8c Mon Sep 17 00:00:00 2001 From: Hannah DeFazio Date: Thu, 20 Feb 2025 16:17:30 -0500 Subject: [PATCH 2/8] Split the e2e test cluster setup into a new script --- test/scripts/openshift-ci/run-e2e-tests.sh | 139 +--------------- test/scripts/openshift-ci/setup-e2e-tests.sh | 159 +++++++++++++++++++ 2 files changed, 163 insertions(+), 135 deletions(-) create mode 100755 test/scripts/openshift-ci/setup-e2e-tests.sh diff --git a/test/scripts/openshift-ci/run-e2e-tests.sh b/test/scripts/openshift-ci/run-e2e-tests.sh index f029682b11c..04380d4635b 100755 --- a/test/scripts/openshift-ci/run-e2e-tests.sh +++ b/test/scripts/openshift-ci/run-e2e-tests.sh @@ -18,150 +18,19 @@ # like kustomize and the mc client are installed by the script if not available. # The oc CLI is assumed to be configured with the credentials of the # target cluster. The target cluster is assumed to be a clean cluster. - -set -eu - -: "${SKLEARN_IMAGE:=kserve/sklearnserver:latest}" -: "${KSERVE_CONTROLLER_IMAGE:=quay.io/opendatahub/kserve-controller:latest}" -: "${KSERVE_AGENT_IMAGE:=quay.io/opendatahub/kserve-agent:latest}" -: "${KSERVE_ROUTER_IMAGE:=quay.io/opendatahub/kserve-router:latest}" -: "${STORAGE_INITIALIZER_IMAGE:=quay.io/opendatahub/kserve-storage-initializer:latest}" - -echo "SKLEARN_IMAGE=$SKLEARN_IMAGE" -echo "KSERVE_CONTROLLER_IMAGE=$KSERVE_CONTROLLER_IMAGE" -echo "KSERVE_AGENT_IMAGE=$KSERVE_AGENT_IMAGE" -echo "KSERVE_ROUTER_IMAGE=$KSERVE_ROUTER_IMAGE" -echo "STORAGE_INITIALIZER_IMAGE=$STORAGE_INITIALIZER_IMAGE" - -# Create directory for installing tooling -# It is assumed that $HOME/.local/bin is in the $PATH -mkdir -p $HOME/.local/bin MY_PATH=$(dirname "$0") PROJECT_ROOT=$MY_PATH/../../../ -# If Kustomize is not installed, install it -if ! command -v kustomize &>/dev/null; then - echo "Installing Kustomize" - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 5.0.1 $HOME/.local/bin -fi - -# If minio CLI is not installed, install it -if ! command -v mc &>/dev/null; then - echo "Installing Minio CLI" - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o $HOME/.local/bin/mc - chmod +x $HOME/.local/bin/mc -fi - -echo "Installing KServe Python SDK ..." +echo "Installing on cluster" pushd $PROJECT_ROOT >/dev/null - ./test/scripts/gh-actions/setup-poetry.sh - ./test/scripts/gh-actions/check-poetry-lockfile.sh +./test/scripts/openshift-ci/setup-e2e-tests.sh "$1" popd -pushd $PROJECT_ROOT/python/kserve >/dev/null - poetry install --with=test --no-interaction -popd - -# Install KServe stack -if [ "$1" != "raw" ]; then - echo "Installing OSSM" - $MY_PATH/deploy.ossm.sh - echo "Installing Serverless" - $MY_PATH/deploy.serverless.sh -fi - -echo "Installing KServe with Minio" -kustomize build $PROJECT_ROOT/config/overlays/test | - sed "s|kserve/storage-initializer:latest|${STORAGE_INITIALIZER_IMAGE}|" | - sed "s|kserve/agent:latest|${KSERVE_AGENT_IMAGE}|" | - sed "s|kserve/router:latest|${KSERVE_ROUTER_IMAGE}|" | - sed "s|kserve/kserve-controller:latest|${KSERVE_CONTROLLER_IMAGE}|" | - oc apply --server-side=true -f - - -# Install DSC/DSCI for test. (sometimes there is timing issue when it is under the same kustomization so it is separated) -oc create -f config/overlays/test/dsci.yaml -oc create -f config/overlays/test/dsc.yaml - -# Patch the inferenceservice-config ConfigMap, when running RawDeployment tests -if [ "$1" == "raw" ]; then - export OPENSHIFT_INGRESS_DOMAIN=$(oc get ingresses.config cluster -o jsonpath='{.spec.domain}') - oc patch configmap inferenceservice-config -n kserve --patch-file <(cat config/overlays/test/configmap/inferenceservice-openshift-ci-raw.yaml | envsubst) - oc delete pod -n kserve -l control-plane=kserve-controller-manager - - oc patch DataScienceCluster test-dsc --type='json' -p='[{"op": "replace", "path": "/spec/components/kserve/defaultDeploymentMode", "value": "RawDeployment"}]' -else - export OPENSHIFT_INGRESS_DOMAIN=$(oc get ingresses.config cluster -o jsonpath='{.spec.domain}') - oc patch configmap inferenceservice-config -n kserve --patch-file <(cat config/overlays/test/configmap/inferenceservice-openshift-ci-serverless.yaml | envsubst) -fi - -# Wait until KServe starts -oc wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n kserve --timeout=300s - -if [ "$1" != "raw" ]; then - echo "Installing odh-model-controller" - oc apply -k $PROJECT_ROOT/test/scripts/openshift-ci -fi - -echo "Add testing models to minio storage ..." # Reference: config/overlays/test/minio/minio-init-job.yaml -oc expose service minio-service -n kserve && sleep 5 -MINIO_ROUTE=$(oc get routes -n kserve minio-service -o jsonpath="{.spec.host}") -mc alias set storage http://$MINIO_ROUTE minio minio123 - -if ! mc ls storage/example-models >/dev/null 2>&1; then - mc mb storage/example-models -else - echo "Bucket 'example-models' already exists." -fi - -if [[ $(mc ls storage/example-models/sklearn/model.joblib |wc -l) == "1" ]]; then - echo "Test model exists" -else - echo "Copy test model" - curl -L https://storage.googleapis.com/kfserving-examples/models/sklearn/1.0/model/model.joblib -o /tmp/sklearn-model.joblib - mc cp /tmp/sklearn-model.joblib storage/example-models/sklearn/model.joblib -fi - -oc delete route -n kserve minio-service - -echo "Prepare CI namespace and install ServingRuntimes" -cat </dev/null # Note: The following images are set by openshift-ci. Uncomment if you are running on your own machine. -# export CUSTOM_MODEL_GRPC_IMG_TAG=kserve/custom-model-grpc:latest -# export IMAGE_TRANSFORMER_IMG_TAG=kserve/image-transformer:latest +export CUSTOM_MODEL_GRPC_IMG_TAG=kserve/custom-model-grpc:latest +export IMAGE_TRANSFORMER_IMG_TAG=kserve/image-transformer:latest export GITHUB_SHA=$(git rev-parse HEAD) export CI_USE_ISVC_HOST="1" diff --git a/test/scripts/openshift-ci/setup-e2e-tests.sh b/test/scripts/openshift-ci/setup-e2e-tests.sh new file mode 100755 index 00000000000..95b5001f33c --- /dev/null +++ b/test/scripts/openshift-ci/setup-e2e-tests.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a helper script to run E2E tests on the openshift-ci operator. +# This script assumes to be run inside a container/machine that has +# python pre-installed and the `oc` command available. Additional tooling, +# like kustomize and the mc client are installed by the script if not available. +# The oc CLI is assumed to be configured with the credentials of the +# target cluster. The target cluster is assumed to be a clean cluster. +set -eu + +: "${SKLEARN_IMAGE:=kserve/sklearnserver:latest}" +: "${KSERVE_CONTROLLER_IMAGE:=quay.io/opendatahub/kserve-controller:latest}" +: "${KSERVE_AGENT_IMAGE:=quay.io/opendatahub/kserve-agent:latest}" +: "${KSERVE_ROUTER_IMAGE:=quay.io/opendatahub/kserve-router:latest}" +: "${STORAGE_INITIALIZER_IMAGE:=quay.io/opendatahub/kserve-storage-initializer:latest}" + +echo "SKLEARN_IMAGE=$SKLEARN_IMAGE" +echo "KSERVE_CONTROLLER_IMAGE=$KSERVE_CONTROLLER_IMAGE" +echo "KSERVE_AGENT_IMAGE=$KSERVE_AGENT_IMAGE" +echo "KSERVE_ROUTER_IMAGE=$KSERVE_ROUTER_IMAGE" +echo "STORAGE_INITIALIZER_IMAGE=$STORAGE_INITIALIZER_IMAGE" + +# Create directory for installing tooling +# It is assumed that $HOME/.local/bin is in the $PATH +mkdir -p $HOME/.local/bin +MY_PATH=$(dirname "$0") +PROJECT_ROOT=$MY_PATH/../../../ + +# If Kustomize is not installed, install it +if ! command -v kustomize &>/dev/null; then + echo "Installing Kustomize" + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 5.0.1 $HOME/.local/bin +fi + +# If minio CLI is not installed, install it +if ! command -v mc &>/dev/null; then + echo "Installing Minio CLI" + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o $HOME/.local/bin/mc + chmod +x $HOME/.local/bin/mc +fi + +echo "Installing KServe Python SDK ..." +pushd $PROJECT_ROOT >/dev/null + ./test/scripts/gh-actions/setup-poetry.sh + ./test/scripts/gh-actions/check-poetry-lockfile.sh +popd +pushd $PROJECT_ROOT/python/kserve >/dev/null + poetry install --with=test --no-interaction +popd + +# Install KServe stack +if [ "$1" != "raw" ]; then + echo "Installing OSSM" + $MY_PATH/deploy.ossm.sh + echo "Installing Serverless" + $MY_PATH/deploy.serverless.sh +fi + +echo "Installing KServe with Minio" +kustomize build $PROJECT_ROOT/config/overlays/test | + sed "s|kserve/storage-initializer:latest|${STORAGE_INITIALIZER_IMAGE}|" | + sed "s|kserve/agent:latest|${KSERVE_AGENT_IMAGE}|" | + sed "s|kserve/router:latest|${KSERVE_ROUTER_IMAGE}|" | + sed "s|kserve/kserve-controller:latest|${KSERVE_CONTROLLER_IMAGE}|" | + oc apply --server-side=true -f - + +# Install DSC/DSCI for test. (sometimes there is timing issue when it is under the same kustomization so it is separated) +oc create -f config/overlays/test/dsci.yaml +oc create -f config/overlays/test/dsc.yaml + +# Patch the inferenceservice-config ConfigMap, when running RawDeployment tests +if [ "$1" == "raw" ]; then + export OPENSHIFT_INGRESS_DOMAIN=$(oc get ingresses.config cluster -o jsonpath='{.spec.domain}') + oc patch configmap inferenceservice-config -n kserve --patch-file <(cat config/overlays/test/configmap/inferenceservice-openshift-ci-raw.yaml | envsubst) + oc delete pod -n kserve -l control-plane=kserve-controller-manager + + oc patch DataScienceCluster test-dsc --type='json' -p='[{"op": "replace", "path": "/spec/components/kserve/defaultDeploymentMode", "value": "RawDeployment"}]' +else + export OPENSHIFT_INGRESS_DOMAIN=$(oc get ingresses.config cluster -o jsonpath='{.spec.domain}') + oc patch configmap inferenceservice-config -n kserve --patch-file <(cat config/overlays/test/configmap/inferenceservice-openshift-ci-serverless.yaml | envsubst) +fi + +# Wait until KServe starts +oc wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n kserve --timeout=300s + +if [ "$1" != "raw" ]; then + echo "Installing odh-model-controller" + oc apply -k $PROJECT_ROOT/test/scripts/openshift-ci +fi + +echo "Add testing models to minio storage ..." # Reference: config/overlays/test/minio/minio-init-job.yaml +oc expose service minio-service -n kserve && sleep 5 +MINIO_ROUTE=$(oc get routes -n kserve minio-service -o jsonpath="{.spec.host}") +mc alias set storage http://$MINIO_ROUTE minio minio123 + +if ! mc ls storage/example-models >/dev/null 2>&1; then + mc mb storage/example-models +else + echo "Bucket 'example-models' already exists." +fi + +if [[ $(mc ls storage/example-models/sklearn/model.joblib |wc -l) == "1" ]]; then + echo "Test model exists" +else + echo "Copy test model" + curl -L https://storage.googleapis.com/kfserving-examples/models/sklearn/1.0/model/model.joblib -o /tmp/sklearn-model.joblib + mc cp /tmp/sklearn-model.joblib storage/example-models/sklearn/model.joblib +fi + +oc delete route -n kserve minio-service + +echo "Prepare CI namespace and install ServingRuntimes" +cat < Date: Tue, 4 Feb 2025 09:52:25 -0500 Subject: [PATCH 3/8] Add authorino and kserve-local-gateway --- test/scripts/openshift-ci/setup-e2e-tests.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/scripts/openshift-ci/setup-e2e-tests.sh b/test/scripts/openshift-ci/setup-e2e-tests.sh index 95b5001f33c..d782f684bad 100755 --- a/test/scripts/openshift-ci/setup-e2e-tests.sh +++ b/test/scripts/openshift-ci/setup-e2e-tests.sh @@ -97,6 +97,19 @@ oc wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n if [ "$1" != "raw" ]; then echo "Installing odh-model-controller" + # authorino + curl -sL https://raw.githubusercontent.com/Kuadrant/authorino-operator/main/utils/install.sh | + bash -s -- -v 0.16.0 + + # kserve-local-gateway + curl https://raw.githubusercontent.com/opendatahub-io/opendatahub-operator/bde4b4e8478b5d03195e2777b9d550922e3cdcbc/components/kserve/resources/servicemesh/routing/istio-kserve-local-gateway.tmpl.yaml | + sed "s/{{ .ControlPlane.Namespace }}/istio-system/g" | + oc create -f - + + curl https://raw.githubusercontent.com/opendatahub-io/opendatahub-operator/bde4b4e8478b5d03195e2777b9d550922e3cdcbc/components/kserve/resources/servicemesh/routing/kserve-local-gateway-svc.tmpl.yaml | + sed "s/{{ .ControlPlane.Namespace }}/istio-system/g" | + oc create -f - + oc apply -k $PROJECT_ROOT/test/scripts/openshift-ci fi @@ -156,4 +169,4 @@ kustomize build $PROJECT_ROOT/config/overlays/test/clusterresources | # no effect, because of missing Knative oc annotate servingruntimes -n kserve-ci-e2e-test --all serving.knative.openshift.io/enablePassthrough=true -echo "Setup complete" \ No newline at end of file +echo "Setup complete" From 44d0da137ea50246ffb29700dc01e4893473109f Mon Sep 17 00:00:00 2001 From: Hannah DeFazio Date: Wed, 5 Feb 2025 10:57:53 -0500 Subject: [PATCH 4/8] Comment out local images for CI --- test/scripts/openshift-ci/run-e2e-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scripts/openshift-ci/run-e2e-tests.sh b/test/scripts/openshift-ci/run-e2e-tests.sh index 04380d4635b..fde3fe208c4 100755 --- a/test/scripts/openshift-ci/run-e2e-tests.sh +++ b/test/scripts/openshift-ci/run-e2e-tests.sh @@ -29,8 +29,8 @@ popd echo "Run E2E tests: $1" pushd $PROJECT_ROOT >/dev/null # Note: The following images are set by openshift-ci. Uncomment if you are running on your own machine. -export CUSTOM_MODEL_GRPC_IMG_TAG=kserve/custom-model-grpc:latest -export IMAGE_TRANSFORMER_IMG_TAG=kserve/image-transformer:latest +#export CUSTOM_MODEL_GRPC_IMG_TAG=kserve/custom-model-grpc:latest +#export IMAGE_TRANSFORMER_IMG_TAG=kserve/image-transformer:latest export GITHUB_SHA=$(git rev-parse HEAD) export CI_USE_ISVC_HOST="1" From bf02e11accb0824199f571eb2332eed4c460fc71 Mon Sep 17 00:00:00 2001 From: Hannah DeFazio Date: Fri, 7 Feb 2025 10:37:37 -0500 Subject: [PATCH 5/8] Toggle on/off the setup commands --- test/scripts/openshift-ci/run-e2e-tests.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/scripts/openshift-ci/run-e2e-tests.sh b/test/scripts/openshift-ci/run-e2e-tests.sh index fde3fe208c4..a5be2436359 100755 --- a/test/scripts/openshift-ci/run-e2e-tests.sh +++ b/test/scripts/openshift-ci/run-e2e-tests.sh @@ -21,10 +21,13 @@ MY_PATH=$(dirname "$0") PROJECT_ROOT=$MY_PATH/../../../ -echo "Installing on cluster" -pushd $PROJECT_ROOT >/dev/null -./test/scripts/openshift-ci/setup-e2e-tests.sh "$1" -popd +SETUP_E2E=true +if $SETUP_E2E; then + echo "Installing on cluster" + pushd $PROJECT_ROOT >/dev/null + ./test/scripts/openshift-ci/setup-e2e-tests.sh "$1" + popd +fi echo "Run E2E tests: $1" pushd $PROJECT_ROOT >/dev/null From 6c02e46e5509073b3992730e1b0b692c08dfafac Mon Sep 17 00:00:00 2001 From: Hannah DeFazio Date: Fri, 7 Feb 2025 12:51:27 -0500 Subject: [PATCH 6/8] Use oc instead of kubectl --- test/scripts/openshift-ci/setup-e2e-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/openshift-ci/setup-e2e-tests.sh b/test/scripts/openshift-ci/setup-e2e-tests.sh index d782f684bad..3c7b7e7184e 100755 --- a/test/scripts/openshift-ci/setup-e2e-tests.sh +++ b/test/scripts/openshift-ci/setup-e2e-tests.sh @@ -98,7 +98,7 @@ oc wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n if [ "$1" != "raw" ]; then echo "Installing odh-model-controller" # authorino - curl -sL https://raw.githubusercontent.com/Kuadrant/authorino-operator/main/utils/install.sh | + curl -sL https://raw.githubusercontent.com/Kuadrant/authorino-operator/main/utils/install.sh | "s|kubectl|oc|" | bash -s -- -v 0.16.0 # kserve-local-gateway From bfca627cb5ad1a10374aac3ff8eec367e21fd645 Mon Sep 17 00:00:00 2001 From: Hannah DeFazio Date: Tue, 11 Feb 2025 12:25:02 -0500 Subject: [PATCH 7/8] Add shell options to error --- test/scripts/openshift-ci/run-e2e-tests.sh | 4 ++++ test/scripts/openshift-ci/setup-e2e-tests.sh | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/scripts/openshift-ci/run-e2e-tests.sh b/test/scripts/openshift-ci/run-e2e-tests.sh index a5be2436359..f021d6d6db3 100755 --- a/test/scripts/openshift-ci/run-e2e-tests.sh +++ b/test/scripts/openshift-ci/run-e2e-tests.sh @@ -18,6 +18,10 @@ # like kustomize and the mc client are installed by the script if not available. # The oc CLI is assumed to be configured with the credentials of the # target cluster. The target cluster is assumed to be a clean cluster. +set -o errexit +set -o nounset +set -o pipefail + MY_PATH=$(dirname "$0") PROJECT_ROOT=$MY_PATH/../../../ diff --git a/test/scripts/openshift-ci/setup-e2e-tests.sh b/test/scripts/openshift-ci/setup-e2e-tests.sh index 3c7b7e7184e..0b83e0fa54d 100755 --- a/test/scripts/openshift-ci/setup-e2e-tests.sh +++ b/test/scripts/openshift-ci/setup-e2e-tests.sh @@ -18,7 +18,9 @@ # like kustomize and the mc client are installed by the script if not available. # The oc CLI is assumed to be configured with the credentials of the # target cluster. The target cluster is assumed to be a clean cluster. -set -eu +set -o errexit +set -o nounset +set -o pipefail : "${SKLEARN_IMAGE:=kserve/sklearnserver:latest}" : "${KSERVE_CONTROLLER_IMAGE:=quay.io/opendatahub/kserve-controller:latest}" From 607f06adf4863146f33900286e0943c6be6dc739 Mon Sep 17 00:00:00 2001 From: Hannah DeFazio Date: Mon, 10 Feb 2025 20:02:58 -0500 Subject: [PATCH 8/8] Fix typo in sed command --- test/scripts/openshift-ci/setup-e2e-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/openshift-ci/setup-e2e-tests.sh b/test/scripts/openshift-ci/setup-e2e-tests.sh index 0b83e0fa54d..4a30c8df39d 100755 --- a/test/scripts/openshift-ci/setup-e2e-tests.sh +++ b/test/scripts/openshift-ci/setup-e2e-tests.sh @@ -100,7 +100,7 @@ oc wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n if [ "$1" != "raw" ]; then echo "Installing odh-model-controller" # authorino - curl -sL https://raw.githubusercontent.com/Kuadrant/authorino-operator/main/utils/install.sh | "s|kubectl|oc|" | + curl -sL https://raw.githubusercontent.com/Kuadrant/authorino-operator/main/utils/install.sh | sed "s|kubectl|oc|" | bash -s -- -v 0.16.0 # kserve-local-gateway