From a3e474e5ec28c47e7ddfe8201c850bc87cf41d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Hern=C3=A1ndez?= Date: Mon, 21 Oct 2024 05:07:54 -0600 Subject: [PATCH 1/3] Reduce E2Es dependency on CI environment (2) (#4008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce E2Es dependency on CI environment Some code of the E2Es assume the environment is GitHub, because it is referring to GitHub-specific variables. This PR focuses on the `kserve/custom-model-grpc` container image, so that no Python code of the E2Es using this image is referencing the `github_sha` variable. Also, a small improvement on the `get_isvc_endpoint` utility function is done to use the schema in the endpoint specified in the status of the InferenceService, rather than hard-coding to plain-text HTTP. This adds compatibility for CI environments where KServe ConfigMap has been configured with `urlScheme: https` for the Ingress. Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> --- .github/workflows/e2e-test.yml | 1 + test/e2e/common/utils.py | 23 +++++++++++---------- test/e2e/custom/test_custom_model_grpc.py | 6 +++--- test/e2e/custom/test_ray.py | 2 +- test/e2e/predictor/test_grpc.py | 2 +- test/e2e/predictor/test_raw_deployment.py | 2 +- test/e2e/predictor/test_response_headers.py | 12 +++++------ 7 files changed, 25 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 573b589dc47..2647f8e5a6b 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -34,6 +34,7 @@ env: PMML_IMG: "pmmlserver" PADDLE_IMG: "paddleserver" CUSTOM_MODEL_GRPC_IMG: "custom-model-grpc" + CUSTOM_MODEL_GRPC_IMG_TAG: "kserve/custom-model-grpc:${{ github.sha }}" HUGGINGFACE_IMG: "huggingfaceserver" # Explainer images ART_IMG: "art-explainer" diff --git a/test/e2e/common/utils.py b/test/e2e/common/utils.py index d72490d7e2c..08fe08d812a 100644 --- a/test/e2e/common/utils.py +++ b/test/e2e/common/utils.py @@ -71,10 +71,10 @@ async def predict_isvc( namespace=KSERVE_TEST_NAMESPACE, version=version, ) - cluster_ip, host, path = get_isvc_endpoint(isvc) + scheme, cluster_ip, host, path = get_isvc_endpoint(isvc) if model_name is None: model_name = service_name - base_url = f"http://{cluster_ip}{path}" + base_url = f"{scheme}://{cluster_ip}{path}" return await predict( client, base_url, @@ -167,8 +167,8 @@ async def predict_ig( namespace=KSERVE_TEST_NAMESPACE, version=version, ) - cluster_ip, host, _ = get_isvc_endpoint(ig) - url = f"http://{cluster_ip}" + scheme, cluster_ip, host, _ = get_isvc_endpoint(ig) + url = f"{scheme}://{cluster_ip}" return await predict(client, url, host, input_path, is_graph=True) @@ -191,8 +191,8 @@ async def explain_response(client, service_name, input_path) -> Dict: namespace=KSERVE_TEST_NAMESPACE, version=constants.KSERVE_V1BETA1_VERSION, ) - cluster_ip, host, path = get_isvc_endpoint(isvc) - url = f"http://{cluster_ip}{path}" + scheme, cluster_ip, host, path = get_isvc_endpoint(isvc) + url = f"{scheme}://{cluster_ip}{path}" headers = {"Host": host} with open(input_path) as json_file: data = json.load(json_file) @@ -257,7 +257,7 @@ async def predict_grpc( namespace=KSERVE_TEST_NAMESPACE, version=version, ) - _, host, _ = get_isvc_endpoint(isvc) + _, _, host, _ = get_isvc_endpoint(isvc) if model_name is None: model_name = service_name @@ -293,13 +293,14 @@ async def predict_modelmesh( def get_isvc_endpoint(isvc): + scheme = urlparse(isvc["status"]["url"]).scheme host = urlparse(isvc["status"]["url"]).netloc path = urlparse(isvc["status"]["url"]).path if os.environ.get("CI_USE_ISVC_HOST") == "1": cluster_ip = host else: cluster_ip = get_cluster_ip() - return cluster_ip, host, path + return scheme, cluster_ip, host, path def generate( @@ -319,13 +320,13 @@ def generate( namespace=KSERVE_TEST_NAMESPACE, version=version, ) - cluster_ip, host, path = get_isvc_endpoint(isvc) + scheme, cluster_ip, host, path = get_isvc_endpoint(isvc) headers = {"Host": host, "Content-Type": "application/json"} if chat_completions: - url = f"http://{cluster_ip}{path}/openai/v1/chat/completions" + url = f"{scheme}://{cluster_ip}{path}/openai/v1/chat/completions" else: - url = f"http://{cluster_ip}{path}/openai/v1/completions" + url = f"{scheme}://{cluster_ip}{path}/openai/v1/completions" logger.info("Sending Header = %s", headers) logger.info("Sending url = %s", url) logger.info("Sending request data: %s", data) diff --git a/test/e2e/custom/test_custom_model_grpc.py b/test/e2e/custom/test_custom_model_grpc.py index 8456ae7b679..a25e187e4fa 100644 --- a/test/e2e/custom/test_custom_model_grpc.py +++ b/test/e2e/custom/test_custom_model_grpc.py @@ -44,7 +44,7 @@ async def test_custom_model_grpc(): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "1Gi"}, @@ -106,7 +106,7 @@ async def test_predictor_grpc_with_transformer_grpc(): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "1Gi"}, @@ -186,7 +186,7 @@ async def test_predictor_grpc_with_transformer_http(rest_v2_client): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "1Gi"}, diff --git a/test/e2e/custom/test_ray.py b/test/e2e/custom/test_ray.py index c3bc16fe027..6e243982c24 100644 --- a/test/e2e/custom/test_ray.py +++ b/test/e2e/custom/test_ray.py @@ -37,7 +37,7 @@ async def test_custom_model_http_ray(rest_v1_client): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), # Override the entrypoint to run the model using ray command=["python", "-m", "custom_model.model_remote"], resources=V1ResourceRequirements( diff --git a/test/e2e/predictor/test_grpc.py b/test/e2e/predictor/test_grpc.py index 655e3e7210f..b173f136340 100644 --- a/test/e2e/predictor/test_grpc.py +++ b/test/e2e/predictor/test_grpc.py @@ -75,7 +75,7 @@ async def test_custom_model_grpc(): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "1Gi"}, diff --git a/test/e2e/predictor/test_raw_deployment.py b/test/e2e/predictor/test_raw_deployment.py index 47ad44219cc..db9541c2ef8 100644 --- a/test/e2e/predictor/test_raw_deployment.py +++ b/test/e2e/predictor/test_raw_deployment.py @@ -134,7 +134,7 @@ async def test_isvc_with_multiple_container_port(): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "1Gi"}, diff --git a/test/e2e/predictor/test_response_headers.py b/test/e2e/predictor/test_response_headers.py index 6892f8ce914..8a9c05dbac1 100644 --- a/test/e2e/predictor/test_response_headers.py +++ b/test/e2e/predictor/test_response_headers.py @@ -30,7 +30,7 @@ def test_predictor_headers_v1(): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), # Override the entrypoint to run the custom model rest server command=["python", "-m", "custom_model.model"], resources=V1ResourceRequirements( @@ -80,13 +80,13 @@ def test_predictor_headers_v1(): namespace=KSERVE_TEST_NAMESPACE, version=constants.KSERVE_V1BETA1_VERSION, ) - cluster_ip, host, path = get_isvc_endpoint(isvc) + scheme, cluster_ip, host, path = get_isvc_endpoint(isvc) headers = {"Host": host, "Content-Type": "application/json"} if model_name is None: model_name = service_name - url = f"http://{cluster_ip}{path}/v1/models/{model_name}:predict" + url = f"{scheme}://{cluster_ip}{path}/v1/models/{model_name}:predict" time.sleep(10) with open(input_json) as json_file: @@ -116,7 +116,7 @@ def test_predictor_headers_v2(): containers=[ V1Container( name="kserve-container", - image="kserve/custom-model-grpc:" + os.environ.get("GITHUB_SHA"), + image=os.environ.get("CUSTOM_MODEL_GRPC_IMG_TAG"), # Override the entrypoint to run the custom model rest server command=["python", "-m", "custom_model.model"], resources=V1ResourceRequirements( @@ -165,13 +165,13 @@ def test_predictor_headers_v2(): namespace=KSERVE_TEST_NAMESPACE, version=constants.KSERVE_V1BETA1_VERSION, ) - cluster_ip, host, path = get_isvc_endpoint(isvc) + scheme, cluster_ip, host, path = get_isvc_endpoint(isvc) headers = {"Host": host, "Content-Type": "application/json"} if model_name is None: model_name = service_name - url = f"http://{cluster_ip}{path}/v2/models/{model_name}/infer" + url = f"{scheme}://{cluster_ip}{path}/v2/models/{model_name}/infer" time.sleep(10) with open(input_json) as json_file: From d955f6f8b40bcedd0f012465ed1fb996147349b3 Mon Sep 17 00:00:00 2001 From: Hannah DeFazio Date: Tue, 29 Oct 2024 10:15:12 -0400 Subject: [PATCH 2/3] Revert 'Increase memory limit of kserve-controller pod' --- config/overlays/odh/set-resources-manager-patch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/overlays/odh/set-resources-manager-patch.yaml b/config/overlays/odh/set-resources-manager-patch.yaml index 7e65bb2137c..164e0cb5ad1 100644 --- a/config/overlays/odh/set-resources-manager-patch.yaml +++ b/config/overlays/odh/set-resources-manager-patch.yaml @@ -11,4 +11,4 @@ spec: resources: limits: cpu: 500m - memory: 5Gi + memory: 500Mi From 4f82fe433a0d923ce0122a1a0636887763b75aa3 Mon Sep 17 00:00:00 2001 From: Jooho Lee Date: Mon, 4 Nov 2024 12:28:36 -0500 Subject: [PATCH 3/3] Fix issues after 0.14.0 sync (#424) * remove patch for webhookconfiguration Signed-off-by: jooho lee * comment out localmodel for now, this need to be reverted Signed-off-by: jooho lee * add dsc/dsci objects for e2e test Signed-off-by: jooho lee * fix e2e test Signed-off-by: jooho lee * follow up comments Signed-off-by: jooho lee * fix e2e test after latest sync Signed-off-by: jooho lee --------- Signed-off-by: jooho lee --- ...er.opendatahub.io_datascienceclusters.yaml | 763 ++++++++++++++++++ ...ion.opendatahub.io_dscinitializations.yaml | 311 +++++++ config/crd/external/kustomization.yaml | 6 + config/default/kustomization.yaml | 29 +- ...erenceservice-openshift-ci-serverless.yaml | 23 + config/overlays/test/dsc.yaml | 42 + config/overlays/test/dsci.yaml | 25 + config/overlays/test/kustomization.yaml | 5 +- test/e2e/custom/test_custom_model_grpc.py | 3 + test/e2e/custom/test_ray.py | 1 + test/e2e/explainer/test_art_explainer.py | 2 + test/e2e/predictor/test_autoscaling.py | 4 +- test/e2e/predictor/test_grpc.py | 2 + test/e2e/predictor/test_lightgbm.py | 1 + test/e2e/predictor/test_pmml.py | 2 + test/e2e/predictor/test_torchserve.py | 1 + test/e2e/predictor/test_xgboost.py | 1 + .../gh-actions/check-poetry-lockfile.sh | 2 +- test/scripts/openshift-ci/run-e2e-tests.sh | 68 +- 19 files changed, 1247 insertions(+), 44 deletions(-) create mode 100644 config/crd/external/datasciencecluster.opendatahub.io_datascienceclusters.yaml create mode 100644 config/crd/external/dscinitialization.opendatahub.io_dscinitializations.yaml create mode 100644 config/crd/external/kustomization.yaml create mode 100644 config/overlays/test/configmap/inferenceservice-openshift-ci-serverless.yaml create mode 100644 config/overlays/test/dsc.yaml create mode 100644 config/overlays/test/dsci.yaml diff --git a/config/crd/external/datasciencecluster.opendatahub.io_datascienceclusters.yaml b/config/crd/external/datasciencecluster.opendatahub.io_datascienceclusters.yaml new file mode 100644 index 00000000000..41d1c76d658 --- /dev/null +++ b/config/crd/external/datasciencecluster.opendatahub.io_datascienceclusters.yaml @@ -0,0 +1,763 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.1 + name: datascienceclusters.datasciencecluster.opendatahub.io +spec: + group: datasciencecluster.opendatahub.io + names: + kind: DataScienceCluster + listKind: DataScienceClusterList + plural: datascienceclusters + shortNames: + - dsc + singular: datasciencecluster + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: DataScienceCluster is the Schema for the datascienceclusters + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: DataScienceClusterSpec defines the desired state of the cluster. + properties: + components: + description: Override and fine tune specific component configurations. + properties: + codeflare: + description: |- + CodeFlare component configuration. + If CodeFlare Operator has been installed in the cluster, it should be uninstalled first before enabled component. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + dashboard: + description: Dashboard component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + datasciencepipelines: + description: |- + DataServicePipeline component configuration. + Require OpenShift Pipelines Operator to be installed before enable component + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + kserve: + description: |- + Kserve component configuration. + Require OpenShift Serverless and OpenShift Service Mesh Operators to be installed before enable component + Does not support enabled ModelMeshServing at the same time + properties: + defaultDeploymentMode: + description: |- + Configures the default deployment mode for Kserve. This can be set to 'Serverless' or 'RawDeployment'. + The value specified in this field will be used to set the default deployment mode in the 'inferenceservice-config' configmap for Kserve. + This field is optional. If no default deployment mode is specified, Kserve will use Serverless mode. + enum: + - Serverless + - RawDeployment + pattern: ^(Serverless|RawDeployment)$ + type: string + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + serving: + description: |- + Serving configures the KNative-Serving stack used for model serving. A Service + Mesh (Istio) is prerequisite, since it is used as networking layer. + properties: + ingressGateway: + description: |- + IngressGateway allows to customize some parameters for the Istio Ingress Gateway + that is bound to KNative-Serving. + properties: + certificate: + description: |- + Certificate specifies configuration of the TLS certificate securing communication + for the gateway. + properties: + secretName: + description: |- + SecretName specifies the name of the Kubernetes Secret resource that contains a + TLS certificate secure HTTP communications for the KNative network. + type: string + type: + default: OpenshiftDefaultIngress + description: |- + Type specifies if the TLS certificate should be generated automatically, or if the certificate + is provided by the user. Allowed values are: + * SelfSigned: A certificate is going to be generated using an own private key. + * Provided: Pre-existence of the TLS Secret (see SecretName) with a valid certificate is assumed. + * OpenshiftDefaultIngress: Default ingress certificate configured for OpenShift + enum: + - SelfSigned + - Provided + - OpenshiftDefaultIngress + type: string + type: object + domain: + description: |- + Domain specifies the host name for intercepting incoming requests. + Most likely, you will want to use a wildcard name, like *.example.com. + If not set, the domain of the OpenShift Ingress is used. + If you choose to generate a certificate, this is the domain used for the certificate request. + type: string + type: object + managementState: + default: Managed + enum: + - Managed + - Unmanaged + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + name: + default: knative-serving + description: |- + Name specifies the name of the KNativeServing resource that is going to be + created to instruct the KNative Operator to deploy KNative serving components. + This resource is created in the "knative-serving" namespace. + type: string + type: object + type: object + kueue: + description: Kueue component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + modelmeshserving: + description: |- + ModelMeshServing component configuration. + Does not support enabled Kserve at the same time + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + modelregistry: + description: ModelRegistry component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + registriesNamespace: + default: odh-model-registries + description: Namespace for model registries to be installed, + configurable only once when model registry is enabled, defaults + to "odh-model-registries" + maxLength: 63 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?)?$ + type: string + type: object + x-kubernetes-validations: + - message: RegistriesNamespace is immutable when model registry + is Managed + rule: (self.managementState != 'Managed') || (oldSelf.registriesNamespace + == '') || (oldSelf.managementState != 'Managed')|| (self.registriesNamespace + == oldSelf.registriesNamespace) + ray: + description: Ray component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + trainingoperator: + description: Training Operator component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + trustyai: + description: TrustyAI component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + workbenches: + description: Workbenches component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to + the folder containing manifests in a repository, + default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: |- + Set to one of the following values: + + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so + + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + type: object + type: object + status: + description: DataScienceClusterStatus defines the observed state of DataScienceCluster. + properties: + components: + description: Expose component's specific status + properties: + modelregistry: + description: ModelRegistry component status + properties: + registriesNamespace: + type: string + type: object + type: object + conditions: + description: Conditions describes the state of the DataScienceCluster + resource. + items: + description: |- + Condition represents the state of the operator's + reconciliation functionality. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: ConditionType is the state of the operator's reconciliation + functionality. + type: string + required: + - status + - type + type: object + type: array + errorMessage: + type: string + installedComponents: + additionalProperties: + type: boolean + description: List of components with status if installed or not + type: object + phase: + description: |- + Phase describes the Phase of DataScienceCluster reconciliation state + This is used by OLM UI to provide status information to the user + type: string + relatedObjects: + description: |- + RelatedObjects is a list of objects created and maintained by this operator. + Object references will be added to this list after they have been created AND found in the cluster. + items: + description: ObjectReference contains enough information to let + you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + release: + description: Version and release type + properties: + name: + type: string + version: + type: string + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/external/dscinitialization.opendatahub.io_dscinitializations.yaml b/config/crd/external/dscinitialization.opendatahub.io_dscinitializations.yaml new file mode 100644 index 00000000000..894772ea9a9 --- /dev/null +++ b/config/crd/external/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -0,0 +1,311 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.1 + name: dscinitializations.dscinitialization.opendatahub.io +spec: + group: dscinitialization.opendatahub.io + names: + kind: DSCInitialization + listKind: DSCInitializationList + plural: dscinitializations + shortNames: + - dsci + singular: dscinitialization + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Current Phase + jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Created At + type: string + name: v1 + schema: + openAPIV3Schema: + description: DSCInitialization is the Schema for the dscinitializations API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: DSCInitializationSpec defines the desired state of DSCInitialization. + properties: + applicationsNamespace: + default: opendatahub + description: Namespace for applications to be installed, non-configurable, + default to "opendatahub" + maxLength: 63 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?)?$ + type: string + x-kubernetes-validations: + - message: ApplicationsNamespace is immutable + rule: self == oldSelf + devFlags: + description: |- + Internal development useful field to test customizations. + This is not recommended to be used in production environment. + properties: + logLevel: + description: Override Zap log level. Can be "debug", "info", "error" + or a number (more verbose). + type: string + logmode: + default: production + description: '## DEPRECATED ##: Ignored, use LogLevel instead' + enum: + - devel + - development + - prod + - production + - default + type: string + manifestsUri: + description: Custom manifests uri for odh-manifests + type: string + type: object + monitoring: + description: Enable monitoring on specified namespace + properties: + managementState: + description: |- + Set to one of the following values: + - "Managed" : the operator is actively managing the component and trying to keep it active. + It will only upgrade the component if it is safe to do so. + - "Removed" : the operator is actively managing the component and will not install it, + or if it is installed, the operator will try to remove it. + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + namespace: + default: opendatahub + description: Namespace for monitoring if it is enabled + maxLength: 63 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?)?$ + type: string + type: object + serviceMesh: + description: |- + Configures Service Mesh as networking layer for Data Science Clusters components. + The Service Mesh is a mandatory prerequisite for single model serving (KServe) and + you should review this configuration if you are planning to use KServe. + For other components, it enhances user experience; e.g. it provides unified + authentication giving a Single Sign On experience. + properties: + auth: + description: |- + Auth holds configuration of authentication and authorization services + used by Service Mesh in Opendatahub. + properties: + audiences: + default: + - https://kubernetes.default.svc + description: |- + Audiences is a list of the identifiers that the resource server presented + with the token identifies as. Audience-aware token authenticators will verify + that the token was intended for at least one of the audiences in this list. + If no audiences are provided, the audience will default to the audience of the + Kubernetes apiserver (kubernetes.default.svc). + items: + type: string + type: array + namespace: + description: |- + Namespace where it is deployed. If not provided, the default is to + use '-auth-provider' suffix on the ApplicationsNamespace of the DSCI. + maxLength: 63 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?)?$ + type: string + type: object + controlPlane: + description: ControlPlane holds configuration of Service Mesh + used by Opendatahub. + properties: + metricsCollection: + default: Istio + description: |- + MetricsCollection specifies if metrics from components on the Mesh namespace + should be collected. Setting the value to "Istio" will collect metrics from the + control plane and any proxies on the Mesh namespace (like gateway pods). Setting + to "None" will disable metrics collection. + enum: + - Istio + - None + type: string + name: + default: data-science-smcp + description: Name is a name Service Mesh Control Plane. Defaults + to "data-science-smcp". + type: string + namespace: + default: istio-system + description: Namespace is a namespace where Service Mesh is + deployed. Defaults to "istio-system". + maxLength: 63 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?)?$ + type: string + type: object + managementState: + default: Removed + enum: + - Managed + - Unmanaged + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + trustedCABundle: + description: |- + When set to `Managed`, adds odh-trusted-ca-bundle Configmap to all namespaces that includes + cluster-wide Trusted CA Bundle in .data["ca-bundle.crt"]. + Additionally, this fields allows admins to add custom CA bundles to the configmap using the .CustomCABundle field. + properties: + customCABundle: + default: "" + description: |- + A custom CA bundle that will be available for all components in the + Data Science Cluster(DSC). This bundle will be stored in odh-trusted-ca-bundle + ConfigMap .data.odh-ca-bundle.crt . + type: string + managementState: + default: Removed + description: managementState indicates whether and how the operator + should manage customized CA bundle + enum: + - Managed + - Removed + - Unmanaged + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + required: + - customCABundle + - managementState + type: object + required: + - applicationsNamespace + type: object + status: + description: DSCInitializationStatus defines the observed state of DSCInitialization. + properties: + conditions: + description: Conditions describes the state of the DSCInitializationStatus + resource + items: + description: |- + Condition represents the state of the operator's + reconciliation functionality. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: ConditionType is the state of the operator's reconciliation + functionality. + type: string + required: + - status + - type + type: object + type: array + errorMessage: + type: string + phase: + description: |- + Phase describes the Phase of DSCInitializationStatus + This is used by OLM UI to provide status information to the user + type: string + relatedObjects: + description: |- + RelatedObjects is a list of objects created and maintained by this operator. + Object references will be added to this list after they have been created AND found in the cluster + items: + description: ObjectReference contains enough information to let + you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + release: + description: Version and release type + properties: + name: + type: string + version: + type: string + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/external/kustomization.yaml b/config/crd/external/kustomization.yaml new file mode 100644 index 00000000000..1742a6c2515 --- /dev/null +++ b/config/crd/external/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - dscinitialization.opendatahub.io_dscinitializations.yaml + - datasciencecluster.opendatahub.io_datascienceclusters.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 4685e57f4dd..f57fd23675c 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -18,7 +18,7 @@ resources: - ../manager - ../webhook # - ../certmanager # not needed, because ODH is using OpenShift's serving certificates for WebHooks -- ../localmodels +# - ../localmodels # This is commented out for 2.16 - network-policies.yaml # ODH specific generatorOptions: @@ -197,7 +197,7 @@ patches: - path: svc_webhook_cainjection_patch.yaml - path: manager_resources_patch.yaml - path: cainjection_conversion_webhook.yaml -- path: localmodel_manager_image_patch.yaml +# - path: localmodel_manager_image_patch.yaml # Since OpenShift serving-certificates are being used, # remove CA bundle placeholders # - patch: |- @@ -206,16 +206,15 @@ patches: # target: # kind: CustomResourceDefinition # name: inferenceservices.serving.kserve.io -- patch: |- - - op: remove - path: "/webhooks/0/clientConfig/caBundle" - - op: remove - path: "/webhooks/1/clientConfig/caBundle" - target: - kind: MutatingWebhookConfiguration -- patch: |- - - op: remove - path: "/webhooks/0/clientConfig/caBundle" - target: - kind: ValidatingWebhookConfiguration - +# - patch: |- +# - op: remove +# path: "/webhooks/0/clientConfig/caBundle" +# - op: remove +# path: "/webhooks/1/clientConfig/caBundle" +# target: +# kind: MutatingWebhookConfiguration +# - patch: |- +# - op: remove +# path: "/webhooks/0/clientConfig/caBundle" +# target: +# kind: ValidatingWebhookConfiguration diff --git a/config/overlays/test/configmap/inferenceservice-openshift-ci-serverless.yaml b/config/overlays/test/configmap/inferenceservice-openshift-ci-serverless.yaml new file mode 100644 index 00000000000..1020c48de00 --- /dev/null +++ b/config/overlays/test/configmap/inferenceservice-openshift-ci-serverless.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: inferenceservice-config + namespace: kserve +data: + deploy: |- + { + "defaultDeploymentMode": "Serverless" + } + ingress: |- + { + "ingressGateway" : "knative-serving/knative-ingress-gateway", + "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local", + "localGateway" : "knative-serving/knative-local-gateway", + "localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local", + "ingressDomain" : "$OPENSHIFT_INGRESS_DOMAIN", + "ingressClassName" : "openshift-default", + "domainTemplate": "{{ .Name }}-{{ .Namespace }}.{{ .IngressDomain }}", + "urlScheme": "http", + "disableIstioVirtualHost": false, + "disableIngressCreation": false + } diff --git a/config/overlays/test/dsc.yaml b/config/overlays/test/dsc.yaml new file mode 100644 index 00000000000..f3b9cb7ec33 --- /dev/null +++ b/config/overlays/test/dsc.yaml @@ -0,0 +1,42 @@ +kind: DataScienceCluster +apiVersion: datasciencecluster.opendatahub.io/v1 +metadata: + name: test-dsc + labels: + app.kubernetes.io/name: datasciencecluster + app.kubernetes.io/instance: rhods + app.kubernetes.io/part-of: rhods-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: rhods-operator +spec: + components: + codeflare: + managementState: Removed + dashboard: + managementState: Removed + datasciencepipelines: + managementState: Removed + kserve: + defaultDeploymentMode: Serverless + managementState: Managed + serving: + ingressGateway: + certificate: + type: OpenshiftDefaultIngress + managementState: Managed + name: knative-serving + modelmeshserving: + managementState: Removed + ray: + managementState: Removed + trustyai: + managementState: Removed + workbenches: + managementState: Removed + trainingoperator: + managementState: Removed + kueue: + managementState: Removed + modelregistry: + registriesNamespace: odh-model-registries + managementState: Removed diff --git a/config/overlays/test/dsci.yaml b/config/overlays/test/dsci.yaml new file mode 100644 index 00000000000..419347080e6 --- /dev/null +++ b/config/overlays/test/dsci.yaml @@ -0,0 +1,25 @@ +apiVersion: dscinitialization.opendatahub.io/v1 +kind: DSCInitialization +metadata: + name: test-dsci + namespace: kserve + labels: + app.kubernetes.io/created-by: rhods-operator + app.kubernetes.io/instance: default-dsci + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: dscinitialization + app.kubernetes.io/part-of: rhods-operator +spec: + applicationsNamespace: kserve + monitoring: + namespace: redhat-ods-applications + managementState: Removed + serviceMesh: + controlPlane: + metricsCollection: Istio + name: basic + namespace: istio-system + managementState: Removed + trustedCABundle: + managementState: Removed + customCABundle: "" diff --git a/config/overlays/test/kustomization.yaml b/config/overlays/test/kustomization.yaml index 0ffe7ff9e09..6cb3ee97c39 100644 --- a/config/overlays/test/kustomization.yaml +++ b/config/overlays/test/kustomization.yaml @@ -2,10 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: +- ../../crd/external - ../../default - minio +- dsci.yaml +- dsc.yaml patches: - path: configmap/inferenceservice.yaml - path: manager_image_patch.yaml -- path: localmodel_manager_image_patch.yaml +# - path: localmodel_manager_image_patch.yaml diff --git a/test/e2e/custom/test_custom_model_grpc.py b/test/e2e/custom/test_custom_model_grpc.py index a25e187e4fa..c7719f82db4 100644 --- a/test/e2e/custom/test_custom_model_grpc.py +++ b/test/e2e/custom/test_custom_model_grpc.py @@ -33,6 +33,7 @@ from ..common.utils import KSERVE_TEST_NAMESPACE, predict_isvc, predict_grpc +@pytest.mark.skip(reason="Not testable in ODH at the moment") @pytest.mark.grpc @pytest.mark.predictor @pytest.mark.asyncio(scope="session") @@ -95,6 +96,7 @@ async def test_custom_model_grpc(): kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) +@pytest.mark.skip(reason="Not testable in ODH at the moment") @pytest.mark.grpc @pytest.mark.transformer @pytest.mark.asyncio(scope="session") @@ -175,6 +177,7 @@ async def test_predictor_grpc_with_transformer_grpc(): kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) +@pytest.mark.skip(reason="Not testable in ODH at the moment") @pytest.mark.grpc @pytest.mark.transformer @pytest.mark.asyncio(scope="session") diff --git a/test/e2e/custom/test_ray.py b/test/e2e/custom/test_ray.py index 6e243982c24..a5426f5a885 100644 --- a/test/e2e/custom/test_ray.py +++ b/test/e2e/custom/test_ray.py @@ -27,6 +27,7 @@ from ..common.utils import KSERVE_TEST_NAMESPACE, predict_isvc +@pytest.mark.skip(reason="Not testable in ODH at the moment") @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_custom_model_http_ray(rest_v1_client): diff --git a/test/e2e/explainer/test_art_explainer.py b/test/e2e/explainer/test_art_explainer.py index f83b1ac905c..0d5a1147426 100644 --- a/test/e2e/explainer/test_art_explainer.py +++ b/test/e2e/explainer/test_art_explainer.py @@ -34,6 +34,8 @@ kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config")) +pytest.skip("ODH does not support art explainer at the moment", allow_module_level=True) + @pytest.mark.path_based_routing @pytest.mark.explainer diff --git a/test/e2e/predictor/test_autoscaling.py b/test/e2e/predictor/test_autoscaling.py index 0331b279ac1..42ed5f8e2f6 100644 --- a/test/e2e/predictor/test_autoscaling.py +++ b/test/e2e/predictor/test_autoscaling.py @@ -281,7 +281,9 @@ async def test_sklearn_rolling_update(): namespace=KSERVE_TEST_NAMESPACE, label_selector="serving.kserve.io/test=rolling-update", ) - kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) + kserve_client.wait_isvc_ready( + service_name, namespace=KSERVE_TEST_NAMESPACE, timeout_seconds=600 + ) # Check if the deployment replicas still remain the same as min_replicas assert deployment.items[0].spec.replicas == min_replicas diff --git a/test/e2e/predictor/test_grpc.py b/test/e2e/predictor/test_grpc.py index b173f136340..c0a075954d9 100644 --- a/test/e2e/predictor/test_grpc.py +++ b/test/e2e/predictor/test_grpc.py @@ -31,6 +31,8 @@ from kubernetes.client import V1Container, V1ContainerPort from ..common.utils import KSERVE_TEST_NAMESPACE, predict_grpc +pytest.skip("Not testable in ODH at the moment", allow_module_level=True) + @pytest.mark.grpc @pytest.mark.predictor diff --git a/test/e2e/predictor/test_lightgbm.py b/test/e2e/predictor/test_lightgbm.py index e0ffdfebe0b..32607b98465 100644 --- a/test/e2e/predictor/test_lightgbm.py +++ b/test/e2e/predictor/test_lightgbm.py @@ -229,6 +229,7 @@ async def test_lightgbm_v2_kserve(rest_v2_client): kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) +@pytest.mark.skip(reason="Not testable in ODH at the moment") @pytest.mark.grpc @pytest.mark.predictor @pytest.mark.asyncio(scope="session") diff --git a/test/e2e/predictor/test_pmml.py b/test/e2e/predictor/test_pmml.py index 819c65d424a..07eed3051eb 100644 --- a/test/e2e/predictor/test_pmml.py +++ b/test/e2e/predictor/test_pmml.py @@ -198,6 +198,8 @@ async def test_pmml_v2_kserve(rest_v2_client): kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) +@pytest.mark.skip(reason="Not testable in ODH at the moment") +@pytest.mark.grpc @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_pmml_v2_grpc(): diff --git a/test/e2e/predictor/test_torchserve.py b/test/e2e/predictor/test_torchserve.py index ef1f3f57034..65fcc1deb6a 100644 --- a/test/e2e/predictor/test_torchserve.py +++ b/test/e2e/predictor/test_torchserve.py @@ -113,6 +113,7 @@ async def test_torchserve_v2_kserve(rest_v2_client): kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) +@pytest.mark.skip(reason="Not testable in ODH at the moment") @pytest.mark.grpc @pytest.mark.predictor @pytest.mark.asyncio(scope="session") diff --git a/test/e2e/predictor/test_xgboost.py b/test/e2e/predictor/test_xgboost.py index 21f89032081..0c07dd8cad8 100644 --- a/test/e2e/predictor/test_xgboost.py +++ b/test/e2e/predictor/test_xgboost.py @@ -256,6 +256,7 @@ async def test_xgboost_v2(rest_v2_client): kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) +@pytest.mark.skip(reason="Not testable in ODH at the moment") @pytest.mark.grpc @pytest.mark.predictor @pytest.mark.asyncio(scope="session") diff --git a/test/scripts/gh-actions/check-poetry-lockfile.sh b/test/scripts/gh-actions/check-poetry-lockfile.sh index 73d31357401..6f7a5758184 100755 --- a/test/scripts/gh-actions/check-poetry-lockfile.sh +++ b/test/scripts/gh-actions/check-poetry-lockfile.sh @@ -29,7 +29,7 @@ packages=() # Read the output of find into an array while IFS= read -r -d '' folder; do packages+=("$folder") -done < <(find . -type f -name "pyproject.toml" -print0) +done < <(find . -type d -name ".venv" -prune -o -type f -name "pyproject.toml" -print0) for file in "${packages[@]}" do diff --git a/test/scripts/openshift-ci/run-e2e-tests.sh b/test/scripts/openshift-ci/run-e2e-tests.sh index 85497b7816b..ee0ee0dd249 100755 --- a/test/scripts/openshift-ci/run-e2e-tests.sh +++ b/test/scripts/openshift-ci/run-e2e-tests.sh @@ -40,26 +40,25 @@ MY_PATH=$(dirname "$0") PROJECT_ROOT=$MY_PATH/../../../ # If Kustomize is not installed, install it -if ! command -v kustomize &> /dev/null; then +if ! command -v kustomize &>/dev/null; then echo "Installing Kustomize" - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 5.0.1 $HOME/.local/bin + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 5.0.1 $HOME/.local/bin fi # If minio CLI is not installed, install it -if ! command -v mc &> /dev/null; then +if ! command -v mc &>/dev/null; then echo "Installing Minio CLI" curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o $HOME/.local/bin/mc chmod +x $HOME/.local/bin/mc fi -# echo "Installing KServe Python SDK ..." pushd $PROJECT_ROOT >/dev/null ./test/scripts/gh-actions/setup-poetry.sh ./test/scripts/gh-actions/check-poetry-lockfile.sh popd pushd $PROJECT_ROOT/python/kserve >/dev/null - poetry install --with=test --no-interaction + poetry install --with=test --no-interaction popd # Install KServe stack @@ -71,19 +70,24 @@ if [ "$1" != "raw" ]; then fi echo "Installing KServe with Minio" -kustomize build $PROJECT_ROOT/config/overlays/test | \ - sed "s|kserve/storage-initializer:latest|${STORAGE_INITIALIZER_IMAGE}|" | \ - sed "s|kserve/agent:latest|${KSERVE_AGENT_IMAGE}|" | \ - sed "s|kserve/router:latest|${KSERVE_ROUTER_IMAGE}|" | \ - sed "s|kserve/kserve-controller:latest|${KSERVE_CONTROLLER_IMAGE}|" | \ - oc apply -f - +kustomize build $PROJECT_ROOT/config/overlays/test | + sed "s|kserve/storage-initializer:latest|${STORAGE_INITIALIZER_IMAGE}|" | + sed "s|kserve/agent:latest|${KSERVE_AGENT_IMAGE}|" | + sed "s|kserve/router:latest|${KSERVE_ROUTER_IMAGE}|" | + sed "s|kserve/kserve-controller:latest|${KSERVE_CONTROLLER_IMAGE}|" | + oc apply --server-side=true -f - # Patch the inferenceservice-config ConfigMap, when running RawDeployment tests - if [ "$1" == "raw" ]; then +if [ "$1" == "raw" ]; then export OPENSHIFT_INGRESS_DOMAIN=$(oc get ingresses.config cluster -o jsonpath='{.spec.domain}') oc patch configmap inferenceservice-config -n kserve --patch-file <(cat config/overlays/test/configmap/inferenceservice-openshift-ci-raw.yaml | envsubst) oc delete pod -n kserve -l control-plane=kserve-controller-manager - fi + + oc patch DataScienceCluster test-dsc --type='json' -p='[{"op": "replace", "path": "/spec/components/kserve/defaultDeploymentMode", "value": "RawDeployment"}]' +else + export OPENSHIFT_INGRESS_DOMAIN=$(oc get ingresses.config cluster -o jsonpath='{.spec.domain}') + oc patch configmap inferenceservice-config -n kserve --patch-file <(cat config/overlays/test/configmap/inferenceservice-openshift-ci-serverless.yaml | envsubst) +fi # Wait until KServe starts oc wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n kserve --timeout=300s @@ -94,15 +98,26 @@ if [ "$1" != "raw" ]; then fi echo "Add testing models to minio storage ..." # Reference: config/overlays/test/minio/minio-init-job.yaml -curl -L https://storage.googleapis.com/kfserving-examples/models/sklearn/1.0/model/model.joblib -o /tmp/sklearn-model.joblib oc expose service minio-service -n kserve && sleep 5 MINIO_ROUTE=$(oc get routes -n kserve minio-service -o jsonpath="{.spec.host}") mc alias set storage http://$MINIO_ROUTE minio minio123 -mc mb storage/example-models -mc cp /tmp/sklearn-model.joblib storage/example-models/sklearn/model.joblib + +if ! mc ls storage/example-models >/dev/null 2>&1; then + mc mb storage/example-models +else + echo "Bucket 'example-models' already exists." +fi + +if [[ $(mc ls storage/example-models/sklearn/model.joblib |wc -l) == "1" ]]; then + echo "Test model exists" +else + echo "Copy test model" + curl -L https://storage.googleapis.com/kfserving-examples/models/sklearn/1.0/model/model.joblib -o /tmp/sklearn-model.joblib + mc cp /tmp/sklearn-model.joblib storage/example-models/sklearn/model.joblib +fi + oc delete route -n kserve minio-service -# echo "Prepare CI namespace and install ServingRuntimes" cat </dev/null - # Note: The following images are set by openshift-ci. Uncomment if you are running on your own machine. - # export CUSTOM_MODEL_GRPC_IMG_TAG=kserve/custom-model-grpc:latest - # export IMAGE_TRANSFORMER_IMG_TAG=kserve/image-transformer:latest +# Note: The following images are set by openshift-ci. Uncomment if you are running on your own machine. +# export CUSTOM_MODEL_GRPC_IMG_TAG=kserve/custom-model-grpc:latest +# export IMAGE_TRANSFORMER_IMG_TAG=kserve/image-transformer:latest - export GITHUB_SHA=$(git rev-parse HEAD) - export CI_USE_ISVC_HOST="1" - ./test/scripts/gh-actions/run-e2e-tests.sh "$1" +export GITHUB_SHA=$(git rev-parse HEAD) +export CI_USE_ISVC_HOST="1" +./test/scripts/gh-actions/run-e2e-tests.sh "$1" popd