Allow KServe to have its own local gateways for Serverless mode

charts/kserve-resources/README.md

@@ -26,9 +26,9 @@ $ helm install kserve oci://ghcr.io/kserve/charts/kserve --version v0.12.1
 | kserve.controller.gateway.domainTemplate | string | `"{{ .Name }}-{{ .Namespace }}.{{ .IngressDomain }}"` |  |
 | kserve.controller.gateway.ingressGateway.className | string | `"istio"` |  |
 | kserve.controller.gateway.ingressGateway.gateway | string | `"knative-serving/knative-ingress-gateway"` |  |
-| kserve.controller.gateway.ingressGateway.gatewayService | string | `"istio-ingressgateway.istio-system.svc.cluster.local"` |  |
 | kserve.controller.gateway.localGateway.gateway | string | `"knative-serving/knative-local-gateway"` |  |
 | kserve.controller.gateway.localGateway.gatewayService | string | `"knative-local-gateway.istio-system.svc.cluster.local"` |  |
+| kserve.controller.gateway.localGateway.knativeGatewayService | string | `""` |  |
 | kserve.controller.gateway.urlScheme | string | `"http"` |  |
 | kserve.controller.image | string | `"kserve/kserve-controller"` |  |
 | kserve.controller.nodeSelector | object | `{}` |  |

charts/kserve-resources/templates/configmap.yaml

@@ -193,7 +193,6 @@ data:
      ingress: |-
        {
            "ingressGateway" : "knative-serving/knative-ingress-gateway",
-           "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
            "localGateway" : "knative-serving/knative-local-gateway",
            "localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
            "ingressDomain"  : "example.com",
@@ -210,9 +209,15 @@ data:
            # NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
            "ingressGateway" : "knative-serving/knative-ingress-gateway",
 
-           # ingressService specifies the hostname of the ingress service.
+           # knativeLocalGatewayService specifies the hostname of the Knative's local gateway serivice.
+           # The default KServe configurations are re-using the Istio local gateways for Knative. In this case, this
+           # knativeLocalGatewayService field can be left unset. When unset, the value of "localGatewayService" will be used.
+           # However, sometimes it may be better to have local gateways specifically for KServe (e.g. when enabling strict mTLS in Istio).
+           # Under such setups where KServe is needed to have its own local gateways, the values of the "localGateway" and
+           # "localGatewayService" should point to the KServe local gateways. Then, this knativeLocalGatewayService field
+           # should point to the Knative's local gateway service.
            # NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
-           "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
+           "knativeLocalGatewayService": "",
 
            # localGateway specifies the gateway which handles the network traffic within the cluster.
            # NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
@@ -507,7 +512,7 @@ data:
   ingress: |-
     {
         "ingressGateway" : "{{ .Values.kserve.controller.gateway.ingressGateway.gateway }}",
-        "ingressService" : "{{ .Values.kserve.controller.gateway.ingressGateway.gatewayService }}",
+        "knativeLocalGatewayService" : "{{ .Values.kserve.controller.gateway.localGateway.knativeGatewayService }}",
         "localGateway" : "{{ .Values.kserve.controller.gateway.localGateway.gateway }}",
         "localGatewayService" : "{{ .Values.kserve.controller.gateway.localGateway.gatewayService }}",
         "ingressClassName" : "{{ .Values.kserve.controller.gateway.ingressGateway.className }}",

charts/kserve-resources/values.yaml

@@ -42,9 +42,9 @@ kserve:
       localGateway:
         gateway: knative-serving/knative-local-gateway
         gatewayService: knative-local-gateway.istio-system.svc.cluster.local
+        knativeGatewayService: ""
       ingressGateway:
         gateway: knative-serving/knative-ingress-gateway
-        gatewayService: istio-ingressgateway.istio-system.svc.cluster.local
         className: istio
     nodeSelector: {}
     tolerations: []

config/configmap/inferenceservice.yaml

@@ -204,7 +204,6 @@ data:
      ingress: |-
        {
            "ingressGateway" : "knative-serving/knative-ingress-gateway",
-           "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
            "localGateway" : "knative-serving/knative-local-gateway",
            "localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
            "ingressDomain"  : "example.com",
@@ -221,9 +220,15 @@ data:
            # NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
            "ingressGateway" : "knative-serving/knative-ingress-gateway",
 
-           # ingressService specifies the hostname of the ingress service.
+           # knativeLocalGatewayService specifies the hostname of the Knative's local gateway serivice.
+           # The default KServe configurations are re-using the Istio local gateways for Knative. In this case, this
+           # knativeLocalGatewayService field can be left unset. When unset, the value of "localGatewayService" will be used.
+           # However, sometimes it may be better to have local gateways specifically for KServe (e.g. when enabling strict mTLS in Istio).
+           # Under such setups where KServe is needed to have its own local gateways, the values of the "localGateway" and
+           # "localGatewayService" should point to the KServe local gateways. Then, this knativeLocalGatewayService field
+           # should point to the Knative's local gateway service.
            # NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
-           "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
+           "knativeLocalGatewayService": "",
 
            # localGateway specifies the gateway which handles the network traffic within the cluster.
            # NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
@@ -506,7 +511,6 @@ data:
   ingress: |-
     {
         "ingressGateway" : "knative-serving/knative-ingress-gateway",
-        "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
         "localGateway" : "knative-serving/knative-local-gateway",
         "localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
         "ingressDomain"  : "example.com",

config/overlays/odh/inferenceservice-config-patch.yaml

@@ -12,19 +12,25 @@ data:
         "memoryLimit": "1Gi",
         "cpuRequest": "100m",
         "cpuLimit": "1",
-        "enableDirectPvcVolumeMount": true
+        "caBundleConfigMapName": "",
+        "caBundleVolumeMountPath": "/etc/ssl/custom-certs",
+        "enableDirectPvcVolumeMount": true,
+        "enableModelcar": false,
+        "cpuModelcar": "10m",
+        "memoryModelcar": "15Mi"
     }
   ingress: |-
     {
         "ingressGateway" : "knative-serving/knative-ingress-gateway",
-        "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
+        "knativeLocalGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
+        "ingressService" : "need.to.delete",
         "localGateway" : "istio-system/kserve-local-gateway",
         "localGatewayService" : "kserve-local-gateway.istio-system.svc.cluster.local",
         "ingressDomain"  : "example.com",
         "ingressClassName" : "istio",
         "domainTemplate": "{{ .Name }}-{{ .Namespace }}.{{ .IngressDomain }}",
         "urlScheme": "https",
-        "disableIstioVirtualHost": false, 
+        "disableIstioVirtualHost": false,
         "disableIngressCreation": true
     }
   logger: |-
@@ -42,7 +48,9 @@ data:
         "memoryRequest": "1Gi",
         "memoryLimit": "1Gi",
         "cpuRequest": "1",
-        "cpuLimit": "1"
+        "cpuLimit": "1",
+        "maxBatchSize": "32",
+        "maxLatency": "5000"
     }
   agent: |-
     {
@@ -60,12 +68,3 @@ data:
         "cpuRequest": "100m",
         "cpuLimit": "1"
     }
-  deploy: |-
-    {
-      "defaultDeploymentMode": "Serverless"
-    }
-  metricsAggregator: |-
-    {
-      "enableMetricAggregation": "false",
-      "enablePrometheusScraping" : "false"
-    }

config/overlays/test/configmap/inferenceservice-disable-istio.yaml

@@ -7,7 +7,6 @@ data:
   ingress: |-
     {
        "ingressGateway" : "knative-serving/knative-ingress-gateway",
-       "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
        "localGateway" : "knative-serving/knative-local-gateway",
        "localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
        "ingressDomain"  : "example.com",

config/overlays/test/configmap/inferenceservice-ingress.yaml

@@ -7,7 +7,6 @@ data:
   ingress: |-
     {
         "ingressGateway" : "knative-serving/knative-ingress-gateway",
-        "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
         "localGateway": "knative-serving/knative-local-gateway",
         "localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
         "ingressClassName" : "istio",

config/overlays/test/configmap/inferenceservice.yaml

@@ -36,7 +36,6 @@ data:
   ingress: |-
     {
         "ingressGateway" : "knative-serving/knative-ingress-gateway",
-        "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
         "localGateway": "knative-serving/knative-local-gateway",
         "localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
         "ingressClassName" : "istio",

hack/violation_exceptions.list

@@ -31,7 +31,6 @@ API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta
 API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ExplainerExtensionSpec,StorageURI
 API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ExplainersConfig,ARTExplainer
 API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ExplainersConfig,AlibiExplainer
-API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,IngressConfig,IngressServiceName
 API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,IngressConfig,LocalGatewayServiceName
 API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ModelStatus,ModelCopies
 API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ModelStatus,ModelRevisionStates

pkg/apis/serving/v1beta1/configmap.go

@@ -66,17 +66,17 @@ type InferenceServicesConfig struct {
 
 // +kubebuilder:object:generate=false
 type IngressConfig struct {
-	IngressGateway          string  `json:"ingressGateway,omitempty"`
-	IngressServiceName      string  `json:"ingressService,omitempty"`
-	LocalGateway            string  `json:"localGateway,omitempty"`
-	LocalGatewayServiceName string  `json:"localGatewayService,omitempty"`
-	IngressDomain           string  `json:"ingressDomain,omitempty"`
-	IngressClassName        *string `json:"ingressClassName,omitempty"`
-	DomainTemplate          string  `json:"domainTemplate,omitempty"`
-	UrlScheme               string  `json:"urlScheme,omitempty"`
-	DisableIstioVirtualHost bool    `json:"disableIstioVirtualHost,omitempty"`
-	PathTemplate            string  `json:"pathTemplate,omitempty"`
-	DisableIngressCreation  bool    `json:"disableIngressCreation,omitempty"`
+	IngressGateway             string  `json:"ingressGateway,omitempty"`
+	KnativeLocalGatewayService string  `json:"knativeLocalGatewayService,omitempty"`
+	LocalGateway               string  `json:"localGateway,omitempty"`
+	LocalGatewayServiceName    string  `json:"localGatewayService,omitempty"`
+	IngressDomain              string  `json:"ingressDomain,omitempty"`
+	IngressClassName           *string `json:"ingressClassName,omitempty"`
+	DomainTemplate             string  `json:"domainTemplate,omitempty"`
+	UrlScheme                  string  `json:"urlScheme,omitempty"`
+	DisableIstioVirtualHost    bool    `json:"disableIstioVirtualHost,omitempty"`
+	PathTemplate               string  `json:"pathTemplate,omitempty"`
+	DisableIngressCreation     bool    `json:"disableIngressCreation,omitempty"`
 }
 
 // +kubebuilder:object:generate=false
@@ -112,8 +112,8 @@ func NewIngressConfig(clientset kubernetes.Interface) (*IngressConfig, error) {
 			return nil, fmt.Errorf("unable to parse ingress config json: %v", err)
 		}
 
-		if ingressConfig.IngressGateway == "" || ingressConfig.IngressServiceName == "" {
-			return nil, fmt.Errorf("invalid ingress config - ingressGateway and ingressService are required")
+		if ingressConfig.IngressGateway == "" {
+			return nil, fmt.Errorf("invalid ingress config - ingressGateway is required")
 		}
 		if ingressConfig.PathTemplate != "" {
 			// TODO: ensure that the generated path is valid, that is:
@@ -128,6 +128,10 @@ func NewIngressConfig(clientset kubernetes.Interface) (*IngressConfig, error) {
 				return nil, fmt.Errorf("invalid ingress config - ingressDomain is required if pathTemplate is given")
 			}
 		}
+
+		if len(ingressConfig.KnativeLocalGatewayService) == 0 {
+			ingressConfig.KnativeLocalGatewayService = ingressConfig.LocalGatewayServiceName
+		}
 	}
 
 	if ingressConfig.DomainTemplate == "" {

pkg/apis/serving/v1beta1/configmap_test.go

@@ -18,6 +18,7 @@ package v1beta1
 
 import (
 	ctx "context"
+	"fmt"
 	logger "log"
 	"testing"
 
@@ -30,6 +31,27 @@ import (
 	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
+var (
+	KnativeIngressGateway      = "knative-serving/knative-ingress-gateway"
+	KnativeLocalGatewayService = "test-destination"
+	KnativeLocalGateway        = "knative-serving/knative-local-gateway"
+	LocalGatewayService        = "knative-local-gateway.istio-system.svc.cluster.local"
+	UrlScheme                  = "https"
+	IngressDomain              = "example.com"
+	AdditionalDomain           = "additional-example.com"
+	AdditionalDomainExtra      = "additional-example-extra.com"
+	IngressConfigData          = fmt.Sprintf(`{
+		"ingressGateway" : "%s",
+		"knativeLocalGatewayService" : "%s",
+		"localGateway" : "%s",
+		"localGatewayService" : "%s",
+		"ingressDomain": "%s",
+		"urlScheme": "https",
+        "additionalIngressDomains": ["%s","%s"]
+	}`, KnativeIngressGateway, KnativeLocalGatewayService, KnativeLocalGateway, LocalGatewayService, IngressDomain,
+		AdditionalDomain, AdditionalDomainExtra)
+)
+
 func createFakeClient() client.WithWatch {
 	clientBuilder := fakeclient.NewClientBuilder()
 	fakeClient := clientBuilder.Build()
@@ -91,10 +113,42 @@ func TestNewIngressConfig(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 	clientset := fakeclientset.NewSimpleClientset(&v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{Name: constants.InferenceServiceConfigMapName, Namespace: constants.KServeNamespace},
+		Data: map[string]string{
+			IngressConfigKeyName: IngressConfigData,
+		},
+	})
+	ingressCfg, err := NewIngressConfig(clientset)
+	g.Expect(err).Should(gomega.BeNil())
+	g.Expect(ingressCfg).ShouldNot(gomega.BeNil())
+
+	g.Expect(ingressCfg.IngressGateway).To(gomega.Equal(KnativeIngressGateway))
+	g.Expect(ingressCfg.KnativeLocalGatewayService).To(gomega.Equal(KnativeLocalGatewayService))
+	g.Expect(ingressCfg.LocalGateway).To(gomega.Equal(KnativeLocalGateway))
+	g.Expect(ingressCfg.LocalGatewayServiceName).To(gomega.Equal(LocalGatewayService))
+	g.Expect(ingressCfg.UrlScheme).To(gomega.Equal(UrlScheme))
+	g.Expect(ingressCfg.IngressDomain).To(gomega.Equal(IngressDomain))
+}
+
+func TestNewIngressConfigDefaultKnativeService(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	clientset := fakeclientset.NewSimpleClientset(&v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{Name: constants.InferenceServiceConfigMapName, Namespace: constants.KServeNamespace},
+		Data: map[string]string{
+			IngressConfigKeyName: fmt.Sprintf(`{
+				"ingressGateway" : "%s",
+				"localGateway" : "%s",
+				"localGatewayService" : "%s",
+				"ingressDomain": "%s",
+				"urlScheme": "https",
+        		"additionalIngressDomains": ["%s","%s"]
+			}`, KnativeIngressGateway, KnativeLocalGateway, LocalGatewayService, IngressDomain,
+				AdditionalDomain, AdditionalDomainExtra),
+		},
 	})
 	ingressCfg, err := NewIngressConfig(clientset)
 	g.Expect(err).Should(gomega.BeNil())
 	g.Expect(ingressCfg).ShouldNot(gomega.BeNil())
+	g.Expect(ingressCfg.KnativeLocalGatewayService).To(gomega.Equal(LocalGatewayService))
 }
 
 func TestNewDeployConfig(t *testing.T) {

pkg/apis/serving/v1beta1/swagger.json

@@ -2912,7 +2912,7 @@
         "ingressGateway": {
           "type": "string"
         },
-        "ingressService": {
+        "knativeLocalGatewayService": {
           "type": "string"
         },
         "localGateway": {

pkg/controller/v1beta1/inferenceservice/controller_test.go

@@ -76,7 +76,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
 			}`,
 			"ingress": `{
 				"ingressGateway": "knative-serving/knative-ingress-gateway",
-				"ingressService": "test-destination",
 				"localGateway": "knative-serving/knative-local-gateway",
 				"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local"
 			}`,
@@ -2079,7 +2078,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
 					copiedConfigs[key] = `{
 						"disableIstioVirtualHost": true,
 						"ingressGateway": "knative-serving/knative-ingress-gateway",
-						"ingressService": "test-destination",
 						"localGateway": "knative-serving/knative-local-gateway",
 						"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local"
 					}`

pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go

@@ -74,7 +74,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
 			}`,
 			"ingress": `{
 				"ingressGateway": "knative-serving/knative-ingress-gateway",
-				"ingressService": "test-destination",
 				"localGateway": "knative-serving/knative-local-gateway",
 				"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local"
 			}`,
@@ -1275,7 +1274,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
             }`,
 			"ingress": `{
                "ingressGateway": "knative-serving/knative-ingress-gateway",
-               "ingressService": "test-destination",
                "localGateway": "knative-serving/knative-local-gateway",
                "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local",
                "ingressDomain": "example.com"
@@ -1707,7 +1705,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
             }`,
 			"ingress": `{
                "ingressGateway": "knative-serving/knative-ingress-gateway",
-               "ingressService": "test-destination",
                "localGateway": "knative-serving/knative-local-gateway",
                "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local",
                "ingressDomain": "example.com",

pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go

@@ -341,7 +341,7 @@ func createIngress(isvc *v1beta1.InferenceService, useDefault bool, config *v1be
 			Match: createHTTPMatchRequest(constants.ExplainPrefix(), serviceHost,
 				network.GetServiceHostname(isvc.Name, isvc.Namespace), isInternal, config),
 			Route: []*istiov1beta1.HTTPRouteDestination{
-				createHTTPRouteDestination(config.LocalGatewayServiceName),
+				createHTTPRouteDestination(config.KnativeLocalGatewayService),
 			},
 			Headers: &istiov1beta1.Headers{
 				Request: &istiov1beta1.Headers_HeaderOperations{
@@ -358,7 +358,7 @@ func createIngress(isvc *v1beta1.InferenceService, useDefault bool, config *v1be
 		Match: createHTTPMatchRequest("", serviceHost,
 			network.GetServiceHostname(isvc.Name, isvc.Namespace), isInternal, config),
 		Route: []*istiov1beta1.HTTPRouteDestination{
-			createHTTPRouteDestination(config.LocalGatewayServiceName),
+			createHTTPRouteDestination(config.KnativeLocalGatewayService),
 		},
 		Headers: &istiov1beta1.Headers{
 			Request: &istiov1beta1.Headers_HeaderOperations{
@@ -422,7 +422,7 @@ func createIngress(isvc *v1beta1.InferenceService, useDefault bool, config *v1be
 				Uri: "/",
 			},
 			Route: []*istiov1beta1.HTTPRouteDestination{
-				createHTTPRouteDestination(config.LocalGatewayServiceName),
+				createHTTPRouteDestination(config.KnativeLocalGatewayService),
 			},
 			Headers: &istiov1beta1.Headers{
 				Request: &istiov1beta1.Headers_HeaderOperations{