Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow KServe to have its own local gateways for Serverless mode #376

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/kserve-resources/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ $ helm install kserve oci://ghcr.io/kserve/charts/kserve --version v0.12.1
| kserve.controller.gateway.domainTemplate | string | `"{{ .Name }}-{{ .Namespace }}.{{ .IngressDomain }}"` | |
| kserve.controller.gateway.ingressGateway.className | string | `"istio"` | |
| kserve.controller.gateway.ingressGateway.gateway | string | `"knative-serving/knative-ingress-gateway"` | |
| kserve.controller.gateway.ingressGateway.gatewayService | string | `"istio-ingressgateway.istio-system.svc.cluster.local"` | |
| kserve.controller.gateway.localGateway.gateway | string | `"knative-serving/knative-local-gateway"` | |
| kserve.controller.gateway.localGateway.gatewayService | string | `"knative-local-gateway.istio-system.svc.cluster.local"` | |
| kserve.controller.gateway.localGateway.knativeGatewayService | string | `""` | |
| kserve.controller.gateway.urlScheme | string | `"http"` | |
| kserve.controller.image | string | `"kserve/kserve-controller"` | |
| kserve.controller.nodeSelector | object | `{}` | |
Expand Down
13 changes: 9 additions & 4 deletions charts/kserve-resources/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,6 @@ data:
ingress: |-
{
"ingressGateway" : "knative-serving/knative-ingress-gateway",
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"localGateway" : "knative-serving/knative-local-gateway",
"localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
"ingressDomain" : "example.com",
Expand All @@ -210,9 +209,15 @@ data:
# NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
"ingressGateway" : "knative-serving/knative-ingress-gateway",

# ingressService specifies the hostname of the ingress service.
# knativeLocalGatewayService specifies the hostname of the Knative's local gateway serivice.
# The default KServe configurations are re-using the Istio local gateways for Knative. In this case, this
# knativeLocalGatewayService field can be left unset. When unset, the value of "localGatewayService" will be used.
# However, sometimes it may be better to have local gateways specifically for KServe (e.g. when enabling strict mTLS in Istio).
# Under such setups where KServe is needed to have its own local gateways, the values of the "localGateway" and
# "localGatewayService" should point to the KServe local gateways. Then, this knativeLocalGatewayService field
# should point to the Knative's local gateway service.
# NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"knativeLocalGatewayService": "",

# localGateway specifies the gateway which handles the network traffic within the cluster.
# NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
Expand Down Expand Up @@ -507,7 +512,7 @@ data:
ingress: |-
{
"ingressGateway" : "{{ .Values.kserve.controller.gateway.ingressGateway.gateway }}",
"ingressService" : "{{ .Values.kserve.controller.gateway.ingressGateway.gatewayService }}",
"knativeLocalGatewayService" : "{{ .Values.kserve.controller.gateway.localGateway.knativeGatewayService }}",
"localGateway" : "{{ .Values.kserve.controller.gateway.localGateway.gateway }}",
"localGatewayService" : "{{ .Values.kserve.controller.gateway.localGateway.gatewayService }}",
"ingressClassName" : "{{ .Values.kserve.controller.gateway.ingressGateway.className }}",
Expand Down
2 changes: 1 addition & 1 deletion charts/kserve-resources/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ kserve:
localGateway:
gateway: knative-serving/knative-local-gateway
gatewayService: knative-local-gateway.istio-system.svc.cluster.local
knativeGatewayService: ""
ingressGateway:
gateway: knative-serving/knative-ingress-gateway
gatewayService: istio-ingressgateway.istio-system.svc.cluster.local
className: istio
nodeSelector: {}
tolerations: []
Expand Down
12 changes: 8 additions & 4 deletions config/configmap/inferenceservice.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,6 @@ data:
ingress: |-
{
"ingressGateway" : "knative-serving/knative-ingress-gateway",
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"localGateway" : "knative-serving/knative-local-gateway",
"localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
"ingressDomain" : "example.com",
Expand All @@ -221,9 +220,15 @@ data:
# NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
"ingressGateway" : "knative-serving/knative-ingress-gateway",

# ingressService specifies the hostname of the ingress service.
# knativeLocalGatewayService specifies the hostname of the Knative's local gateway serivice.
# The default KServe configurations are re-using the Istio local gateways for Knative. In this case, this
# knativeLocalGatewayService field can be left unset. When unset, the value of "localGatewayService" will be used.
# However, sometimes it may be better to have local gateways specifically for KServe (e.g. when enabling strict mTLS in Istio).
# Under such setups where KServe is needed to have its own local gateways, the values of the "localGateway" and
# "localGatewayService" should point to the KServe local gateways. Then, this knativeLocalGatewayService field
# should point to the Knative's local gateway service.
# NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"knativeLocalGatewayService": "",

# localGateway specifies the gateway which handles the network traffic within the cluster.
# NOTE: This configuration only applicable for serverless deployment with Istio configured as network layer.
Expand Down Expand Up @@ -506,7 +511,6 @@ data:
ingress: |-
{
"ingressGateway" : "knative-serving/knative-ingress-gateway",
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"localGateway" : "knative-serving/knative-local-gateway",
"localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
"ingressDomain" : "example.com",
Expand Down
25 changes: 12 additions & 13 deletions config/overlays/odh/inferenceservice-config-patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,25 @@ data:
"memoryLimit": "1Gi",
"cpuRequest": "100m",
"cpuLimit": "1",
"enableDirectPvcVolumeMount": true
"caBundleConfigMapName": "",
"caBundleVolumeMountPath": "/etc/ssl/custom-certs",
"enableDirectPvcVolumeMount": true,
"enableModelcar": false,
"cpuModelcar": "10m",
"memoryModelcar": "15Mi"
}
ingress: |-
{
"ingressGateway" : "knative-serving/knative-ingress-gateway",
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"knativeLocalGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
"ingressService" : "need.to.delete",
"localGateway" : "istio-system/kserve-local-gateway",
"localGatewayService" : "kserve-local-gateway.istio-system.svc.cluster.local",
"ingressDomain" : "example.com",
"ingressClassName" : "istio",
"domainTemplate": "{{ .Name }}-{{ .Namespace }}.{{ .IngressDomain }}",
"urlScheme": "https",
"disableIstioVirtualHost": false,
"disableIstioVirtualHost": false,
"disableIngressCreation": true
}
logger: |-
Expand All @@ -42,7 +48,9 @@ data:
"memoryRequest": "1Gi",
"memoryLimit": "1Gi",
"cpuRequest": "1",
"cpuLimit": "1"
"cpuLimit": "1",
"maxBatchSize": "32",
"maxLatency": "5000"
}
agent: |-
{
Expand All @@ -60,12 +68,3 @@ data:
"cpuRequest": "100m",
"cpuLimit": "1"
}
deploy: |-
{
"defaultDeploymentMode": "Serverless"
}
metricsAggregator: |-
{
"enableMetricAggregation": "false",
"enablePrometheusScraping" : "false"
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ data:
ingress: |-
{
"ingressGateway" : "knative-serving/knative-ingress-gateway",
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"localGateway" : "knative-serving/knative-local-gateway",
"localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
"ingressDomain" : "example.com",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ data:
ingress: |-
{
"ingressGateway" : "knative-serving/knative-ingress-gateway",
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"localGateway": "knative-serving/knative-local-gateway",
"localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
"ingressClassName" : "istio",
Expand Down
1 change: 0 additions & 1 deletion config/overlays/test/configmap/inferenceservice.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ data:
ingress: |-
{
"ingressGateway" : "knative-serving/knative-ingress-gateway",
"ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local",
"localGateway": "knative-serving/knative-local-gateway",
"localGatewayService" : "knative-local-gateway.istio-system.svc.cluster.local",
"ingressClassName" : "istio",
Expand Down
1 change: 0 additions & 1 deletion hack/violation_exceptions.list
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta
API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ExplainerExtensionSpec,StorageURI
API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ExplainersConfig,ARTExplainer
API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ExplainersConfig,AlibiExplainer
API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,IngressConfig,IngressServiceName
API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,IngressConfig,LocalGatewayServiceName
API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ModelStatus,ModelCopies
API rule violation: names_match,github.com/kserve/kserve/pkg/apis/serving/v1beta1,ModelStatus,ModelRevisionStates
Expand Down
30 changes: 17 additions & 13 deletions pkg/apis/serving/v1beta1/configmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,17 @@ type InferenceServicesConfig struct {

// +kubebuilder:object:generate=false
type IngressConfig struct {
IngressGateway string `json:"ingressGateway,omitempty"`
IngressServiceName string `json:"ingressService,omitempty"`
LocalGateway string `json:"localGateway,omitempty"`
LocalGatewayServiceName string `json:"localGatewayService,omitempty"`
IngressDomain string `json:"ingressDomain,omitempty"`
IngressClassName *string `json:"ingressClassName,omitempty"`
DomainTemplate string `json:"domainTemplate,omitempty"`
UrlScheme string `json:"urlScheme,omitempty"`
DisableIstioVirtualHost bool `json:"disableIstioVirtualHost,omitempty"`
PathTemplate string `json:"pathTemplate,omitempty"`
DisableIngressCreation bool `json:"disableIngressCreation,omitempty"`
IngressGateway string `json:"ingressGateway,omitempty"`
KnativeLocalGatewayService string `json:"knativeLocalGatewayService,omitempty"`
LocalGateway string `json:"localGateway,omitempty"`
LocalGatewayServiceName string `json:"localGatewayService,omitempty"`
IngressDomain string `json:"ingressDomain,omitempty"`
IngressClassName *string `json:"ingressClassName,omitempty"`
DomainTemplate string `json:"domainTemplate,omitempty"`
UrlScheme string `json:"urlScheme,omitempty"`
DisableIstioVirtualHost bool `json:"disableIstioVirtualHost,omitempty"`
PathTemplate string `json:"pathTemplate,omitempty"`
DisableIngressCreation bool `json:"disableIngressCreation,omitempty"`
}

// +kubebuilder:object:generate=false
Expand Down Expand Up @@ -112,8 +112,8 @@ func NewIngressConfig(clientset kubernetes.Interface) (*IngressConfig, error) {
return nil, fmt.Errorf("unable to parse ingress config json: %v", err)
}

if ingressConfig.IngressGateway == "" || ingressConfig.IngressServiceName == "" {
return nil, fmt.Errorf("invalid ingress config - ingressGateway and ingressService are required")
if ingressConfig.IngressGateway == "" {
return nil, fmt.Errorf("invalid ingress config - ingressGateway is required")
}
if ingressConfig.PathTemplate != "" {
// TODO: ensure that the generated path is valid, that is:
Expand All @@ -128,6 +128,10 @@ func NewIngressConfig(clientset kubernetes.Interface) (*IngressConfig, error) {
return nil, fmt.Errorf("invalid ingress config - ingressDomain is required if pathTemplate is given")
}
}

if len(ingressConfig.KnativeLocalGatewayService) == 0 {
ingressConfig.KnativeLocalGatewayService = ingressConfig.LocalGatewayServiceName
}
}

if ingressConfig.DomainTemplate == "" {
Expand Down
54 changes: 54 additions & 0 deletions pkg/apis/serving/v1beta1/configmap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package v1beta1

import (
ctx "context"
"fmt"
logger "log"
"testing"

Expand All @@ -30,6 +31,27 @@ import (
fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
)

var (
KnativeIngressGateway = "knative-serving/knative-ingress-gateway"
KnativeLocalGatewayService = "test-destination"
KnativeLocalGateway = "knative-serving/knative-local-gateway"
LocalGatewayService = "knative-local-gateway.istio-system.svc.cluster.local"
UrlScheme = "https"
IngressDomain = "example.com"
AdditionalDomain = "additional-example.com"
AdditionalDomainExtra = "additional-example-extra.com"
IngressConfigData = fmt.Sprintf(`{
"ingressGateway" : "%s",
"knativeLocalGatewayService" : "%s",
"localGateway" : "%s",
"localGatewayService" : "%s",
"ingressDomain": "%s",
"urlScheme": "https",
"additionalIngressDomains": ["%s","%s"]
}`, KnativeIngressGateway, KnativeLocalGatewayService, KnativeLocalGateway, LocalGatewayService, IngressDomain,
AdditionalDomain, AdditionalDomainExtra)
)

func createFakeClient() client.WithWatch {
clientBuilder := fakeclient.NewClientBuilder()
fakeClient := clientBuilder.Build()
Expand Down Expand Up @@ -91,10 +113,42 @@ func TestNewIngressConfig(t *testing.T) {
g := gomega.NewGomegaWithT(t)
clientset := fakeclientset.NewSimpleClientset(&v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: constants.InferenceServiceConfigMapName, Namespace: constants.KServeNamespace},
Data: map[string]string{
IngressConfigKeyName: IngressConfigData,
},
})
ingressCfg, err := NewIngressConfig(clientset)
g.Expect(err).Should(gomega.BeNil())
g.Expect(ingressCfg).ShouldNot(gomega.BeNil())

g.Expect(ingressCfg.IngressGateway).To(gomega.Equal(KnativeIngressGateway))
g.Expect(ingressCfg.KnativeLocalGatewayService).To(gomega.Equal(KnativeLocalGatewayService))
g.Expect(ingressCfg.LocalGateway).To(gomega.Equal(KnativeLocalGateway))
g.Expect(ingressCfg.LocalGatewayServiceName).To(gomega.Equal(LocalGatewayService))
g.Expect(ingressCfg.UrlScheme).To(gomega.Equal(UrlScheme))
g.Expect(ingressCfg.IngressDomain).To(gomega.Equal(IngressDomain))
}

func TestNewIngressConfigDefaultKnativeService(t *testing.T) {
g := gomega.NewGomegaWithT(t)
clientset := fakeclientset.NewSimpleClientset(&v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: constants.InferenceServiceConfigMapName, Namespace: constants.KServeNamespace},
Data: map[string]string{
IngressConfigKeyName: fmt.Sprintf(`{
"ingressGateway" : "%s",
"localGateway" : "%s",
"localGatewayService" : "%s",
"ingressDomain": "%s",
"urlScheme": "https",
"additionalIngressDomains": ["%s","%s"]
}`, KnativeIngressGateway, KnativeLocalGateway, LocalGatewayService, IngressDomain,
AdditionalDomain, AdditionalDomainExtra),
},
})
ingressCfg, err := NewIngressConfig(clientset)
g.Expect(err).Should(gomega.BeNil())
g.Expect(ingressCfg).ShouldNot(gomega.BeNil())
g.Expect(ingressCfg.KnativeLocalGatewayService).To(gomega.Equal(LocalGatewayService))
}

func TestNewDeployConfig(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/serving/v1beta1/openapi_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pkg/apis/serving/v1beta1/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -2912,7 +2912,7 @@
"ingressGateway": {
"type": "string"
},
"ingressService": {
"knativeLocalGatewayService": {
"type": "string"
},
"localGateway": {
Expand Down
2 changes: 0 additions & 2 deletions pkg/controller/v1beta1/inferenceservice/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
}`,
"ingress": `{
"ingressGateway": "knative-serving/knative-ingress-gateway",
"ingressService": "test-destination",
"localGateway": "knative-serving/knative-local-gateway",
"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local"
}`,
Expand Down Expand Up @@ -2079,7 +2078,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
copiedConfigs[key] = `{
"disableIstioVirtualHost": true,
"ingressGateway": "knative-serving/knative-ingress-gateway",
"ingressService": "test-destination",
"localGateway": "knative-serving/knative-local-gateway",
"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local"
}`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
}`,
"ingress": `{
"ingressGateway": "knative-serving/knative-ingress-gateway",
"ingressService": "test-destination",
"localGateway": "knative-serving/knative-local-gateway",
"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local"
}`,
Expand Down Expand Up @@ -1275,7 +1274,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
}`,
"ingress": `{
"ingressGateway": "knative-serving/knative-ingress-gateway",
"ingressService": "test-destination",
"localGateway": "knative-serving/knative-local-gateway",
"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local",
"ingressDomain": "example.com"
Expand Down Expand Up @@ -1707,7 +1705,6 @@ var _ = Describe("v1beta1 inference service controller", func() {
}`,
"ingress": `{
"ingressGateway": "knative-serving/knative-ingress-gateway",
"ingressService": "test-destination",
"localGateway": "knative-serving/knative-local-gateway",
"localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local",
"ingressDomain": "example.com",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ func createIngress(isvc *v1beta1.InferenceService, useDefault bool, config *v1be
Match: createHTTPMatchRequest(constants.ExplainPrefix(), serviceHost,
network.GetServiceHostname(isvc.Name, isvc.Namespace), isInternal, config),
Route: []*istiov1beta1.HTTPRouteDestination{
createHTTPRouteDestination(config.LocalGatewayServiceName),
createHTTPRouteDestination(config.KnativeLocalGatewayService),
},
Headers: &istiov1beta1.Headers{
Request: &istiov1beta1.Headers_HeaderOperations{
Expand All @@ -358,7 +358,7 @@ func createIngress(isvc *v1beta1.InferenceService, useDefault bool, config *v1be
Match: createHTTPMatchRequest("", serviceHost,
network.GetServiceHostname(isvc.Name, isvc.Namespace), isInternal, config),
Route: []*istiov1beta1.HTTPRouteDestination{
createHTTPRouteDestination(config.LocalGatewayServiceName),
createHTTPRouteDestination(config.KnativeLocalGatewayService),
},
Headers: &istiov1beta1.Headers{
Request: &istiov1beta1.Headers_HeaderOperations{
Expand Down Expand Up @@ -422,7 +422,7 @@ func createIngress(isvc *v1beta1.InferenceService, useDefault bool, config *v1be
Uri: "/",
},
Route: []*istiov1beta1.HTTPRouteDestination{
createHTTPRouteDestination(config.LocalGatewayServiceName),
createHTTPRouteDestination(config.KnativeLocalGatewayService),
},
Headers: &istiov1beta1.Headers{
Request: &istiov1beta1.Headers_HeaderOperations{
Expand Down
Loading
Loading