From e97ec963f90a5185b8a42ea10b9960bbc9f36c44 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 9 Oct 2024 15:14:02 -0400 Subject: [PATCH 01/20] add oauth-proxy to rawdeployments if odh auth label is present Signed-off-by: Vedant Mahabaleshwarkar --- .../odh/inferenceservice-config-patch.yaml | 1 - pkg/constants/constants.go | 12 + .../rawkube_controller_test.go | 547 ++++++++++++++++++ .../deployment/deployment_reconciler.go | 105 ++++ .../ingress/kube_ingress_reconciler.go | 37 +- .../reconcilers/raw/raw_kube_reconciler.go | 20 +- .../reconcilers/service/service_reconciler.go | 20 + 7 files changed, 722 insertions(+), 20 deletions(-) diff --git a/config/overlays/odh/inferenceservice-config-patch.yaml b/config/overlays/odh/inferenceservice-config-patch.yaml index 9fada7f1c54..9109221ec33 100644 --- a/config/overlays/odh/inferenceservice-config-patch.yaml +++ b/config/overlays/odh/inferenceservice-config-patch.yaml @@ -23,7 +23,6 @@ data: "localGateway" : "istio-system/kserve-local-gateway", "localGatewayService" : "kserve-local-gateway.istio-system.svc.cluster.local", "ingressDomain" : "example.com", - "ingressClassName" : "istio", "domainTemplate": "{{ .Name }}-{{ .Namespace }}.{{ .IngressDomain }}", "urlScheme": "https", "disableIstioVirtualHost": false, diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 97543756fa7..35c1e1ff496 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -38,6 +38,7 @@ var ( KnativeServingAPIGroupName = KnativeServingAPIGroupNamePrefix + ".dev" KServeNamespace = getEnvOrDefault("POD_NAMESPACE", "kserve") KServeDefaultVersion = "v0.5.0" + KserveServiceAccountName = "kserve-sa" ) // InferenceService Constants @@ -135,6 +136,7 @@ const ( ClusterLocalDomain = "svc.cluster.local" IsvcNameHeader = "KServe-Isvc-Name" IsvcNamespaceHeader = "KServe-Isvc-Namespace" + ODHKserveRawAuth = "networking.kserve.io/odh-auth" ) // StorageSpec Constants @@ -446,6 +448,16 @@ const ( SupportedModelMLFlow = "mlflow" ) +// opendatahub rawDeployment Auth +const ( + OauthProxyImage = "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:4bef31eb993feb6f1096b51b4876c65a6fb1f4401fee97fa4f4542b6b7c9bc46" + OauthProxyPort = 8443 + OauthProxyResourceMemoryLimit = "256Mi" + OauthProxyResourceCPULimit = "100m" + OauthProxyResourceMemoryRequest = "256Mi" + OauthProxyResourceCPURequest = "100m" +) + type ProtocolVersion int const ( diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index 93c43b84d49..e7e9b619c8d 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -1272,6 +1272,109 @@ var _ = Describe("v1beta1 inference service controller", func() { Eventually(func() error { return k8sClient.Get(context.TODO(), predictorHPAKey, actualHPA) }, timeout). Should(HaveOccurred()) }) + It("Should have no ingress created if labeled as cluster-local", func() { + By("By creating a new InferenceService") + // Create configmap + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + // Create ServingRuntime + servingRuntime := &v1alpha1.ServingRuntime{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tf-serving-raw", + Namespace: "default", + }, + Spec: v1alpha1.ServingRuntimeSpec{ + SupportedModelFormats: []v1alpha1.SupportedModelFormat{ + { + Name: "tensorflow", + Version: proto.String("1"), + AutoSelect: proto.Bool(true), + }, + }, + ServingRuntimePodSpec: v1alpha1.ServingRuntimePodSpec{ + Containers: []v1.Container{ + { + Name: "kserve-container", + Image: "tensorflow/serving:1.14.0", + Command: []string{"/usr/bin/tensorflow_model_server"}, + Args: []string{ + "--port=9000", + "--rest_api_port=8080", + "--model_base_path=/mnt/models", + "--rest_api_timeout_in_ms=60000", + }, + Resources: defaultResource, + }, + }, + }, + Disabled: proto.Bool(false), + }, + } + k8sClient.Create(context.TODO(), servingRuntime) + defer k8sClient.Delete(context.TODO(), servingRuntime) + serviceName := "raw-cluster-local" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + Annotations: map[string]string{ + "serving.kserve.io/deploymentMode": "RawDeployment", + "serving.kserve.io/autoscalerClass": "hpa", + "serving.kserve.io/metrics": "cpu", + "serving.kserve.io/targetUtilizationPercentage": "75", + }, + Labels: map[string]string{ + "networking.kserve.io/visibility": "cluster-local", + }, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + MaxReplicas: 3, + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + isvc.DefaultInferenceService(nil, nil) + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + + inferenceService := &v1beta1.InferenceService{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, serviceKey, inferenceService) + if err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + actualIngress := &netv1.Ingress{} + predictorIngressKey := types.NamespacedName{Name: serviceKey.Name, + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorIngressKey, actualIngress) }, timeout). + ShouldNot(Succeed()) + }) }) Context("When creating inference service with raw kube predictor and empty ingressClassName", func() { configs := map[string]string{ @@ -2138,6 +2241,450 @@ var _ = Describe("v1beta1 inference service controller", func() { Expect(actualHPA.Spec).To(gomega.Equal(expectedHPA.Spec)) }) }) + Context("When creating an inferenceservice with raw kube predictor and ODH auth enabled", func() { + configs := map[string]string{ + "explainers": `{ + "alibi": { + "image": "kserve/alibi-explainer", + "defaultImageVersion": "latest" + } + }`, + "ingress": `{ + "ingressGateway": "knative-serving/knative-ingress-gateway", + "ingressService": "test-destination", + "localGateway": "knative-serving/knative-local-gateway", + "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local" + }`, + "storageInitializer": `{ + "image" : "kserve/storage-initializer:latest", + "memoryRequest": "100Mi", + "memoryLimit": "1Gi", + "cpuRequest": "100m", + "cpuLimit": "1", + "CaBundleConfigMapName": "", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + "enableDirectPvcVolumeMount": false + }`, + } + + It("Should have ingress/service/deployment/hpa created", func() { + By("By creating a new InferenceService") + // Create configmap + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + // Create ServingRuntime + servingRuntime := &v1alpha1.ServingRuntime{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tf-serving-raw", + Namespace: "default", + }, + Spec: v1alpha1.ServingRuntimeSpec{ + SupportedModelFormats: []v1alpha1.SupportedModelFormat{ + { + Name: "tensorflow", + Version: proto.String("1"), + AutoSelect: proto.Bool(true), + }, + }, + ServingRuntimePodSpec: v1alpha1.ServingRuntimePodSpec{ + Containers: []v1.Container{ + { + Name: "kserve-container", + Image: "tensorflow/serving:1.14.0", + Command: []string{"/usr/bin/tensorflow_model_server"}, + Args: []string{ + "--port=9000", + "--rest_api_port=8080", + "--model_base_path=/mnt/models", + "--rest_api_timeout_in_ms=60000", + }, + Resources: defaultResource, + }, + }, + }, + Disabled: proto.Bool(false), + }, + } + k8sClient.Create(context.TODO(), servingRuntime) + defer k8sClient.Delete(context.TODO(), servingRuntime) + serviceName := "raw-auth" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + Annotations: map[string]string{ + "serving.kserve.io/deploymentMode": "RawDeployment", + }, + Labels: map[string]string{ + "networking.kserve.io/odh-auth": "true", + }, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + MaxReplicas: 3, + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + isvc.DefaultInferenceService(nil, nil) + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + + inferenceService := &v1beta1.InferenceService{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, serviceKey, inferenceService) + if err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + actualDeployment := &appsv1.Deployment{} + predictorDeploymentKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name), + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorDeploymentKey, actualDeployment) }, timeout). + Should(Succeed()) + var replicas int32 = 1 + var revisionHistory int32 = 10 + var progressDeadlineSeconds int32 = 600 + var gracePeriod int64 = 30 + expectedDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: predictorDeploymentKey.Name, + Namespace: predictorDeploymentKey.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "isvc." + predictorDeploymentKey.Name, + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: predictorDeploymentKey.Name, + Namespace: "default", + Labels: map[string]string{ + "app": "isvc." + predictorDeploymentKey.Name, + constants.KServiceComponentLabel: constants.Predictor.String(), + constants.InferenceServicePodLabelKey: serviceName, + "serving.kserve.io/inferenceservice": serviceName, + "networking.kserve.io/odh-auth": "true", + }, + Annotations: map[string]string{ + constants.StorageInitializerSourceUriInternalAnnotationKey: *isvc.Spec.Predictor.Model.StorageURI, + "serving.kserve.io/deploymentMode": "RawDeployment", + "service.beta.openshift.io/serving-cert-secret-name": predictorDeploymentKey.Name, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "tensorflow/serving:" + + *isvc.Spec.Predictor.Model.RuntimeVersion, + Name: constants.InferenceServiceContainerName, + Command: []string{v1beta1.TensorflowEntrypointCommand}, + Args: []string{ + "--port=" + v1beta1.TensorflowServingGRPCPort, + "--rest_api_port=" + v1beta1.TensorflowServingRestPort, + "--model_base_path=" + constants.DefaultModelLocalMountPath, + "--rest_api_timeout_in_ms=60000", + }, + Resources: defaultResource, + ReadinessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{ + IntVal: 8080, + }, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + ImagePullPolicy: "IfNotPresent", + }, + { + Name: "oauth-proxy", + Image: constants.OauthProxyImage, + Args: []string{ + `--https-address=:8443`, + `--provider=openshift`, + `--openshift-service-account=kserve-sa`, + `--upstream=http://localhost:8080`, + `--tls-cert=/etc/tls/private/tls.crt`, + `--tls-key=/etc/tls/private/tls.key`, + `--cookie-secret=SECRET`, + `--openshift-delegate-urls={"/": {"namespace": "` + serviceKey.Namespace + `", "resource": "services", "verb": "get"}}`, + `--openshift-sar={"namespace": "` + serviceKey.Namespace + `", "resource": "services", "verb": "get"}`, + `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, + }, + Ports: []v1.ContainerPort{ + { + ContainerPort: constants.OauthProxyPort, + Name: "https", + Protocol: v1.ProtocolTCP, + }, + }, + LivenessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/oauth/healthz", + Port: intstr.FromInt(constants.OauthProxyPort), + Scheme: v1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + ReadinessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/oauth/healthz", + Port: intstr.FromInt(constants.OauthProxyPort), + Scheme: v1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(constants.OauthProxyResourceCPULimit), + v1.ResourceMemory: resource.MustParse(constants.OauthProxyResourceMemoryLimit), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(constants.OauthProxyResourceCPURequest), + v1.ResourceMemory: resource.MustParse(constants.OauthProxyResourceMemoryRequest), + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "proxy-tls", + MountPath: "/etc/tls/private", + }, + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + ImagePullPolicy: "IfNotPresent", + }, + }, + Volumes: []v1.Volume{ + { + Name: "proxy-tls", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: predictorDeploymentKey.Name, + DefaultMode: func(i int32) *int32 { return &i }(420), + }, + }, + }, + }, + ServiceAccountName: constants.KserveServiceAccountName, + DeprecatedServiceAccount: constants.KserveServiceAccountName, + SchedulerName: "default-scheduler", + RestartPolicy: "Always", + TerminationGracePeriodSeconds: &gracePeriod, + DNSPolicy: "ClusterFirst", + SecurityContext: &v1.PodSecurityContext{ + SELinuxOptions: nil, + WindowsOptions: nil, + RunAsUser: nil, + RunAsGroup: nil, + RunAsNonRoot: nil, + SupplementalGroups: nil, + FSGroup: nil, + Sysctls: nil, + FSGroupChangePolicy: nil, + SeccompProfile: nil, + }, + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: "RollingUpdate", + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: &intstr.IntOrString{Type: 1, IntVal: 0, StrVal: "25%"}, + MaxSurge: &intstr.IntOrString{Type: 1, IntVal: 0, StrVal: "25%"}, + }, + }, + RevisionHistoryLimit: &revisionHistory, + ProgressDeadlineSeconds: &progressDeadlineSeconds, + }, + } + Expect(actualDeployment.Spec).To(gomega.Equal(expectedDeployment.Spec)) + + //check service + actualService := &v1.Service{} + predictorServiceKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name), + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorServiceKey, actualService) }, timeout). + Should(Succeed()) + + expectedService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: predictorServiceKey.Name, + Namespace: predictorServiceKey.Namespace, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: constants.PredictorServiceName(serviceName), + Protocol: "TCP", + Port: 80, + TargetPort: intstr.IntOrString{Type: 0, IntVal: 8080, StrVal: ""}, + }, + { + Name: "https", + Protocol: "TCP", + Port: 8443, + TargetPort: intstr.IntOrString{Type: intstr.String, StrVal: "https"}, + }, + }, + Type: "ClusterIP", + SessionAffinity: "None", + Selector: map[string]string{ + "app": fmt.Sprintf("isvc.%s", constants.PredictorServiceName(serviceName)), + }, + }, + } + actualService.Spec.ClusterIP = "" + actualService.Spec.ClusterIPs = nil + actualService.Spec.IPFamilies = nil + actualService.Spec.IPFamilyPolicy = nil + actualService.Spec.InternalTrafficPolicy = nil + Expect(actualService.Spec).To(gomega.Equal(expectedService.Spec)) + + //check isvc status + updatedDeployment := actualDeployment.DeepCopy() + updatedDeployment.Status.Conditions = []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentAvailable, + Status: v1.ConditionTrue, + }, + } + Expect(k8sClient.Status().Update(context.TODO(), updatedDeployment)).NotTo(gomega.HaveOccurred()) + + //check ingress + pathType := netv1.PathTypePrefix + actualIngress := &netv1.Ingress{} + predictorIngressKey := types.NamespacedName{Name: serviceKey.Name, + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorIngressKey, actualIngress) }, timeout). + Should(Succeed()) + expectedIngress := netv1.Ingress{ + Spec: netv1.IngressSpec{ + Rules: []netv1.IngressRule{ + { + Host: "raw-auth-predictor-default.example.com", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "raw-auth-predictor", + Port: netv1.ServiceBackendPort{ + Number: 8443, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + Expect(actualIngress.Spec).To(gomega.Equal(expectedIngress.Spec)) + // verify if InferenceService status is updated + expectedIsvcStatus := v1beta1.InferenceServiceStatus{ + Status: duckv1.Status{ + Conditions: duckv1.Conditions{ + { + Type: v1beta1.IngressReady, + Status: "True", + }, + { + Type: v1beta1.PredictorReady, + Status: "True", + }, + { + Type: apis.ConditionReady, + Status: "True", + }, + }, + }, + URL: &apis.URL{ + Scheme: "http", + Host: "raw-auth-default.example.com", + }, + Address: &duckv1.Addressable{ + URL: &apis.URL{ + Scheme: "http", + Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local", serviceKey.Name, serviceKey.Namespace), + }, + }, + Components: map[v1beta1.ComponentType]v1beta1.ComponentStatusSpec{ + v1beta1.PredictorComponent: { + LatestCreatedRevision: "", + URL: &apis.URL{ + Scheme: "http", + Host: "raw-auth-predictor-default.example.com", + }, + }, + }, + ModelStatus: v1beta1.ModelStatus{ + TransitionStatus: "InProgress", + ModelRevisionStates: &v1beta1.ModelRevisionStates{TargetModelState: "Pending"}, + }, + } + Eventually(func() string { + isvc := &v1beta1.InferenceService{} + if err := k8sClient.Get(context.TODO(), serviceKey, isvc); err != nil { + return err.Error() + } + return cmp.Diff(&expectedIsvcStatus, &isvc.Status, cmpopts.IgnoreTypes(apis.VolatileTime{})) + }, timeout).Should(gomega.BeEmpty()) + + }) + }) Context("When creating inference service with raw kube predictor with workerSpec", func() { var ( ctx context.Context diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 4433c3998df..c68e35aa0db 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -23,6 +23,7 @@ import ( "strconv" "strings" + "k8s.io/apimachinery/pkg/api/resource" "github.com/google/go-cmp/cmp/cmpopts" "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" @@ -130,6 +131,24 @@ func createRawDefaultDeployment(componentMeta metav1.ObjectMeta, podMetadata := componentMeta podMetadata.Labels["app"] = constants.GetRawServiceLabel(componentMeta.Name) setDefaultPodSpec(podSpec) + if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + kserveContainerPort := GetKServeContainerPort(podSpec) + if kserveContainerPort == "" { + kserveContainerPort = constants.InferenceServiceDefaultHttpPort + } + oauthProxyContainer := generateOauthProxyContainer(kserveContainerPort, componentMeta.Namespace) + podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) + tlsSecretVolume := corev1.Volume{ + Name: tlsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: componentMeta.Name, + DefaultMode: func(i int32) *int32 { return &i }(420), // Directly use a pointer + }, + }, + } + podSpec.Volumes = append(podSpec.Volumes, tlsSecretVolume) + } deployment := &appsv1.Deployment{ ObjectMeta: componentMeta, Spec: appsv1.DeploymentSpec{ @@ -180,6 +199,90 @@ func createRawWorkerDeployment(componentMeta metav1.ObjectMeta, return deployment } +func GetKServeContainerPort(podSpec *corev1.PodSpec) string { + for _, container := range podSpec.Containers { + if container.Name == "kserve-container" { + if len(container.Ports) > 0 { + return strconv.Itoa(int(container.Ports[0].ContainerPort)) + } + } + } + return "" +} + +func generateOauthProxyContainer(upstreamPort string, namespace string) corev1.Container { + args := []string{ + `--https-address=:8443`, + `--provider=openshift`, + `--openshift-service-account=kserve-sa`, + `--upstream=http://localhost:$upstreamPort`, + `--tls-cert=/etc/tls/private/tls.crt`, + `--tls-key=/etc/tls/private/tls.key`, + `--cookie-secret=SECRET`, + `--openshift-delegate-urls={"/": {"namespace": "$isvcNamespace", "resource": "services", "verb": "get"}}`, + `--openshift-sar={"namespace": "$isvcNamespace", "resource": "services", "verb": "get"}`, + `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, + } + args[3] = strings.ReplaceAll(args[3], "$upstreamPort", upstreamPort) + args[7] = strings.ReplaceAll(args[7], "$isvcNamespace", namespace) + args[8] = strings.ReplaceAll(args[8], "$isvcNamespace", namespace) + return corev1.Container{ + Name: "oauth-proxy", + Args: args, + Image: constants.OauthProxyImage, + Ports: []corev1.ContainerPort{ + { + ContainerPort: constants.OauthProxyPort, + Name: "https", + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/oauth/healthz", + Port: intstr.FromInt(constants.OauthProxyPort), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/oauth/healthz", + Port: intstr.FromInt(constants.OauthProxyPort), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(constants.OauthProxyResourceCPULimit), + corev1.ResourceMemory: resource.MustParse(constants.OauthProxyResourceMemoryLimit), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(constants.OauthProxyResourceCPURequest), + corev1.ResourceMemory: resource.MustParse(constants.OauthProxyResourceMemoryRequest), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: tlsVolumeName, + MountPath: "/etc/tls/private", + }, + }, + } +} + // checkDeploymentExist checks if the deployment exists? func (r *DeploymentReconciler) checkDeploymentExist(client kclient.Client, deployment *appsv1.Deployment) (constants.CheckResultType, *appsv1.Deployment, error) { // get deployment @@ -295,6 +398,8 @@ func setDefaultDeploymentSpec(spec *appsv1.DeploymentSpec) { progressDeadlineSeconds := int32(600) spec.ProgressDeadlineSeconds = &progressDeadlineSeconds } + + spec.Template.Spec.ServiceAccountName = constants.KserveServiceAccountName } func addGPUResourceToDeployment(deployment *appsv1.Deployment, targetContainerName string, tensorParallelSize string) { diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index f0d508415bc..9011aed8a85 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -150,7 +150,7 @@ func generateIngressHost(ingressConfig *v1beta1.IngressConfig, } func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, - ingressConfig *v1beta1.IngressConfig, client client.Client) (*netv1.Ingress, error) { + ingressConfig *v1beta1.IngressConfig, client client.Client, port int32) (*netv1.Ingress, error) { if !isvc.Status.IsConditionReady(v1beta1.PredictorReady) { isvc.Status.SetCondition(v1beta1.IngressReady, &apis.Condition{ Type: v1beta1.IngressReady, @@ -193,11 +193,11 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, if err != nil { return nil, fmt.Errorf("failed creating explainer ingress host: %w", err) } - rules = append(rules, generateRule(explainerHost, explainerName, "/", constants.CommonDefaultHttpPort)) + rules = append(rules, generateRule(explainerHost, explainerName, "/", port)) } // :predict routes to the transformer when there are both predictor and transformer - rules = append(rules, generateRule(host, transformerName, "/", constants.CommonDefaultHttpPort)) - rules = append(rules, generateRule(transformerHost, predictorName, "/", constants.CommonDefaultHttpPort)) + rules = append(rules, generateRule(host, transformerName, "/", port)) + rules = append(rules, generateRule(transformerHost, predictorName, "/", port)) case isvc.Spec.Explainer != nil: if !isvc.Status.IsConditionReady(v1beta1.ExplainerReady) { isvc.Status.SetCondition(v1beta1.IngressReady, &apis.Condition{ @@ -222,8 +222,8 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, return nil, fmt.Errorf("failed creating explainer ingress host: %w", err) } // :predict routes to the predictor when there is only predictor and explainer - rules = append(rules, generateRule(host, predictorName, "/", constants.CommonDefaultHttpPort)) - rules = append(rules, generateRule(explainerHost, explainerName, "/", constants.CommonDefaultHttpPort)) + rules = append(rules, generateRule(host, predictorName, "/", port)) + rules = append(rules, generateRule(explainerHost, explainerName, "/", port)) default: err := client.Get(context.TODO(), types.NamespacedName{Name: constants.DefaultPredictorServiceName(isvc.Name), Namespace: isvc.Namespace}, existing) if err == nil { @@ -233,15 +233,19 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, if err != nil { return nil, fmt.Errorf("failed creating top level predictor ingress host: %w", err) } - rules = append(rules, generateRule(host, predictorName, "/", constants.CommonDefaultHttpPort)) + rules = append(rules, generateRule(host, predictorName, "/", port)) } // add predictor rule predictorHost, err := generateIngressHost(ingressConfig, isvc, string(constants.Predictor), false, predictorName) if err != nil { return nil, fmt.Errorf("failed creating predictor ingress host: %w", err) } - rules = append(rules, generateRule(predictorHost, predictorName, "/", constants.CommonDefaultHttpPort)) + rules = append(rules, generateRule(predictorHost, predictorName, "/", port)) + //ODH specific change to only have the predictor host be in the ingress + //TODO figure out how raw will work for inferencegraphs where likely multiple hosts will be needed + var predictorRule []netv1.IngressRule + predictorRule = append(predictorRule, generateRule(predictorHost, predictorName, "/", port)) ingress := &netv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: isvc.ObjectMeta.Name, @@ -250,7 +254,8 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, }, Spec: netv1.IngressSpec{ IngressClassName: ingressConfig.IngressClassName, - Rules: rules, + //Rules: rules, + Rules: predictorRule, }, } if err := controllerutil.SetControllerReference(isvc, ingress, scheme); err != nil { @@ -266,6 +271,8 @@ func semanticIngressEquals(desired, existing *netv1.Ingress) bool { func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { var err error isInternal := false + hasAuth := false + port := constants.CommonDefaultHttpPort // disable ingress creation if service is labelled with cluster local or kserve domain is cluster local if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ClusterLocalVisibility { isInternal = true @@ -273,14 +280,24 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { if r.ingressConfig.IngressDomain == constants.ClusterLocalDomain { isInternal = true } + if val, ok := isvc.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + port = constants.OauthProxyPort + hasAuth = true + } if !isInternal && !r.ingressConfig.DisableIngressCreation { - ingress, err := createRawIngress(r.scheme, isvc, r.ingressConfig, r.client) + ingress, err := createRawIngress(r.scheme, isvc, r.ingressConfig, r.client, int32(port)) if ingress == nil { return nil } if err != nil { return err } + if hasAuth { + if ingress.Annotations == nil { + ingress.Annotations = make(map[string]string) + } + ingress.Annotations["route.openshift.io/termination"] = "reencrypt" + } // reconcile ingress existingIngress := &netv1.Ingress{} err = r.client.Get(context.TODO(), types.NamespacedName{ diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go index 1f6e1821843..6f11cf60d3b 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go @@ -19,11 +19,6 @@ package raw import ( "fmt" - "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - autoscaler "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/autoscaler" - deployment "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment" - "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress" - service "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/service" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,6 +26,12 @@ import ( "k8s.io/client-go/kubernetes" knapis "knative.dev/pkg/apis" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + autoscaler "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/autoscaler" + deployment "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment" + "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress" + service "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/service" ) // RawKubeReconciler reconciles the Native K8S Resources @@ -92,13 +93,14 @@ func createRawURL(clientset kubernetes.Interface, metadata metav1.ObjectMeta) (* // Reconcile ... func (r *RawKubeReconciler) Reconcile() ([]*appsv1.Deployment, error) { - // reconcile Deployment - deploymentList, err := r.Deployment.Reconcile() + //reconciling service before deployment because we want to use "service.beta.openshift.io/serving-cert-secret-name" + // reconcile Service + _, err := r.Service.Reconcile() if err != nil { return nil, err } - // reconcile Service - _, err = r.Service.Reconcile() + // reconcile Deployment + deploymentList, err := r.Deployment.Reconcile() if err != nil { return nil, err } diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go index c171d07facb..e3c7f6ba0ec 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go @@ -168,6 +168,26 @@ func createDefaultSvc(componentMeta metav1.ObjectMeta, componentExt *v1beta1.Com ClusterIP: corev1.ClusterIPNone, }, } + if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + if service.ObjectMeta.Annotations == nil { + service.ObjectMeta.Annotations = make(map[string]string) + } + service.ObjectMeta.Annotations["service.beta.openshift.io/serving-cert-secret-name"] = componentMeta.Name + httpsPort := corev1.ServicePort{ + Name: "https", + Port: constants.OauthProxyPort, + TargetPort: intstr.IntOrString{ + Type: intstr.String, + StrVal: "https", + }, + Protocol: corev1.ProtocolTCP, + } + service.Spec.Ports = append(service.Spec.Ports, httpsPort) + } + + for index, port := range service.Spec.Ports { + fmt.Println(index, port.Name, port.Port, port.TargetPort) + } return service } From bb253d6987905c9e6418de332470eaa028a54a78 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Tue, 15 Oct 2024 19:35:30 -0400 Subject: [PATCH 02/20] remove ingress modifications Signed-off-by: Vedant Mahabaleshwarkar --- cmd/manager/main.go | 6 +- config/rbac/role.yaml | 8 ++ pkg/constants/constants.go | 3 +- .../v1beta1/inferenceservice/controller.go | 1 + .../rawkube_controller_test.go | 45 +-------- .../ingress/kube_ingress_reconciler.go | 95 +++++++++++++------ .../reconcilers/service/service_reconciler.go | 20 +++- 7 files changed, 100 insertions(+), 78 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index f9ee225bb78..1ed8530d754 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -50,6 +50,7 @@ import ( v1beta1controller "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice" "github.com/kserve/kserve/pkg/webhook/admission/pod" "github.com/kserve/kserve/pkg/webhook/admission/servingruntime" + routev1 "github.com/openshift/api/route/v1" ) var ( @@ -187,7 +188,10 @@ func main() { } } } - + if err = routev1.AddToScheme(mgr.GetScheme()); err != nil { + setupLog.Error(err, "unable to add routev1 APIs to scheme") + os.Exit(1) + } setupLog.Info("Setting up core scheme") if err := v1.AddToScheme(mgr.GetScheme()); err != nil { setupLog.Error(err, "unable to add Core APIs to scheme") diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 79e3896b837..7b817d46050 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -111,6 +111,14 @@ rules: - patch - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - get + - list + - watch - apiGroups: - serving.knative.dev resources: diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 35c1e1ff496..8818ab6aaed 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -136,7 +136,8 @@ const ( ClusterLocalDomain = "svc.cluster.local" IsvcNameHeader = "KServe-Isvc-Name" IsvcNamespaceHeader = "KServe-Isvc-Namespace" - ODHKserveRawAuth = "networking.kserve.io/odh-auth" + ODHKserveRawAuth = "security.opendatahub.io/enable-auth" + ODHRouteEnabled = "enable-route" ) // StorageSpec Constants diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index c8ca033344b..7b01742acb8 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -76,6 +76,7 @@ import ( // +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch // InferenceServiceState describes the Readiness of the InferenceService type InferenceServiceState string diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index e7e9b619c8d..8fbb70368a5 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -2327,7 +2327,7 @@ var _ = Describe("v1beta1 inference service controller", func() { "serving.kserve.io/deploymentMode": "RawDeployment", }, Labels: map[string]string{ - "networking.kserve.io/odh-auth": "true", + constants.ODHKserveRawAuth: "true", }, }, Spec: v1beta1.InferenceServiceSpec{ @@ -2392,7 +2392,7 @@ var _ = Describe("v1beta1 inference service controller", func() { constants.KServiceComponentLabel: constants.Predictor.String(), constants.InferenceServicePodLabelKey: serviceName, "serving.kserve.io/inferenceservice": serviceName, - "networking.kserve.io/odh-auth": "true", + constants.ODHKserveRawAuth: "true", }, Annotations: map[string]string{ constants.StorageInitializerSourceUriInternalAnnotationKey: *isvc.Spec.Predictor.Model.StorageURI, @@ -2561,12 +2561,6 @@ var _ = Describe("v1beta1 inference service controller", func() { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - { - Name: constants.PredictorServiceName(serviceName), - Protocol: "TCP", - Port: 80, - TargetPort: intstr.IntOrString{Type: 0, IntVal: 8080, StrVal: ""}, - }, { Name: "https", Protocol: "TCP", @@ -2598,41 +2592,6 @@ var _ = Describe("v1beta1 inference service controller", func() { } Expect(k8sClient.Status().Update(context.TODO(), updatedDeployment)).NotTo(gomega.HaveOccurred()) - //check ingress - pathType := netv1.PathTypePrefix - actualIngress := &netv1.Ingress{} - predictorIngressKey := types.NamespacedName{Name: serviceKey.Name, - Namespace: serviceKey.Namespace} - Eventually(func() error { return k8sClient.Get(context.TODO(), predictorIngressKey, actualIngress) }, timeout). - Should(Succeed()) - expectedIngress := netv1.Ingress{ - Spec: netv1.IngressSpec{ - Rules: []netv1.IngressRule{ - { - Host: "raw-auth-predictor-default.example.com", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - PathType: &pathType, - Backend: netv1.IngressBackend{ - Service: &netv1.IngressServiceBackend{ - Name: "raw-auth-predictor", - Port: netv1.ServiceBackendPort{ - Number: 8443, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - Expect(actualIngress.Spec).To(gomega.Equal(expectedIngress.Spec)) // verify if InferenceService status is updated expectedIsvcStatus := v1beta1.InferenceServiceStatus{ Status: duckv1.Status{ diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 9011aed8a85..31215aab5db 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -19,10 +19,12 @@ package ingress import ( "context" "fmt" + "strconv" v1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" "github.com/kserve/kserve/pkg/utils" + routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -150,7 +152,7 @@ func generateIngressHost(ingressConfig *v1beta1.IngressConfig, } func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, - ingressConfig *v1beta1.IngressConfig, client client.Client, port int32) (*netv1.Ingress, error) { + ingressConfig *v1beta1.IngressConfig, client client.Client) (*netv1.Ingress, error) { if !isvc.Status.IsConditionReady(v1beta1.PredictorReady) { isvc.Status.SetCondition(v1beta1.IngressReady, &apis.Condition{ Type: v1beta1.IngressReady, @@ -193,11 +195,11 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, if err != nil { return nil, fmt.Errorf("failed creating explainer ingress host: %w", err) } - rules = append(rules, generateRule(explainerHost, explainerName, "/", port)) + rules = append(rules, generateRule(explainerHost, explainerName, "/", constants.CommonDefaultHttpPort)) } // :predict routes to the transformer when there are both predictor and transformer - rules = append(rules, generateRule(host, transformerName, "/", port)) - rules = append(rules, generateRule(transformerHost, predictorName, "/", port)) + rules = append(rules, generateRule(host, transformerName, "/", constants.CommonDefaultHttpPort)) + rules = append(rules, generateRule(transformerHost, predictorName, "/", constants.CommonDefaultHttpPort)) case isvc.Spec.Explainer != nil: if !isvc.Status.IsConditionReady(v1beta1.ExplainerReady) { isvc.Status.SetCondition(v1beta1.IngressReady, &apis.Condition{ @@ -222,8 +224,8 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, return nil, fmt.Errorf("failed creating explainer ingress host: %w", err) } // :predict routes to the predictor when there is only predictor and explainer - rules = append(rules, generateRule(host, predictorName, "/", port)) - rules = append(rules, generateRule(explainerHost, explainerName, "/", port)) + rules = append(rules, generateRule(host, predictorName, "/", constants.CommonDefaultHttpPort)) + rules = append(rules, generateRule(explainerHost, explainerName, "/", constants.CommonDefaultHttpPort)) default: err := client.Get(context.TODO(), types.NamespacedName{Name: constants.DefaultPredictorServiceName(isvc.Name), Namespace: isvc.Namespace}, existing) if err == nil { @@ -233,19 +235,15 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, if err != nil { return nil, fmt.Errorf("failed creating top level predictor ingress host: %w", err) } - rules = append(rules, generateRule(host, predictorName, "/", port)) + rules = append(rules, generateRule(host, predictorName, "/", constants.CommonDefaultHttpPort)) } // add predictor rule predictorHost, err := generateIngressHost(ingressConfig, isvc, string(constants.Predictor), false, predictorName) if err != nil { return nil, fmt.Errorf("failed creating predictor ingress host: %w", err) } - rules = append(rules, generateRule(predictorHost, predictorName, "/", port)) + rules = append(rules, generateRule(predictorHost, predictorName, "/", constants.CommonDefaultHttpPort)) - //ODH specific change to only have the predictor host be in the ingress - //TODO figure out how raw will work for inferencegraphs where likely multiple hosts will be needed - var predictorRule []netv1.IngressRule - predictorRule = append(predictorRule, generateRule(predictorHost, predictorName, "/", port)) ingress := &netv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: isvc.ObjectMeta.Name, @@ -254,8 +252,7 @@ func createRawIngress(scheme *runtime.Scheme, isvc *v1beta1.InferenceService, }, Spec: netv1.IngressSpec{ IngressClassName: ingressConfig.IngressClassName, - //Rules: rules, - Rules: predictorRule, + Rules: rules, }, } if err := controllerutil.SetControllerReference(isvc, ingress, scheme); err != nil { @@ -268,11 +265,59 @@ func semanticIngressEquals(desired, existing *netv1.Ingress) bool { return equality.Semantic.DeepEqual(desired.Spec, existing.Spec) } +func getRouteURLIfExists(cli client.Client, isvc *v1beta1.InferenceService) (*apis.URL, error) { + foundRoute := false + routeReady := false + route := &routev1.Route{} + err := cli.Get(context.TODO(), types.NamespacedName{Name: isvc.Name, Namespace: isvc.Namespace}, route) + if err != nil { + return nil, err + } + + // Check if the route is owned by the InferenceService + for _, ownerRef := range route.OwnerReferences { + if ownerRef.UID == isvc.UID { + foundRoute = true + } + } + + // Check if the route is admitted + for _, ingress := range route.Status.Ingress { + for _, condition := range ingress.Conditions { + if condition.Type == "Admitted" && condition.Status == "True" { + routeReady = true + } + } + } + + if !(foundRoute && routeReady) { + return nil, fmt.Errorf("route %s/%s not found or not ready", isvc.Namespace, isvc.Name) + } + + // Construct the URL + host := route.Spec.Host + isSecure := false + if isSecure, err = strconv.ParseBool(isvc.Labels[constants.ODHKserveRawAuth]); err != nil { + isSecure = false + } + + scheme := "http" + if isSecure { + scheme = "https" + } + + // Create the URL as an apis.URL object + routeURL := &apis.URL{ + Scheme: scheme, + Host: host, + } + + return routeURL, nil +} + func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { var err error isInternal := false - hasAuth := false - port := constants.CommonDefaultHttpPort // disable ingress creation if service is labelled with cluster local or kserve domain is cluster local if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ClusterLocalVisibility { isInternal = true @@ -280,24 +325,14 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { if r.ingressConfig.IngressDomain == constants.ClusterLocalDomain { isInternal = true } - if val, ok := isvc.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { - port = constants.OauthProxyPort - hasAuth = true - } if !isInternal && !r.ingressConfig.DisableIngressCreation { - ingress, err := createRawIngress(r.scheme, isvc, r.ingressConfig, r.client, int32(port)) + ingress, err := createRawIngress(r.scheme, isvc, r.ingressConfig, r.client) if ingress == nil { return nil } if err != nil { return err } - if hasAuth { - if ingress.Annotations == nil { - ingress.Annotations = make(map[string]string) - } - ingress.Annotations["route.openshift.io/termination"] = "reencrypt" - } // reconcile ingress existingIngress := &netv1.Ingress{} err = r.client.Get(context.TODO(), types.NamespacedName{ @@ -321,7 +356,11 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { return err } } - isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig) + if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ODHRouteEnabled { + isvc.Status.URL, err = getRouteURLIfExists(r.client, isvc) + } else { + isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig) + } if err != nil { return err } diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go index e3c7f6ba0ec..46564ea8709 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go @@ -110,6 +110,9 @@ func createDefaultSvc(componentMeta metav1.ObjectMeta, componentExt *v1beta1.Com }, Protocol: container.Ports[0].Protocol, } + if servicePort.Name == "" { + servicePort.Name = "http" + } servicePorts = append(servicePorts, servicePort) for i := 1; i < len(container.Ports); i++ { @@ -182,11 +185,18 @@ func createDefaultSvc(componentMeta metav1.ObjectMeta, componentExt *v1beta1.Com }, Protocol: corev1.ProtocolTCP, } - service.Spec.Ports = append(service.Spec.Ports, httpsPort) - } - - for index, port := range service.Spec.Ports { - fmt.Println(index, port.Name, port.Port, port.TargetPort) + ports := service.Spec.Ports + replaced := false + for i, port := range ports { + if port.Port == constants.CommonDefaultHttpPort { + ports[i] = httpsPort + replaced = true + } + } + if !replaced { + ports = append(ports, httpsPort) + } + service.Spec.Ports = ports } return service } From c6ade9bc5db6b47586af03defeb96f99ce5db174 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 16 Oct 2024 11:04:39 -0400 Subject: [PATCH 03/20] bug fix Signed-off-by: Vedant Mahabaleshwarkar --- .../reconcilers/ingress/kube_ingress_reconciler.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 31215aab5db..55c59bcc46a 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -356,10 +356,12 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { return err } } + isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig) if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ODHRouteEnabled { - isvc.Status.URL, err = getRouteURLIfExists(r.client, isvc) - } else { - isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig) + routeUrl, err := getRouteURLIfExists(r.client, isvc) + if err == nil && routeUrl != nil { + isvc.Status.URL = routeUrl + } } if err != nil { return err From 79fd6d59ecc488d9135950c7797585160de8b3aa Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 16 Oct 2024 13:40:32 -0400 Subject: [PATCH 04/20] consume oauth proxy params from configmap Signed-off-by: Vedant Mahabaleshwarkar --- .../odh/inferenceservice-config-patch.yaml | 8 ++ config/overlays/odh/kustomization.yaml | 7 ++ config/overlays/odh/params.env | 1 + .../deployment/deployment_reconciler.go | 93 +++++++++++++------ 4 files changed, 80 insertions(+), 29 deletions(-) diff --git a/config/overlays/odh/inferenceservice-config-patch.yaml b/config/overlays/odh/inferenceservice-config-patch.yaml index 9109221ec33..679e2d454af 100644 --- a/config/overlays/odh/inferenceservice-config-patch.yaml +++ b/config/overlays/odh/inferenceservice-config-patch.yaml @@ -5,6 +5,14 @@ metadata: namespace: kserve data: explainers: "{}" + oauthProxy: |- + { + "image" : "$(oauth-proxy)", + "memoryRequest": "256Mi", + "memoryLimit": "256Mi", + "cpuRequest": "100m", + "cpuLimit": "100m", + } storageInitializer: |- { "image" : "$(kserve-storage-initializer)", diff --git a/config/overlays/odh/kustomization.yaml b/config/overlays/odh/kustomization.yaml index 863bc66a360..5ae94a23073 100644 --- a/config/overlays/odh/kustomization.yaml +++ b/config/overlays/odh/kustomization.yaml @@ -54,6 +54,13 @@ vars: apiVersion: v1 kind: ConfigMap name: kserve-parameters +- fieldref: + fieldpath: data.oauth-proxy + name: oauth-proxy + objref: + apiVersion: v1 + kind: ConfigMap + name: kserve-parameters configurations: - params.yaml diff --git a/config/overlays/odh/params.env b/config/overlays/odh/params.env index f4233dc6a6e..80c20e3f75b 100644 --- a/config/overlays/odh/params.env +++ b/config/overlays/odh/params.env @@ -2,3 +2,4 @@ kserve-controller=quay.io/opendatahub/kserve-controller:latest kserve-agent=quay.io/opendatahub/kserve-agent:latest kserve-router=quay.io/opendatahub/kserve-router:latest kserve-storage-initializer=quay.io/opendatahub/kserve-storage-initializer:latest +oauth-proxy=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:4bef31eb993feb6f1096b51b4876c65a6fb1f4401fee97fa4f4542b6b7c9bc46 \ No newline at end of file diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index c68e35aa0db..5027f54a4b9 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -53,6 +53,10 @@ type DeploymentReconciler struct { componentExt *v1beta1.ComponentExtensionSpec } +const ( + tlsVolumeName = "proxy-tls" +) + func NewDeploymentReconciler(client kclient.Client, scheme *runtime.Scheme, componentMeta metav1.ObjectMeta, @@ -62,11 +66,11 @@ func NewDeploymentReconciler(client kclient.Client, return &DeploymentReconciler{ client: client, scheme: scheme, - DeploymentList: createRawDeployment(componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), + DeploymentList: createRawDeployment(client, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), componentExt: componentExt, } } -func createRawDeployment(componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, +func createRawDeployment(cli kclient.Client, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) []*appsv1.Deployment { var deploymentList []*appsv1.Deployment @@ -94,7 +98,7 @@ func createRawDeployment(componentMeta metav1.ObjectMeta, workerComponentMeta me } } - defaultDeployment := createRawDefaultDeployment(componentMeta, componentExt, podSpec) + defaultDeployment := createRawDefaultDeployment(cli, componentMeta, componentExt, podSpec) if multiNodeEnabled { // Use defaut value(1) if tensor-parallel-size is not set (gpu count) tensorParallelSize = constants.DefaultTensorParallelSize @@ -125,7 +129,7 @@ func createRawDeployment(componentMeta metav1.ObjectMeta, workerComponentMeta me return deploymentList } -func createRawDefaultDeployment(componentMeta metav1.ObjectMeta, +func createRawDefaultDeployment(cli kclient.Client, componentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec) *appsv1.Deployment { podMetadata := componentMeta @@ -136,8 +140,10 @@ func createRawDefaultDeployment(componentMeta metav1.ObjectMeta, if kserveContainerPort == "" { kserveContainerPort = constants.InferenceServiceDefaultHttpPort } - oauthProxyContainer := generateOauthProxyContainer(kserveContainerPort, componentMeta.Namespace) - podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) + oauthProxyContainer, err := generateOauthProxyContainer(cli, kserveContainerPort, componentMeta.Namespace) + if err != nil { + podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) + } tlsSecretVolume := corev1.Volume{ Name: tlsVolumeName, VolumeSource: corev1.VolumeSource{ @@ -210,26 +216,55 @@ func GetKServeContainerPort(podSpec *corev1.PodSpec) string { return "" } -func generateOauthProxyContainer(upstreamPort string, namespace string) corev1.Container { - args := []string{ - `--https-address=:8443`, - `--provider=openshift`, - `--openshift-service-account=kserve-sa`, - `--upstream=http://localhost:$upstreamPort`, - `--tls-cert=/etc/tls/private/tls.crt`, - `--tls-key=/etc/tls/private/tls.key`, - `--cookie-secret=SECRET`, - `--openshift-delegate-urls={"/": {"namespace": "$isvcNamespace", "resource": "services", "verb": "get"}}`, - `--openshift-sar={"namespace": "$isvcNamespace", "resource": "services", "verb": "get"}`, - `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, +func generateOauthProxyContainer(cli kclient.Client, upstreamPort string, namespace string) (corev1.Container, error) { + oauthImage := constants.OauthProxyImage + oauthCpuLimit := constants.OauthProxyResourceCPULimit + oauthMemoryLimit := constants.OauthProxyResourceMemoryLimit + oauthCpuRequest := constants.OauthProxyResourceCPURequest + oauthMemoryRequest := constants.OauthProxyResourceMemoryRequest + inferenceServiceConfigMap := &corev1.ConfigMap{} + err := cli.Get(context.TODO(), client.ObjectKey{ + Namespace: constants.KServeNamespace, + Name: constants.InferenceServiceConfigMapName, + }, inferenceServiceConfigMap) + if err == nil { + var oauthData map[string]interface{} + if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &oauthData); err != nil { + return corev1.Container{}, fmt.Errorf("error retrieving value for key 'oauthProxy' from configmap %s. %w", + constants.InferenceServiceConfigMapName, err) + } + if str, ok := oauthData["image"].(string); !ok { + oauthImage = str + } + if str, ok := oauthData["cpuLimit"].(string); !ok { + oauthCpuLimit = str + } + if str, ok := oauthData["memoryLimit"].(string); !ok { + oauthMemoryLimit = str + } + if str, ok := oauthData["cpuRequest"].(string); !ok { + oauthCpuRequest = str + } + if str, ok := oauthData["memoryRequest"].(string); !ok { + oauthMemoryRequest = str + } } - args[3] = strings.ReplaceAll(args[3], "$upstreamPort", upstreamPort) - args[7] = strings.ReplaceAll(args[7], "$isvcNamespace", namespace) - args[8] = strings.ReplaceAll(args[8], "$isvcNamespace", namespace) + return corev1.Container{ - Name: "oauth-proxy", - Args: args, - Image: constants.OauthProxyImage, + Name: "oauth-proxy", + Args: []string{ + `--https-address=:8443`, + `--provider=openshift`, + `--openshift-service-account=kserve-sa`, + `--upstream=http://localhost:` + upstreamPort, + `--tls-cert=/etc/tls/private/tls.crt`, + `--tls-key=/etc/tls/private/tls.key`, + `--cookie-secret=SECRET`, + `--openshift-delegate-urls={"/": {"namespace": "` + namespace + `", "resource": "services", "verb": "get"}}`, + `--openshift-sar={"namespace": "` + namespace + `", "resource": "services", "verb": "get"}`, + `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, + }, + Image: oauthImage, Ports: []corev1.ContainerPort{ { ContainerPort: constants.OauthProxyPort, @@ -266,12 +301,12 @@ func generateOauthProxyContainer(upstreamPort string, namespace string) corev1.C }, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(constants.OauthProxyResourceCPULimit), - corev1.ResourceMemory: resource.MustParse(constants.OauthProxyResourceMemoryLimit), + corev1.ResourceCPU: resource.MustParse(oauthCpuLimit), + corev1.ResourceMemory: resource.MustParse(oauthMemoryLimit), }, Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(constants.OauthProxyResourceCPURequest), - corev1.ResourceMemory: resource.MustParse(constants.OauthProxyResourceMemoryRequest), + corev1.ResourceCPU: resource.MustParse(oauthCpuRequest), + corev1.ResourceMemory: resource.MustParse(oauthMemoryRequest), }, }, VolumeMounts: []corev1.VolumeMount{ @@ -280,7 +315,7 @@ func generateOauthProxyContainer(upstreamPort string, namespace string) corev1.C MountPath: "/etc/tls/private", }, }, - } + }, nil } // checkDeploymentExist checks if the deployment exists? From 970938d6349adc802f4fa6973a3ccebe2ca58213 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 16 Oct 2024 15:28:23 -0400 Subject: [PATCH 05/20] fix oauth proxy sar and minor bugs Signed-off-by: Vedant Mahabaleshwarkar --- .../rawkube_controller_test.go | 8 +-- .../deployment/deployment_reconciler.go | 69 ++++++++++--------- .../reconcilers/raw/raw_kube_reconciler.go | 2 +- .../reconcilers/service/service_reconciler.go | 6 +- 4 files changed, 46 insertions(+), 39 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index 8fbb70368a5..c5e8bd5dde3 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -2397,7 +2397,7 @@ var _ = Describe("v1beta1 inference service controller", func() { Annotations: map[string]string{ constants.StorageInitializerSourceUriInternalAnnotationKey: *isvc.Spec.Predictor.Model.StorageURI, "serving.kserve.io/deploymentMode": "RawDeployment", - "service.beta.openshift.io/serving-cert-secret-name": predictorDeploymentKey.Name, + "service.beta.openshift.io/serving-cert-secret-name": serviceName, }, }, Spec: v1.PodSpec{ @@ -2443,8 +2443,8 @@ var _ = Describe("v1beta1 inference service controller", func() { `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, `--cookie-secret=SECRET`, - `--openshift-delegate-urls={"/": {"namespace": "` + serviceKey.Namespace + `", "resource": "services", "verb": "get"}}`, - `--openshift-sar={"namespace": "` + serviceKey.Namespace + `", "resource": "services", "verb": "get"}`, + `--openshift-delegate-urls={"/": {"namespace": "` + serviceKey.Namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + serviceName + `", "verb": "get"}}`, + `--openshift-sar={"namespace": "` + serviceKey.Namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + serviceName + `", "verb": "get"}`, `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, }, Ports: []v1.ContainerPort{ @@ -2508,7 +2508,7 @@ var _ = Describe("v1beta1 inference service controller", func() { Name: "proxy-tls", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: predictorDeploymentKey.Name, + SecretName: serviceName, DefaultMode: func(i int32) *int32 { return &i }(420), }, }, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 5027f54a4b9..3abca6d2cc4 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -23,7 +23,9 @@ import ( "strconv" "strings" + "k8s.io/client-go/kubernetes" "k8s.io/apimachinery/pkg/api/resource" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" @@ -31,14 +33,12 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierr "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/strategicpatch" "knative.dev/pkg/kmp" - "sigs.k8s.io/controller-runtime/pkg/client" kclient "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -58,6 +58,7 @@ const ( ) func NewDeploymentReconciler(client kclient.Client, + clientset kubernetes.Interface, scheme *runtime.Scheme, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, @@ -66,7 +67,7 @@ func NewDeploymentReconciler(client kclient.Client, return &DeploymentReconciler{ client: client, scheme: scheme, - DeploymentList: createRawDeployment(client, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), + DeploymentList: createRawDeployment(clientset, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), componentExt: componentExt, } } @@ -129,27 +130,31 @@ func createRawDeployment(cli kclient.Client, componentMeta metav1.ObjectMeta, wo return deploymentList } -func createRawDefaultDeployment(cli kclient.Client, componentMeta metav1.ObjectMeta, +func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec) *appsv1.Deployment { podMetadata := componentMeta podMetadata.Labels["app"] = constants.GetRawServiceLabel(componentMeta.Name) setDefaultPodSpec(podSpec) + isvcname := componentMeta.Name if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { + isvcname = val + } kserveContainerPort := GetKServeContainerPort(podSpec) if kserveContainerPort == "" { kserveContainerPort = constants.InferenceServiceDefaultHttpPort } - oauthProxyContainer, err := generateOauthProxyContainer(cli, kserveContainerPort, componentMeta.Namespace) - if err != nil { + oauthProxyContainer, err := generateOauthProxyContainer(clientset, isvcname, componentMeta.Namespace, kserveContainerPort) + if err == nil { podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) } tlsSecretVolume := corev1.Volume{ Name: tlsVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: componentMeta.Name, - DefaultMode: func(i int32) *int32 { return &i }(420), // Directly use a pointer + SecretName: isvcname, + DefaultMode: func(i int32) *int32 { return &i }(420), }, }, } @@ -216,37 +221,35 @@ func GetKServeContainerPort(podSpec *corev1.PodSpec) string { return "" } -func generateOauthProxyContainer(cli kclient.Client, upstreamPort string, namespace string) (corev1.Container, error) { +func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string) (corev1.Container, error) { oauthImage := constants.OauthProxyImage oauthCpuLimit := constants.OauthProxyResourceCPULimit oauthMemoryLimit := constants.OauthProxyResourceMemoryLimit oauthCpuRequest := constants.OauthProxyResourceCPURequest oauthMemoryRequest := constants.OauthProxyResourceMemoryRequest - inferenceServiceConfigMap := &corev1.ConfigMap{} - err := cli.Get(context.TODO(), client.ObjectKey{ - Namespace: constants.KServeNamespace, - Name: constants.InferenceServiceConfigMapName, - }, inferenceServiceConfigMap) + inferenceServiceConfigMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) + configDataSuccess := false if err == nil { var oauthData map[string]interface{} - if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &oauthData); err != nil { - return corev1.Container{}, fmt.Errorf("error retrieving value for key 'oauthProxy' from configmap %s. %w", - constants.InferenceServiceConfigMapName, err) - } - if str, ok := oauthData["image"].(string); !ok { - oauthImage = str - } - if str, ok := oauthData["cpuLimit"].(string); !ok { - oauthCpuLimit = str - } - if str, ok := oauthData["memoryLimit"].(string); !ok { - oauthMemoryLimit = str + if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &oauthData); err == nil { + configDataSuccess = true } - if str, ok := oauthData["cpuRequest"].(string); !ok { - oauthCpuRequest = str - } - if str, ok := oauthData["memoryRequest"].(string); !ok { - oauthMemoryRequest = str + if configDataSuccess { + if str, ok := oauthData["image"].(string); ok { + oauthImage = str + } + if str, ok := oauthData["cpuLimit"].(string); ok { + oauthCpuLimit = str + } + if str, ok := oauthData["memoryLimit"].(string); ok { + oauthMemoryLimit = str + } + if str, ok := oauthData["cpuRequest"].(string); ok { + oauthCpuRequest = str + } + if str, ok := oauthData["memoryRequest"].(string); ok { + oauthMemoryRequest = str + } } } @@ -260,8 +263,8 @@ func generateOauthProxyContainer(cli kclient.Client, upstreamPort string, namesp `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, `--cookie-secret=SECRET`, - `--openshift-delegate-urls={"/": {"namespace": "` + namespace + `", "resource": "services", "verb": "get"}}`, - `--openshift-sar={"namespace": "` + namespace + `", "resource": "services", "verb": "get"}`, + `--openshift-delegate-urls={"/": {"namespace": "` + namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + isvc + `", "verb": "get"}}`, + `--openshift-sar={"namespace": "` + namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + isvc + `", "verb": "get"}`, `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, }, Image: oauthImage, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go index 6f11cf60d3b..e934b18e0c2 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go @@ -68,7 +68,7 @@ func NewRawKubeReconciler(client client.Client, return &RawKubeReconciler{ client: client, scheme: scheme, - Deployment: deployment.NewDeploymentReconciler(client, scheme, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), + Deployment: deployment.NewDeploymentReconciler(client, clientset, scheme, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), Service: service.NewServiceReconciler(client, scheme, componentMeta, componentExt, podSpec, multiNodeEnabled), Scaler: as, URL: url, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go index 46564ea8709..63a067e927a 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go @@ -175,7 +175,11 @@ func createDefaultSvc(componentMeta metav1.ObjectMeta, componentExt *v1beta1.Com if service.ObjectMeta.Annotations == nil { service.ObjectMeta.Annotations = make(map[string]string) } - service.ObjectMeta.Annotations["service.beta.openshift.io/serving-cert-secret-name"] = componentMeta.Name + isvcname := componentMeta.Name + if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { + isvcname = val + } + service.ObjectMeta.Annotations["service.beta.openshift.io/serving-cert-secret-name"] = isvcname httpsPort := corev1.ServicePort{ Name: "https", Port: constants.OauthProxyPort, From 6b3f7af68e3e162db192de743aecf72ceff60d9c Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 16 Oct 2024 15:40:39 -0400 Subject: [PATCH 06/20] revert some unneeded changes Signed-off-by: Vedant Mahabaleshwarkar --- config/overlays/odh/inferenceservice-config-patch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/overlays/odh/inferenceservice-config-patch.yaml b/config/overlays/odh/inferenceservice-config-patch.yaml index 679e2d454af..e0c1271f970 100644 --- a/config/overlays/odh/inferenceservice-config-patch.yaml +++ b/config/overlays/odh/inferenceservice-config-patch.yaml @@ -31,6 +31,7 @@ data: "localGateway" : "istio-system/kserve-local-gateway", "localGatewayService" : "kserve-local-gateway.istio-system.svc.cluster.local", "ingressDomain" : "example.com", + "ingressClassName" : "istio", "domainTemplate": "{{ .Name }}-{{ .Namespace }}.{{ .IngressDomain }}", "urlScheme": "https", "disableIstioVirtualHost": false, From 6abba54b603c1d7aba95a7202fc3c979c31a6309 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Thu, 17 Oct 2024 12:56:13 -0400 Subject: [PATCH 07/20] add oauth proxy flag to prevent login page redirect on invalid request Signed-off-by: Vedant Mahabaleshwarkar --- .../reconcilers/deployment/deployment_reconciler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 3abca6d2cc4..eecd0bac5b0 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -258,6 +258,7 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na Args: []string{ `--https-address=:8443`, `--provider=openshift`, + `--skip-provider-button`, `--openshift-service-account=kserve-sa`, `--upstream=http://localhost:` + upstreamPort, `--tls-cert=/etc/tls/private/tls.crt`, From 8028a3bcaf0316a01ebc7591184e7235cfddbdab Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Fri, 8 Nov 2024 16:01:21 -0500 Subject: [PATCH 08/20] address feedback Signed-off-by: Vedant Mahabaleshwarkar 1 Signed-off-by: Vedant Mahabaleshwarkar 2 Signed-off-by: Vedant Mahabaleshwarkar --- go.mod | 17 +- go.sum | 270 ++---------------- pkg/constants/constants.go | 18 +- .../rawkube_controller_test.go | 16 +- .../deployment/deployment_reconciler.go | 42 +-- .../ingress/kube_ingress_reconciler.go | 9 +- .../reconcilers/service/service_reconciler.go | 6 +- 7 files changed, 70 insertions(+), 308 deletions(-) diff --git a/go.mod b/go.mod index b12eef28da6..4c32d5e1c55 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.2 + github.com/openshift/api v0.0.0-20241108213852-e22f17d9b7f5 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 @@ -30,10 +31,10 @@ require ( gopkg.in/go-playground/validator.v9 v9.31.0 istio.io/api v1.23.0 istio.io/client-go v1.23.0 - k8s.io/api v0.30.4 - k8s.io/apimachinery v0.30.4 - k8s.io/client-go v0.30.4 - k8s.io/code-generator v0.30.4 + k8s.io/api v0.31.2 + k8s.io/apimachinery v0.31.2 + k8s.io/client-go v0.31.0 + k8s.io/code-generator v0.31.0 k8s.io/component-helpers v0.30.4 k8s.io/klog v1.0.0 k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 @@ -41,7 +42,7 @@ require ( knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c knative.dev/serving v0.42.2 - sigs.k8s.io/controller-runtime v0.18.5 + sigs.k8s.io/controller-runtime v0.19.1 sigs.k8s.io/yaml v1.4.0 ) @@ -59,9 +60,9 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -107,6 +108,7 @@ require ( github.com/prometheus/statsd_exporter v0.27.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect + github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect @@ -130,11 +132,12 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/grpc v1.66.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.30.4 // indirect + k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 // indirect k8s.io/klog/v2 v2.130.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index da09421d8fa..d4b08833c45 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -17,127 +16,28 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= -cloud.google.com/go/accessapproval v1.8.0/go.mod h1:ycc7qSIXOrH6gGOGQsuBwpRZw3QhZLi0OWeej3rA5Mg= -cloud.google.com/go/accesscontextmanager v1.9.0/go.mod h1:EmdQRGq5FHLrjGjGTp2X2tlRBvU3LDCUqfnysFYooxQ= -cloud.google.com/go/aiplatform v1.68.0/go.mod h1:105MFA3svHjC3Oazl7yjXAmIR89LKhRAeNdnDKJczME= -cloud.google.com/go/analytics v0.25.0/go.mod h1:LZMfjJnKU1GDkvJV16dKnXm7KJJaMZfvUXx58ujgVLg= -cloud.google.com/go/apigateway v1.7.0/go.mod h1:miZGNhmrC+SFhxjA7ayjKHk1cA+7vsSINp9K+JxKwZI= -cloud.google.com/go/apigeeconnect v1.7.0/go.mod h1:fd8NFqzu5aXGEUpxiyeCyb4LBLU7B/xIPztfBQi+1zg= -cloud.google.com/go/apigeeregistry v0.9.0/go.mod h1:4S/btGnijdt9LSIZwBDHgtYfYkFGekzNyWkyYTP8Qzs= -cloud.google.com/go/appengine v1.9.0/go.mod h1:y5oI+JT3/6s77QmxbTnLHyiMKz3NPHYOjuhmVi+FyYU= -cloud.google.com/go/area120 v0.9.0/go.mod h1:ujIhRz2gJXutmFYGAUgz3KZ5IRJ6vOwL4CYlNy/jDo4= -cloud.google.com/go/artifactregistry v1.15.0/go.mod h1:4xrfigx32/3N7Pp7YSPOZZGs4VPhyYeRyJ67ZfVdOX4= -cloud.google.com/go/asset v1.20.0/go.mod h1:CT3ME6xNZKsPSvi0lMBPgW3azvRhiurJTFSnNl6ahw8= -cloud.google.com/go/assuredworkloads v1.12.0/go.mod h1:jX84R+0iANggmSbzvVgrGWaqdhRsQihAv4fF7IQ4r7Q= cloud.google.com/go/auth v0.9.1 h1:+pMtLEV2k0AXKvs/tGZojuj6QaioxfUjOpMsG5Gtx+w= cloud.google.com/go/auth v0.9.1/go.mod h1:Sw8ocT5mhhXxFklyhT12Eiy0ed6tTrPMCJjSI8KhYLk= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= -cloud.google.com/go/automl v1.14.0/go.mod h1:Kr7rN9ANSjlHyBLGvwhrnt35/vVZy3n/CP4Xmyj0shM= -cloud.google.com/go/baremetalsolution v1.3.0/go.mod h1:E+n44UaDVO5EeSa4SUsDFxQLt6dD1CoE2h+mtxxaJKo= -cloud.google.com/go/batch v1.10.0/go.mod h1:JlktZqyKbcUJWdHOV8juvAiQNH8xXHXTqLp6bD9qreE= -cloud.google.com/go/beyondcorp v1.1.0/go.mod h1:F6Rl20QbayaloWIsMhuz+DICcJxckdFKc7R2HCe6iNA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.62.0/go.mod h1:5ee+ZkF1x/ntgCsFQJAQTM3QkAZOecfCmvxhkJsWRSA= -cloud.google.com/go/bigtable v1.29.0/go.mod h1:5p909nNdWaNUcWs6KGZO8mI5HUovstlmrIi7+eA5PTQ= -cloud.google.com/go/billing v1.19.0/go.mod h1:bGvChbZguyaWRGmu5pQHfFN1VxTDPFmabnCVA/dNdRM= -cloud.google.com/go/binaryauthorization v1.9.0/go.mod h1:fssQuxfI9D6dPPqfvDmObof+ZBKsxA9iSigd8aSA1ik= -cloud.google.com/go/certificatemanager v1.9.0/go.mod h1:hQBpwtKNjUq+er6Rdg675N7lSsNGqMgt7Bt7Dbcm7d0= -cloud.google.com/go/channel v1.18.0/go.mod h1:gQr50HxC/FGvufmqXD631ldL1Ee7CNMU5F4pDyJWlt0= -cloud.google.com/go/cloudbuild v1.17.0/go.mod h1:/RbwgDlbQEwIKoWLIYnW72W3cWs+e83z7nU45xRKnj8= -cloud.google.com/go/clouddms v1.8.0/go.mod h1:JUgTgqd1M9iPa7p3jodjLTuecdkGTcikrg7nz++XB5E= -cloud.google.com/go/cloudtasks v1.13.0/go.mod h1:O1jFRGb1Vm3sN2u/tBdPiVGVTWIsrsbEs3K3N3nNlEU= -cloud.google.com/go/compute v1.28.0/go.mod h1:DEqZBtYrDnD5PvjsKwb3onnhX+qjdCVM7eshj1XdjV4= cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= -cloud.google.com/go/contactcenterinsights v1.14.0/go.mod h1:APmWYHDN4sASnUBnXs4o68t1EUfnqadA53//CzXZ1xE= -cloud.google.com/go/container v1.39.0/go.mod h1:gNgnvs1cRHXjYxrotVm+0nxDfZkqzBbXCffh5WtqieI= -cloud.google.com/go/containeranalysis v0.13.0/go.mod h1:OpufGxsNzMOZb6w5yqwUgHr5GHivsAD18KEI06yGkQs= -cloud.google.com/go/datacatalog v1.22.0/go.mod h1:4Wff6GphTY6guF5WphrD76jOdfBiflDiRGFAxq7t//I= -cloud.google.com/go/dataflow v0.10.0/go.mod h1:zAv3YUNe/2pXWKDSPvbf31mCIUuJa+IHtKmhfzaeGww= -cloud.google.com/go/dataform v0.10.0/go.mod h1:0NKefI6v1ppBEDnwrp6gOMEA3s/RH3ypLUM0+YWqh6A= -cloud.google.com/go/datafusion v1.8.0/go.mod h1:zHZ5dJYHhMP1P8SZDZm+6yRY9BCCcfm7Xg7YmP+iA6E= -cloud.google.com/go/datalabeling v0.9.0/go.mod h1:GVX4sW4cY5OPKu/9v6dv20AU9xmGr4DXR6K26qN0mzw= -cloud.google.com/go/dataplex v1.19.0/go.mod h1:5H9ftGuZWMtoEIUpTdGUtGgje36YGmtRXoC8wx6QSUc= -cloud.google.com/go/dataproc/v2 v2.6.0/go.mod h1:amsKInI+TU4GcXnz+gmmApYbiYM4Fw051SIMDoWCWeE= -cloud.google.com/go/dataqna v0.9.0/go.mod h1:WlRhvLLZv7TfpONlb/rEQx5Qrr7b5sxgSuz5NP6amrw= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.19.0/go.mod h1:KGzkszuj87VT8tJe67GuB+qLolfsOt6bZq/KFuWaahc= -cloud.google.com/go/datastream v1.11.0/go.mod h1:vio/5TQ0qNtGcIj7sFb0gucFoqZW19gZ7HztYtkzq9g= -cloud.google.com/go/deploy v1.22.0/go.mod h1:qXJgBcnyetoOe+w/79sCC99c5PpHJsgUXCNhwMjG0e4= -cloud.google.com/go/dialogflow v1.57.0/go.mod h1:wegtnocuYEfue6IGlX96n5mHu3JGZUaZxv1L5HzJUJY= -cloud.google.com/go/dlp v1.18.0/go.mod h1:RVO9zkh+xXgUa7+YOf9IFNHL/2FXt9Vnv/GKNYmc1fE= -cloud.google.com/go/documentai v1.33.0/go.mod h1:lI9Mti9COZ5qVjdpfDZxNjOrTVf6tJ//vaqbtt81214= -cloud.google.com/go/domains v0.10.0/go.mod h1:VpPXnkCNRsxkieDFDfjBIrLv3p1kRjJ03wLoPeL30To= -cloud.google.com/go/edgecontainer v1.3.0/go.mod h1:dV1qTl2KAnQOYG+7plYr53KSq/37aga5/xPgOlYXh3A= -cloud.google.com/go/errorreporting v0.3.1/go.mod h1:6xVQXU1UuntfAf+bVkFk6nld41+CPyF2NSPCyXE3Ztk= -cloud.google.com/go/essentialcontacts v1.7.0/go.mod h1:0JEcNuyjyg43H/RJynZzv2eo6MkmnvRPUouBpOh6akY= -cloud.google.com/go/eventarc v1.14.0/go.mod h1:60ZzZfOekvsc/keHc7uGHcoEOMVa+p+ZgRmTjpdamnA= -cloud.google.com/go/filestore v1.9.0/go.mod h1:GlQK+VBaAGb19HqprnOMqYYpn7Gev5ZA9SSHpxFKD7Q= -cloud.google.com/go/firestore v1.16.0/go.mod h1:+22v/7p+WNBSQwdSwP57vz47aZiY+HrDkrOsJNhk7rg= -cloud.google.com/go/functions v1.19.0/go.mod h1:WDreEDZoUVoOkXKDejFWGnprrGYn2cY2KHx73UQERC0= -cloud.google.com/go/gkebackup v1.6.0/go.mod h1:1rskt7NgawoMDHTdLASX8caXXYG3MvDsoZ7qF4RMamQ= -cloud.google.com/go/gkeconnect v0.10.0/go.mod h1:d8TE+YAlX7mvq8pWy1Q4yOnmxbN0SimmcQdtJwBdUHk= -cloud.google.com/go/gkehub v0.15.0/go.mod h1:obpeROly2mjxZJbRkFfHEflcH54XhJI+g2QgfHphL0I= -cloud.google.com/go/gkemulticloud v1.3.0/go.mod h1:XmcOUQ+hJI62fi/klCjEGs6lhQ56Zjs14sGPXsGP0mE= -cloud.google.com/go/gsuiteaddons v1.7.0/go.mod h1:/B1L8ANPbiSvxCgdSwqH9CqHIJBzTt6v50fPr3vJCtg= cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8= cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q= -cloud.google.com/go/iap v1.10.0/go.mod h1:gDT6LZnKnWNCaov/iQbj7NMUpknFDOkhhlH8PwIrpzU= -cloud.google.com/go/ids v1.5.0/go.mod h1:4NOlC1m9hAJL50j2cRV4PS/J6x/f4BBM0Xg54JQLCWw= -cloud.google.com/go/iot v1.8.0/go.mod h1:/NMFENPnQ2t1UByUC1qFvA80fo1KFB920BlyUPn1m3s= -cloud.google.com/go/kms v1.19.0/go.mod h1:e4imokuPJUc17Trz2s6lEXFDt8bgDmvpVynH39bdrHM= -cloud.google.com/go/language v1.14.0/go.mod h1:ldEdlZOFwZREnn/1yWtXdNzfD7hHi9rf87YDkOY9at4= -cloud.google.com/go/lifesciences v0.10.0/go.mod h1:1zMhgXQ7LbMbA5n4AYguFgbulbounfUoYvkV8dtsLcA= -cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI= cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts= -cloud.google.com/go/managedidentities v1.7.0/go.mod h1:o4LqQkQvJ9Pt7Q8CyZV39HrzCfzyX8zBzm8KIhRw91E= -cloud.google.com/go/maps v1.12.0/go.mod h1:qjErDNStn3BaGx06vHner5d75MRMgGflbgCuWTuslMc= -cloud.google.com/go/mediatranslation v0.9.0/go.mod h1:udnxo0i4YJ5mZfkwvvQQrQ6ra47vcX8jeGV+6I5x+iU= -cloud.google.com/go/memcache v1.11.0/go.mod h1:99MVF02m5TByT1NKxsoKDnw5kYmMrjbGSeikdyfCYZk= -cloud.google.com/go/metastore v1.14.0/go.mod h1:vtPt5oVF/+ocXO4rv4GUzC8Si5s8gfmo5OIt6bACDuE= -cloud.google.com/go/monitoring v1.21.0/go.mod h1:tuJ+KNDdJbetSsbSGTqnaBvbauS5kr3Q/koy3Up6r+4= -cloud.google.com/go/networkconnectivity v1.15.0/go.mod h1:uBQqx/YHI6gzqfV5J/7fkKwTGlXvQhHevUuzMpos9WY= -cloud.google.com/go/networkmanagement v1.14.0/go.mod h1:4myfd4A0uULCOCGHL1npZN0U+kr1Z2ENlbHdCCX4cE8= -cloud.google.com/go/networksecurity v0.10.0/go.mod h1:IcpI5pyzlZyYG8cNRCJmY1AYKajsd9Uz575HoeyYoII= -cloud.google.com/go/notebooks v1.12.0/go.mod h1:euIZBbGY6G0J+UHzQ0XflysP0YoAUnDPZU7Fq0KXNw8= -cloud.google.com/go/optimization v1.7.0/go.mod h1:6KvAB1HtlsMMblT/lsQRIlLjUhKjmMWNqV1AJUctbWs= -cloud.google.com/go/orchestration v1.10.0/go.mod h1:pGiFgTTU6c/nXHTPpfsGT8N4Dax8awccCe6kjhVdWjI= -cloud.google.com/go/orgpolicy v1.13.0/go.mod h1:oKtT56zEFSsYORUunkN2mWVQBc9WGP7yBAPOZW1XCXc= -cloud.google.com/go/osconfig v1.14.0/go.mod h1:GhZzWYVrnQ42r+K5pA/hJCsnWVW2lB6bmVg+GnZ6JkM= -cloud.google.com/go/oslogin v1.14.0/go.mod h1:VtMzdQPRP3T+w5OSFiYhaT/xOm7H1wo1HZUD2NAoVK4= -cloud.google.com/go/phishingprotection v0.9.0/go.mod h1:CzttceTk9UskH9a8BycYmHL64zakEt3EXaM53r4i0Iw= -cloud.google.com/go/policytroubleshooter v1.11.0/go.mod h1:yTqY8n60lPLdU5bRbImn9IazrmF1o5b0VBshVxPzblQ= -cloud.google.com/go/privatecatalog v0.10.0/go.mod h1:/Lci3oPTxJpixjiTBoiVv3PmUZg/IdhPvKHcLEgObuc= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.42.0/go.mod h1:KADJ6s4MbTwhXmse/50SebEhE4SmUwHi48z3/dHar1Y= -cloud.google.com/go/pubsublite v1.8.2/go.mod h1:4r8GSa9NznExjuLPEJlF1VjOPOpgf3IT6k8x/YgaOPI= -cloud.google.com/go/recaptchaenterprise/v2 v2.16.0/go.mod h1:iq7s8lR3dXv4mDXE3/qyPtZEXOK7wHC1r3bX2fQyU9s= -cloud.google.com/go/recommendationengine v0.9.0/go.mod h1:59ydKXFyXO4Y8S0Bk224sKfj6YvIyzgcpG6w8kXIMm4= -cloud.google.com/go/recommender v1.13.0/go.mod h1:+XkXkeB9k6zG222ZH70U6DBkmvEL0na+pSjZRmlWcrk= -cloud.google.com/go/redis v1.17.0/go.mod h1:pzTdaIhriMLiXu8nn2CgiS52SYko0tO1Du4d3MPOG5I= -cloud.google.com/go/resourcemanager v1.10.0/go.mod h1:kIx3TWDCjLnUQUdjQ/e8EXsS9GJEzvcY+YMOHpADxrk= -cloud.google.com/go/resourcesettings v1.8.0/go.mod h1:/hleuSOq8E6mF1sRYZrSzib8BxFHprQXrPluWTuZ6Ys= -cloud.google.com/go/retail v1.18.0/go.mod h1:vaCabihbSrq88mKGKcKc4/FDHvVcPP0sQDAt0INM+v8= -cloud.google.com/go/run v1.5.0/go.mod h1:Z4Tv/XNC/veO6rEpF0waVhR7vEu5RN1uJQ8dD1PeMtI= -cloud.google.com/go/scheduler v1.11.0/go.mod h1:RBSu5/rIsF5mDbQUiruvIE6FnfKpLd3HlTDu8aWk0jw= -cloud.google.com/go/secretmanager v1.14.0/go.mod h1:q0hSFHzoW7eRgyYFH8trqEFavgrMeiJI4FETNN78vhM= -cloud.google.com/go/security v1.18.0/go.mod h1:oS/kRVUNmkwEqzCgSmK2EaGd8SbDUvliEiADjSb/8Mo= -cloud.google.com/go/securitycenter v1.35.0/go.mod h1:gotw8mBfCxX0CGrRK917CP/l+Z+QoDchJ9HDpSR8eDc= -cloud.google.com/go/servicedirectory v1.12.0/go.mod h1:lKKBoVStJa+8S+iH7h/YRBMUkkqFjfPirkOTEyYAIUk= -cloud.google.com/go/shell v1.8.0/go.mod h1:EoQR8uXuEWHUAMoB4+ijXqRVYatDCdKYOLAaay1R/yw= -cloud.google.com/go/spanner v1.67.0/go.mod h1:Um+TNmxfcCHqNCKid4rmAMvoe/Iu1vdz6UfxJ9GPxRQ= -cloud.google.com/go/speech v1.25.0/go.mod h1:2IUTYClcJhqPgee5Ko+qJqq29/bglVizgIap0c5MvYs= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -145,122 +45,53 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= -cloud.google.com/go/storagetransfer v1.11.0/go.mod h1:arcvgzVC4HPcSikqV8D4h4PwrvGQHfKtbL4OwKPirjs= -cloud.google.com/go/talent v1.7.0/go.mod h1:8zfRPWWV4GNZuUmBwQub0gWAe2KaKhsthyGtV8fV1bY= -cloud.google.com/go/texttospeech v1.8.0/go.mod h1:hAgeA01K5QNfLy2sPUAVETE0L4WdEpaCMfwKH1qjCQU= -cloud.google.com/go/tpu v1.7.0/go.mod h1:/J6Co458YHMD60nM3cCjA0msvFU/miCGMfx/nYyxv/o= -cloud.google.com/go/trace v1.11.0/go.mod h1:Aiemdi52635dBR7o3zuc9lLjXo3BwGaChEjCa3tJNmM= -cloud.google.com/go/translate v1.12.0/go.mod h1:4/C4shFIY5hSZ3b3g+xXWM5xhBLqcUqksSMrQ7tyFtc= -cloud.google.com/go/video v1.23.0/go.mod h1:EGLQv3Ce/VNqcl/+Amq7jlrnpg+KMgQcr6YOOBfE9oc= -cloud.google.com/go/videointelligence v1.12.0/go.mod h1:3rjmafNpCEqAb1CElGTA7dsg8dFDsx7RQNHS7o088D0= -cloud.google.com/go/vision/v2 v2.9.0/go.mod h1:sejxShqNOEucObbGNV5Gk85hPCgiVPP4sWv0GrgKuNw= -cloud.google.com/go/vmmigration v1.8.0/go.mod h1:+AQnGUabjpYKnkfdXJZ5nteUfzNDCmwbj/HSLGPFG5E= -cloud.google.com/go/vmwareengine v1.3.0/go.mod h1:7W/C/YFpelGyZzRUfOYkbgUfbN1CK5ME3++doIkh1Vk= -cloud.google.com/go/vpcaccess v1.8.0/go.mod h1:7fz79sxE9DbGm9dbbIdir3tsJhwCxiNAs8aFG8MEhR8= -cloud.google.com/go/webrisk v1.10.0/go.mod h1:ztRr0MCLtksoeSOQCEERZXdzwJGoH+RGYQ2qodGOy2U= -cloud.google.com/go/websecurityscanner v1.7.0/go.mod h1:d5OGdHnbky9MAZ8SGzdWIm3/c9p0r7t+5BerY5JYdZc= -cloud.google.com/go/workflows v1.13.0/go.mod h1:StCuY3jhBj1HYMjCPqZs7J0deQLHPhF6hDtzWJaVF+Y= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d h1:LblfooH1lKOpp1hIhukktmSAxFkqMPFk9KR6iZ0MJNI= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= -contrib.go.opencensus.io/exporter/zipkin v0.1.2/go.mod h1:mP5xM3rrgOjpn79MM8fZbj3gsxcuytSqtH0dxSWW1RE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= -github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/ahmetb/gen-crd-api-reference-docs v0.3.1-0.20210609063737-0067dc6dcea2/go.mod h1:TdjdkYhlOifCQWPs1UdTma97kQQMozf5h26hTuG70u8= -github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= -github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA= -github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= -github.com/aws/aws-sdk-go-v2/service/ecr v1.17.18/go.mod h1:DQtDYmexqR+z+B6HBCvY7zK/tuXKv6Zy/IwOXOK3eow= -github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.17/go.mod h1:r1Vuka0kyzqN0sZm4lYTXf0Vhl+o/mTLq6vKpBBZYaQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= -github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20221004211355-a250ad2ca1e3/go.mod h1:m06KtrZgOloUaePAQMv+Ha8kRmTnKdozTHZrweepIrw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= -github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cert-manager/cert-manager v1.13.3/go.mod h1:BM2+Pt/NmSv1Zr25/MHv6BgIEF9IUxA1xAjp80qkxgc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chrismellard/docker-credential-acr-env v0.0.0-20221002210726-e883f69e0206/go.mod h1:1UmFRnmMnVsHwD+ZntmLkoVBB1ZLa6V+XXEbF6hZCxU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= -github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= -github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v25.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155/go.mod h1:5Wkq+JduFtdAXihLmeTJf+tRYIT4KBc2vPXDhwVo1pA= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= @@ -269,7 +100,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY= github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -307,20 +139,16 @@ github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/Nu github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk= github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -351,11 +179,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -374,9 +199,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= -github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20230209165335-3624968304fd/go.mod h1:x5fIlj5elU+/eYF60q4eASMQ9kDc+GMFa7UU9M3mFFw= -github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20230209165335-3624968304fd/go.mod h1:6pjZpt+0dg+Z0kUEn53qLtD57raiZo/bqWzsuX6dDjo= -github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -412,11 +234,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -424,22 +242,16 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/influxdata/influxdb-client-go/v2 v2.9.0/go.mod h1:x7Jo5UHHl+w8wu8UnGiNobDDHygojXwJX4mx7rXGKMk= -github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY= github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -475,10 +287,6 @@ github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjS github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -488,29 +296,24 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c= +github.com/openshift/api v0.0.0-20241108213852-e22f17d9b7f5 h1:GJXiIZyRkQs4b++xfkz58m8T0ExcWf6PYJDZIeJFj7s= +github.com/openshift/api v0.0.0-20241108213852-e22f17d9b7f5/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -537,7 +340,6 @@ github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJ github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= -github.com/prometheus/exporter-toolkit v0.11.0/go.mod h1:BVnENhnNecpwoTLiABx7mrPB/OLRIgN74qlQbV+FK1Q= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -553,23 +355,18 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -587,30 +384,16 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo= -github.com/tsenart/vegeta/v12 v12.12.0/go.mod h1:gpdfR++WHV9/RZh4oux0f6lNPhsOH8pCjIGUlcPQe1M= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= -go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= -go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA= -go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= -go.etcd.io/etcd/pkg/v3 v3.5.10/go.mod h1:TKTuCKKcF1zxmfKWDkfz5qqYaE3JncKKZPFf8c1nFUs= -go.etcd.io/etcd/raft/v3 v3.5.10/go.mod h1:odD6kr8XQXTy9oQnyMPBOr0TVe+gT0neQhElQ6jbGRc= -go.etcd.io/etcd/server/v3 v3.5.10/go.mod h1:gBplPHfs6YI0L+RpGkTQO7buDbHv5HJGG/Bst0/zIPo= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -625,19 +408,15 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+n go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -789,7 +568,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= @@ -883,7 +661,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -919,7 +696,6 @@ google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed h1:4C4dbrVFtfIp3GX google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:ICjniACoWvcDz8c8bOsHVKuuSGDJy1z5M4G0DM3HzTc= google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1:3RgNmBoI9MZhsj3QxC+AP/qQhNwpCLOvYDYYsFrhFt0= google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20240823204242-4ba0660f739c/go.mod h1:gQizMG9jZ0L2ADJaM+JdZV4yTCON/CQpnHRPoM+54w4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -960,13 +736,14 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -979,7 +756,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -991,34 +767,28 @@ istio.io/api v1.23.0 h1:yqv3lNW6XSYS5XkbEkxsmFROXIQznp4lFWqj7xKEqCA= istio.io/api v1.23.0/go.mod h1:QPSTGXuIQdnZFEm3myf9NZ5uBMwCdJWUvfj9ZZ+2oBM= istio.io/client-go v1.23.0 h1://xojbifr84q29WE3eMx74p36hD4lvcejX1KxE3iJvY= istio.io/client-go v1.23.0/go.mod h1:3qX/KBS5aR47QV4JhphcZl5ysnZ53x78TBjNQLM2TC4= -k8s.io/api v0.30.4 h1:XASIELmW8w8q0i1Y4124LqPoWMycLjyQti/fdYHYjCs= -k8s.io/api v0.30.4/go.mod h1:ZqniWRKu7WIeLijbbzetF4U9qZ03cg5IRwl8YVs8mX0= -k8s.io/apiextensions-apiserver v0.30.4 h1:FwOMIk/rzZvM/Gx0IOz0+biZ+dlnlCeyfXW17uzV1qE= -k8s.io/apiextensions-apiserver v0.30.4/go.mod h1:m8cAkJ9PVU8Olb4cPW4hrUDBZGvoSJ0kY0G0CfdGQac= -k8s.io/apimachinery v0.30.4 h1:5QHQI2tInzr8LsT4kU/2+fSeibH1eIHswNx480cqIoY= -k8s.io/apimachinery v0.30.4/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/apiserver v0.30.4/go.mod h1:oyGAj9B9/0+I9huJyf4/8SMBF2mNh2bTMlu7703dkH8= -k8s.io/client-go v0.30.4 h1:eculUe+HPQoPbixfwmaSZGsKcOf7D288tH6hDAdd+wY= -k8s.io/client-go v0.30.4/go.mod h1:IBS0R/Mt0LHkNHF4E6n+SUDPG7+m2po6RZU7YHeOpzc= -k8s.io/code-generator v0.30.4 h1:1J2AcpPNBGh/NH9+m4TDh8Yj+mSbM+JyQhH0QdIMwmE= -k8s.io/code-generator v0.30.4/go.mod h1:Dd8gxOr5ieh9yHCLKnIkKDmk1H2glH8nYCAqwFogD2M= -k8s.io/component-base v0.30.4/go.mod h1:Qd3h+OJxV/LrnriXG/E15ZK83dzd306qJHW9+87S5ls= +k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= +k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= +k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= +k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= +k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/code-generator v0.31.0 h1:w607nrMi1KeDKB3/F/J4lIoOgAwc+gV9ZKew4XRfMp8= +k8s.io/code-generator v0.31.0/go.mod h1:84y4w3es8rOJOUUP1rLsIiGlO1JuEaPFXQPA9e/K6U0= k8s.io/component-helpers v0.30.4 h1:A4KYmrz12HZtGZ8TAnanl0SUx7n6tKduxzB3NHvinr0= k8s.io/component-helpers v0.30.4/go.mod h1:h5D4gI8hGQXMHw90qJq41PRUJrn2dvFA3ElZFUTzRps= -k8s.io/gengo v0.0.0-20240404160639-a0386bf69313/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 h1:cErOOTkQ3JW19o4lo91fFurouhP8NcoBvb7CkvhZZpk= k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.30.4/go.mod h1:GrMurD0qk3G4yNgGcsCEmepqf9KyyIrTXYR2lyUOJC4= k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 h1:GKE9U8BH16uynoxQii0auTjmmmuZ3O0LFMN6S0lPPhI= k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2duqHWf5OX9S3zjI= k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -knative.dev/caching v0.0.0-20240716132144-989f54c83776/go.mod h1:Uj74eO9rLiK1eb8wmDBED1hJBZQ7MJ9cvq/d8Ktsm3c= -knative.dev/hack v0.0.0-20240814130635-06f7aff93954/go.mod h1:R0ritgYtjLDO9527h5vb5X6gfvt5LCrJ55BNbVDsWiY= knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b h1:ws/Jeho6on84+5tfNKLAKriVVGIwivHbgPEtZjBfcs0= knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b/go.mod h1:2eMQVGLBZ5Kj1C4kKPuPhO7BsUeF6fkmhZFDQPIP+88= knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c h1:2crXVk4FG0dSG6WHaIT+WKbUzn7qG2wn0AfYmvA22zs= @@ -1028,10 +798,8 @@ knative.dev/serving v0.42.2/go.mod h1:3cgU8/864RcqA0ZPrc3jFcmS3uJL/mOlUZiYsXonwa rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= -sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk= -sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= -sigs.k8s.io/gateway-api v0.8.0/go.mod h1:okOnjPNBFbIS/Rw9kAhuIUaIkLhTKEu+ARIuXk2dgaM= +sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk= +sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 8818ab6aaed..696645105ef 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -38,7 +38,6 @@ var ( KnativeServingAPIGroupName = KnativeServingAPIGroupNamePrefix + ".dev" KServeNamespace = getEnvOrDefault("POD_NAMESPACE", "kserve") KServeDefaultVersion = "v0.5.0" - KserveServiceAccountName = "kserve-sa" ) // InferenceService Constants @@ -131,13 +130,14 @@ var ( // kserve networking constants const ( - NetworkVisibility = "networking.kserve.io/visibility" - ClusterLocalVisibility = "cluster-local" - ClusterLocalDomain = "svc.cluster.local" - IsvcNameHeader = "KServe-Isvc-Name" - IsvcNamespaceHeader = "KServe-Isvc-Namespace" - ODHKserveRawAuth = "security.opendatahub.io/enable-auth" - ODHRouteEnabled = "enable-route" + NetworkVisibility = "networking.kserve.io/visibility" + ClusterLocalVisibility = "cluster-local" + ClusterLocalDomain = "svc.cluster.local" + IsvcNameHeader = "KServe-Isvc-Name" + IsvcNamespaceHeader = "KServe-Isvc-Namespace" + ODHKserveRawAuth = "security.opendatahub.io/enable-auth" + ODHRouteEnabled = "exposed" + ServingCertSecretSuffix = "-serving-cert" ) // StorageSpec Constants @@ -451,7 +451,7 @@ const ( // opendatahub rawDeployment Auth const ( - OauthProxyImage = "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:4bef31eb993feb6f1096b51b4876c65a6fb1f4401fee97fa4f4542b6b7c9bc46" + OauthProxyImage = "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:234af927030921ab8f7333f61f967b4b4dee37a1b3cf85689e9e63240dd62800" OauthProxyPort = 8443 OauthProxyResourceMemoryLimit = "256Mi" OauthProxyResourceCPULimit = "100m" diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index c5e8bd5dde3..1553ff98d3d 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -1357,7 +1357,7 @@ var _ = Describe("v1beta1 inference service controller", func() { }, }, } - isvc.DefaultInferenceService(nil, nil) + isvc.DefaultInferenceService(nil, nil, &v1beta1.SecurityConfig{AutoMountServiceAccountToken: false}, nil) Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) inferenceService := &v1beta1.InferenceService{} @@ -2327,7 +2327,8 @@ var _ = Describe("v1beta1 inference service controller", func() { "serving.kserve.io/deploymentMode": "RawDeployment", }, Labels: map[string]string{ - constants.ODHKserveRawAuth: "true", + constants.ODHKserveRawAuth: "true", + constants.NetworkVisibility: constants.ODHRouteEnabled, }, }, Spec: v1beta1.InferenceServiceSpec{ @@ -2349,7 +2350,7 @@ var _ = Describe("v1beta1 inference service controller", func() { }, }, } - isvc.DefaultInferenceService(nil, nil) + isvc.DefaultInferenceService(nil, nil, &v1beta1.SecurityConfig{AutoMountServiceAccountToken: false}, nil) Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) inferenceService := &v1beta1.InferenceService{} @@ -2393,11 +2394,12 @@ var _ = Describe("v1beta1 inference service controller", func() { constants.InferenceServicePodLabelKey: serviceName, "serving.kserve.io/inferenceservice": serviceName, constants.ODHKserveRawAuth: "true", + constants.NetworkVisibility: constants.ODHRouteEnabled, }, Annotations: map[string]string{ constants.StorageInitializerSourceUriInternalAnnotationKey: *isvc.Spec.Predictor.Model.StorageURI, "serving.kserve.io/deploymentMode": "RawDeployment", - "service.beta.openshift.io/serving-cert-secret-name": serviceName, + "service.beta.openshift.io/serving-cert-secret-name": predictorDeploymentKey.Name + constants.ServingCertSecretSuffix, }, }, Spec: v1.PodSpec{ @@ -2438,7 +2440,7 @@ var _ = Describe("v1beta1 inference service controller", func() { Args: []string{ `--https-address=:8443`, `--provider=openshift`, - `--openshift-service-account=kserve-sa`, + `--skip-provider-button`, `--upstream=http://localhost:8080`, `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, @@ -2508,14 +2510,12 @@ var _ = Describe("v1beta1 inference service controller", func() { Name: "proxy-tls", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: serviceName, + SecretName: predictorDeploymentKey.Name + constants.ServingCertSecretSuffix, DefaultMode: func(i int32) *int32 { return &i }(420), }, }, }, }, - ServiceAccountName: constants.KserveServiceAccountName, - DeprecatedServiceAccount: constants.KserveServiceAccountName, SchedulerName: "default-scheduler", RestartPolicy: "Always", TerminationGracePeriodSeconds: &gracePeriod, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index eecd0bac5b0..098b61372cb 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -23,8 +23,8 @@ import ( "strconv" "strings" - "k8s.io/client-go/kubernetes" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" "github.com/google/go-cmp/cmp/cmpopts" "github.com/kserve/kserve/pkg/apis/serving/v1beta1" @@ -71,7 +71,7 @@ func NewDeploymentReconciler(client kclient.Client, componentExt: componentExt, } } -func createRawDeployment(cli kclient.Client, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, +func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) []*appsv1.Deployment { var deploymentList []*appsv1.Deployment @@ -99,7 +99,7 @@ func createRawDeployment(cli kclient.Client, componentMeta metav1.ObjectMeta, wo } } - defaultDeployment := createRawDefaultDeployment(cli, componentMeta, componentExt, podSpec) + defaultDeployment := createRawDefaultDeployment(clientset, componentMeta, componentExt, podSpec) if multiNodeEnabled { // Use defaut value(1) if tensor-parallel-size is not set (gpu count) tensorParallelSize = constants.DefaultTensorParallelSize @@ -136,16 +136,25 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me podMetadata := componentMeta podMetadata.Labels["app"] = constants.GetRawServiceLabel(componentMeta.Name) setDefaultPodSpec(podSpec) - isvcname := componentMeta.Name + var isvcname string + if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { + isvcname = val + } else { + isvcname = componentMeta.Name + } + var upstreamPort string if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { - if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { - isvcname = val - } - kserveContainerPort := GetKServeContainerPort(podSpec) - if kserveContainerPort == "" { - kserveContainerPort = constants.InferenceServiceDefaultHttpPort + if componentExt != nil && componentExt.Batcher != nil { + upstreamPort = constants.InferenceServiceDefaultAgentPortStr + } else if componentExt != nil && componentExt.Logger != nil { + upstreamPort = constants.InferenceServiceDefaultAgentPortStr + } else { + upstreamPort = GetKServeContainerPort(podSpec) + if upstreamPort == "" { + upstreamPort = constants.InferenceServiceDefaultHttpPort + } } - oauthProxyContainer, err := generateOauthProxyContainer(clientset, isvcname, componentMeta.Namespace, kserveContainerPort) + oauthProxyContainer, err := generateOauthProxyContainer(clientset, isvcname, componentMeta.Namespace, upstreamPort) if err == nil { podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) } @@ -153,7 +162,7 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me Name: tlsVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: isvcname, + SecretName: componentMeta.Name + constants.ServingCertSecretSuffix, DefaultMode: func(i int32) *int32 { return &i }(420), }, }, @@ -228,13 +237,9 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na oauthCpuRequest := constants.OauthProxyResourceCPURequest oauthMemoryRequest := constants.OauthProxyResourceMemoryRequest inferenceServiceConfigMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) - configDataSuccess := false if err == nil { var oauthData map[string]interface{} if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &oauthData); err == nil { - configDataSuccess = true - } - if configDataSuccess { if str, ok := oauthData["image"].(string); ok { oauthImage = str } @@ -259,7 +264,6 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na `--https-address=:8443`, `--provider=openshift`, `--skip-provider-button`, - `--openshift-service-account=kserve-sa`, `--upstream=http://localhost:` + upstreamPort, `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, @@ -437,8 +441,6 @@ func setDefaultDeploymentSpec(spec *appsv1.DeploymentSpec) { progressDeadlineSeconds := int32(600) spec.ProgressDeadlineSeconds = &progressDeadlineSeconds } - - spec.Template.Spec.ServiceAccountName = constants.KserveServiceAccountName } func addGPUResourceToDeployment(deployment *appsv1.Deployment, targetContainerName string, tensorParallelSize string) { @@ -519,7 +521,7 @@ func (r *DeploymentReconciler) Reconcile() ([]*appsv1.Deployment, error) { } // Patch the deployment object with the strategic merge patch - opErr = r.client.Patch(context.TODO(), deployment, client.RawPatch(types.StrategicMergePatchType, patchByte)) + opErr = r.client.Patch(context.TODO(), deployment, kclient.RawPatch(types.StrategicMergePatchType, patchByte)) } if opErr != nil { diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 55c59bcc46a..8bcdd0842bb 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -19,8 +19,6 @@ package ingress import ( "context" "fmt" - "strconv" - v1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" "github.com/kserve/kserve/pkg/utils" @@ -296,13 +294,8 @@ func getRouteURLIfExists(cli client.Client, isvc *v1beta1.InferenceService) (*ap // Construct the URL host := route.Spec.Host - isSecure := false - if isSecure, err = strconv.ParseBool(isvc.Labels[constants.ODHKserveRawAuth]); err != nil { - isSecure = false - } - scheme := "http" - if isSecure { + if route.Spec.TLS != nil && route.Spec.TLS.Termination != "" { scheme = "https" } diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go index 63a067e927a..d9d31b47791 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go @@ -175,11 +175,7 @@ func createDefaultSvc(componentMeta metav1.ObjectMeta, componentExt *v1beta1.Com if service.ObjectMeta.Annotations == nil { service.ObjectMeta.Annotations = make(map[string]string) } - isvcname := componentMeta.Name - if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { - isvcname = val - } - service.ObjectMeta.Annotations["service.beta.openshift.io/serving-cert-secret-name"] = isvcname + service.ObjectMeta.Annotations["service.beta.openshift.io/serving-cert-secret-name"] = componentMeta.Name + constants.ServingCertSecretSuffix httpsPort := corev1.ServicePort{ Name: "https", Port: constants.OauthProxyPort, From b506272711e606913cce66c0d59800ca2cd882e1 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Tue, 12 Nov 2024 09:19:00 -0500 Subject: [PATCH 09/20] update to newer oauth proxy image Signed-off-by: Vedant Mahabaleshwarkar --- config/overlays/odh/params.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/overlays/odh/params.env b/config/overlays/odh/params.env index 80c20e3f75b..b79c28f18b6 100644 --- a/config/overlays/odh/params.env +++ b/config/overlays/odh/params.env @@ -2,4 +2,4 @@ kserve-controller=quay.io/opendatahub/kserve-controller:latest kserve-agent=quay.io/opendatahub/kserve-agent:latest kserve-router=quay.io/opendatahub/kserve-router:latest kserve-storage-initializer=quay.io/opendatahub/kserve-storage-initializer:latest -oauth-proxy=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:4bef31eb993feb6f1096b51b4876c65a6fb1f4401fee97fa4f4542b6b7c9bc46 \ No newline at end of file +oauth-proxy=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:234af927030921ab8f7333f61f967b4b4dee37a1b3cf85689e9e63240dd62800 \ No newline at end of file From 57543a88ae5047f7930392dfc42e8723ef22f435 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Tue, 12 Nov 2024 11:06:16 -0500 Subject: [PATCH 10/20] minor fix Signed-off-by: Vedant Mahabaleshwarkar --- .../reconcilers/ingress/kube_ingress_reconciler.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 8bcdd0842bb..a61c9c69f45 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -269,7 +269,11 @@ func getRouteURLIfExists(cli client.Client, isvc *v1beta1.InferenceService) (*ap route := &routev1.Route{} err := cli.Get(context.TODO(), types.NamespacedName{Name: isvc.Name, Namespace: isvc.Namespace}, route) if err != nil { - return nil, err + if apierr.IsNotFound(err) { + return nil, nil + } else { + return nil, err + } } // Check if the route is owned by the InferenceService From 1b2642161303ab07c81791e83842303b139eb2c3 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Tue, 12 Nov 2024 11:36:13 -0500 Subject: [PATCH 11/20] fix unit test Signed-off-by: Vedant Mahabaleshwarkar --- .../inferenceservice/rawkube_controller_test.go | 1 + .../deployment/deployment_reconciler_test.go | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index 1553ff98d3d..997a54cd8ad 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -2532,6 +2532,7 @@ var _ = Describe("v1beta1 inference service controller", func() { FSGroupChangePolicy: nil, SeccompProfile: nil, }, + AutomountServiceAccountToken: proto.Bool(false), }, }, Strategy: appsv1.DeploymentStrategy{ diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go index 0336bafbab6..ac12c7eb308 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go @@ -16,6 +16,7 @@ limitations under the License. package deployment import ( + "k8s.io/client-go/kubernetes" "strings" "testing" @@ -30,17 +31,20 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + testclient "k8s.io/client-go/kubernetes/fake" ) func TestCreateDefaultDeployment(t *testing.T) { type args struct { + clientset kubernetes.Interface objectMeta metav1.ObjectMeta workerObjectMeta metav1.ObjectMeta componentExt *v1beta1.ComponentExtensionSpec podSpec *corev1.PodSpec workerPodSpec *corev1.PodSpec } + clientset := testclient.NewClientset() testInput := map[string]args{ "defaultDeployment": { objectMeta: metav1.ObjectMeta{ @@ -389,7 +393,7 @@ func TestCreateDefaultDeployment(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := createRawDeployment(tt.args.objectMeta, tt.args.workerObjectMeta, tt.args.componentExt, tt.args.podSpec, tt.args.workerPodSpec) + got := createRawDeployment(clientset, tt.args.objectMeta, tt.args.workerObjectMeta, tt.args.componentExt, tt.args.podSpec, tt.args.workerPodSpec) for i, deploy := range got { if diff := cmp.Diff(tt.expected[i], deploy, cmpopts.IgnoreFields(appsv1.Deployment{}, "Spec.Template.Spec.SecurityContext"), cmpopts.IgnoreFields(appsv1.Deployment{}, "Spec.Template.Spec.RestartPolicy"), @@ -457,7 +461,7 @@ func TestCreateDefaultDeployment(t *testing.T) { ttExpected := getDefaultExpectedDeployment() // update objectMeta using modify func - got := createRawDeployment(ttArgs.objectMeta, ttArgs.workerObjectMeta, ttArgs.componentExt, tt.modifyArgs(ttArgs).podSpec, tt.modifyArgs(ttArgs).workerPodSpec) + got := createRawDeployment(clientset, ttArgs.objectMeta, ttArgs.workerObjectMeta, ttArgs.componentExt, tt.modifyArgs(ttArgs).podSpec, tt.modifyArgs(ttArgs).workerPodSpec) // update expected value using modifyExpected func expected := tt.modifyExpected(ttExpected) @@ -760,7 +764,7 @@ func TestCreateDefaultDeployment(t *testing.T) { ttExpected := getDefaultExpectedDeployment() // update objectMeta using modify func - got := createRawDeployment(tt.modifyObjectMetaArgs(ttArgs).objectMeta, tt.modifyWorkerObjectMetaArgs(ttArgs).workerObjectMeta, ttArgs.componentExt, tt.modifyPodSpecArgs(ttArgs).podSpec, tt.modifyWorkerPodSpecArgs(ttArgs).workerPodSpec) + got := createRawDeployment(clientset, tt.modifyObjectMetaArgs(ttArgs).objectMeta, tt.modifyWorkerObjectMetaArgs(ttArgs).workerObjectMeta, ttArgs.componentExt, tt.modifyPodSpecArgs(ttArgs).podSpec, tt.modifyWorkerPodSpecArgs(ttArgs).workerPodSpec) // update expected value using modifyExpected func expected := tt.modifyExpected(ttExpected) From 3c21d1c5c292d9b3bebff93451d62011f5d5702a Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 13 Nov 2024 14:01:45 -0500 Subject: [PATCH 12/20] more feedback Signed-off-by: Vedant Mahabaleshwarkar --- .../odh/inferenceservice-config-patch.yaml | 6 +- pkg/constants/constants.go | 7 +- .../v1beta1/inferenceservice/controller.go | 3 + .../deployment/deployment_reconciler.go | 78 +++++++++++-------- .../ingress/kube_ingress_reconciler.go | 11 ++- .../reconcilers/raw/raw_kube_reconciler.go | 2 +- test/e2e/common/utils.py | 22 +++++- test/e2e/predictor/test_raw_deployment.py | 44 +++++++++++ 8 files changed, 122 insertions(+), 51 deletions(-) diff --git a/config/overlays/odh/inferenceservice-config-patch.yaml b/config/overlays/odh/inferenceservice-config-patch.yaml index e0c1271f970..f1e81bfe517 100644 --- a/config/overlays/odh/inferenceservice-config-patch.yaml +++ b/config/overlays/odh/inferenceservice-config-patch.yaml @@ -8,10 +8,10 @@ data: oauthProxy: |- { "image" : "$(oauth-proxy)", - "memoryRequest": "256Mi", - "memoryLimit": "256Mi", + "memoryRequest": "64Mi", + "memoryLimit": "128Mi", "cpuRequest": "100m", - "cpuLimit": "100m", + "cpuLimit": "200m", } storageInitializer: |- { diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 696645105ef..11cb76d4a2e 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -451,12 +451,7 @@ const ( // opendatahub rawDeployment Auth const ( - OauthProxyImage = "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:234af927030921ab8f7333f61f967b4b4dee37a1b3cf85689e9e63240dd62800" - OauthProxyPort = 8443 - OauthProxyResourceMemoryLimit = "256Mi" - OauthProxyResourceCPULimit = "100m" - OauthProxyResourceMemoryRequest = "256Mi" - OauthProxyResourceCPURequest = "100m" + OauthProxyPort = 8443 ) type ProtocolVersion int diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index 7b01742acb8..9b625f3f344 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -250,6 +250,9 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req reconciler := ingress.NewIngressReconciler(r.Client, r.Clientset, r.Scheme, ingressConfig) r.Log.Info("Reconciling ingress for inference service", "isvc", isvc.Name) if err := reconciler.Reconcile(isvc); err != nil { + if apierr.IsNotFound(err) { + return reconcile.Result{Requeue: true}, errors.Wrapf(err, "ISVC Route not found. Retrying") + } return reconcile.Result{}, errors.Wrapf(err, "fails to reconcile ingress") } } diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 098b61372cb..4c286771431 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -63,17 +63,21 @@ func NewDeploymentReconciler(client kclient.Client, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, - podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) *DeploymentReconciler { + podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) (*DeploymentReconciler, error) { + deploymentList, err := createRawDeployment(clientset, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec) + if err != nil { + return nil, err + } return &DeploymentReconciler{ client: client, scheme: scheme, - DeploymentList: createRawDeployment(clientset, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), + DeploymentList: deploymentList, componentExt: componentExt, - } + }, nil } func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, - podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) []*appsv1.Deployment { + podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) ([]*appsv1.Deployment, error) { var deploymentList []*appsv1.Deployment var workerNodeReplicas int32 var tensorParallelSize string @@ -99,7 +103,10 @@ func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.Ob } } - defaultDeployment := createRawDefaultDeployment(clientset, componentMeta, componentExt, podSpec) + defaultDeployment, err := createRawDefaultDeployment(clientset, componentMeta, componentExt, podSpec) + if err != nil { + return nil, err + } if multiNodeEnabled { // Use defaut value(1) if tensor-parallel-size is not set (gpu count) tensorParallelSize = constants.DefaultTensorParallelSize @@ -127,12 +134,12 @@ func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.Ob deploymentList = append(deploymentList, workerDeployment) } - return deploymentList + return deploymentList, nil } func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, - podSpec *corev1.PodSpec) *appsv1.Deployment { + podSpec *corev1.PodSpec) (*appsv1.Deployment, error) { podMetadata := componentMeta podMetadata.Labels["app"] = constants.GetRawServiceLabel(componentMeta.Name) setDefaultPodSpec(podSpec) @@ -155,9 +162,10 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me } } oauthProxyContainer, err := generateOauthProxyContainer(clientset, isvcname, componentMeta.Namespace, upstreamPort) - if err == nil { - podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) + if err != nil { + return nil, err } + podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) tlsSecretVolume := corev1.Volume{ Name: tlsVolumeName, VolumeSource: corev1.VolumeSource{ @@ -187,7 +195,7 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me deployment.Spec.Strategy = *componentExt.DeploymentStrategy } setDefaultDeploymentSpec(&deployment.Spec) - return deployment + return deployment, nil } func createRawWorkerDeployment(componentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, @@ -231,31 +239,33 @@ func GetKServeContainerPort(podSpec *corev1.PodSpec) string { } func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string) (corev1.Container, error) { - oauthImage := constants.OauthProxyImage - oauthCpuLimit := constants.OauthProxyResourceCPULimit - oauthMemoryLimit := constants.OauthProxyResourceMemoryLimit - oauthCpuRequest := constants.OauthProxyResourceCPURequest - oauthMemoryRequest := constants.OauthProxyResourceMemoryRequest + var oauthImage string + var oauthCpuLimit string + var oauthCpuRequest string + var oauthMemoryLimit string + var oauthMemoryRequest string inferenceServiceConfigMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) - if err == nil { - var oauthData map[string]interface{} - if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &oauthData); err == nil { - if str, ok := oauthData["image"].(string); ok { - oauthImage = str - } - if str, ok := oauthData["cpuLimit"].(string); ok { - oauthCpuLimit = str - } - if str, ok := oauthData["memoryLimit"].(string); ok { - oauthMemoryLimit = str - } - if str, ok := oauthData["cpuRequest"].(string); ok { - oauthCpuRequest = str - } - if str, ok := oauthData["memoryRequest"].(string); ok { - oauthMemoryRequest = str - } - } + if err != nil { + return corev1.Container{}, err + } + var oauthData map[string]interface{} + if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &oauthData); err != nil { + return corev1.Container{}, err + } + if str, ok := oauthData["image"].(string); ok { + oauthImage = str + } + if str, ok := oauthData["cpuLimit"].(string); ok { + oauthCpuLimit = str + } + if str, ok := oauthData["memoryLimit"].(string); ok { + oauthMemoryLimit = str + } + if str, ok := oauthData["cpuRequest"].(string); ok { + oauthCpuRequest = str + } + if str, ok := oauthData["memoryRequest"].(string); ok { + oauthMemoryRequest = str } return corev1.Container{ diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index a61c9c69f45..ffb4e987437 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -269,11 +269,7 @@ func getRouteURLIfExists(cli client.Client, isvc *v1beta1.InferenceService) (*ap route := &routev1.Route{} err := cli.Get(context.TODO(), types.NamespacedName{Name: isvc.Name, Namespace: isvc.Namespace}, route) if err != nil { - if apierr.IsNotFound(err) { - return nil, nil - } else { - return nil, err - } + return nil, err } // Check if the route is owned by the InferenceService @@ -356,7 +352,10 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig) if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ODHRouteEnabled { routeUrl, err := getRouteURLIfExists(r.client, isvc) - if err == nil && routeUrl != nil { + if err != nil { + return err + } + if routeUrl != nil { isvc.Status.URL = routeUrl } } diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go index e934b18e0c2..e8584b0a5ee 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go @@ -93,7 +93,7 @@ func createRawURL(clientset kubernetes.Interface, metadata metav1.ObjectMeta) (* // Reconcile ... func (r *RawKubeReconciler) Reconcile() ([]*appsv1.Deployment, error) { - //reconciling service before deployment because we want to use "service.beta.openshift.io/serving-cert-secret-name" + // reconciling service before deployment because we want to use "service.beta.openshift.io/serving-cert-secret-name" // reconcile Service _, err := r.Service.Reconcile() if err != nil { diff --git a/test/e2e/common/utils.py b/test/e2e/common/utils.py index 08fe08d812a..5cee6b34864 100644 --- a/test/e2e/common/utils.py +++ b/test/e2e/common/utils.py @@ -22,7 +22,7 @@ import portforward import requests -from kubernetes import client as k8s_client +from kubernetes import client as k8s_client, config from orjson import orjson from kserve import KServeClient, InferResponse, InferRequest @@ -30,6 +30,8 @@ from kserve.inference_client import InferenceGRPCClient, InferenceRESTClient from kserve.protocol.grpc import grpc_predict_v2_pb2 as pb from kserve.logging import trace_logger as logger +from openshift.dynamic import DynamicClient + KSERVE_NAMESPACE = "kserve" KSERVE_TEST_NAMESPACE = "kserve-ci-e2e-test" @@ -302,6 +304,24 @@ def get_isvc_endpoint(isvc): cluster_ip = get_cluster_ip() return scheme, cluster_ip, host, path +def isvc_route_ready(isvc): + # Load Kubernetes config and initialize OpenShift client + k8s_client_config = config.load_kube_config(os.environ.get("KUBECONFIG", "~/.kube/config")) + dyn_client = DynamicClient(k8s_client.ApiClient(k8s_client_config)) + + # Define the namespace and route name + namespace = isvc['metadata']['namespace'] + route_name = isvc['metadata']['name'] + + # Access the OpenShift routes API + route_resource = dyn_client.resources.get(api_version='route.openshift.io/v1', kind='Route') + + try: + # Attempt to get the route by name + route = route_resource.get(name=route_name, namespace=namespace) + return True # Route exists + except k8s_client.exceptions.ApiException as e: + return False def generate( service_name, diff --git a/test/e2e/predictor/test_raw_deployment.py b/test/e2e/predictor/test_raw_deployment.py index db9541c2ef8..9fafea762d2 100644 --- a/test/e2e/predictor/test_raw_deployment.py +++ b/test/e2e/predictor/test_raw_deployment.py @@ -35,6 +35,7 @@ from ..common.utils import KSERVE_TEST_NAMESPACE, predict_grpc from ..common.utils import predict_isvc +from ..common.utils import isvc_route_ready api_version = constants.KSERVE_V1BETA1 @@ -119,6 +120,49 @@ async def test_raw_deployment_runtime_kserve(rest_v1_client): assert res["predictions"] == [1, 1] kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) +@pytest.mark.raw +@pytest.mark.asyncio(scope="session") +async def test_raw_deployment_runtime_kserve_route_auth(rest_v1_client): + service_name = "raw-sklearn-runtime" + annotations = dict() + labels = dict() + annotations["serving.kserve.io/deploymentMode"] = "RawDeployment" + labels["networking.kserve.io/visibility"] = "exposed" + predictor = V1beta1PredictorSpec( + min_replicas=1, + model=V1beta1ModelSpec( + model_format=V1beta1ModelFormat( + name="sklearn", + ), + storage_uri="gs://kfserving-examples/models/sklearn/1.0/model", + resources=V1ResourceRequirements( + requests={"cpu": "50m", "memory": "128Mi"}, + limits={"cpu": "100m", "memory": "256Mi"}, + ), + ), + ) + + isvc = V1beta1InferenceService( + api_version=constants.KSERVE_V1BETA1, + kind=constants.KSERVE_KIND, + metadata=client.V1ObjectMeta( + name=service_name, + namespace=KSERVE_TEST_NAMESPACE, + annotations=annotations, + labels = labels + ), + spec=V1beta1InferenceServiceSpec(predictor=predictor), + ) + + kserve_client = KServeClient( + config_file=os.environ.get("KUBECONFIG", "~/.kube/config") + ) + kserve_client.create(isvc) + kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) + assert isvc_route_ready(isvc) + res = await predict_isvc(rest_v1_client, service_name, "./data/iris_input.json") + assert res["predictions"] == [1, 1] + kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) @pytest.mark.grpc @pytest.mark.raw From fbbc15f64aa8526b6510ca4634c88d86904e1cdb Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 13 Nov 2024 14:11:55 -0500 Subject: [PATCH 13/20] cookie secret Signed-off-by: Vedant Mahabaleshwarkar --- .../deployment/deployment_reconciler.go | 18 +++++++++++++++++- .../reconcilers/raw/raw_kube_reconciler.go | 6 +++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 4c286771431..e1ec4fce888 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -18,6 +18,8 @@ package deployment import ( "context" + "crypto/rand" + "encoding/base64" "encoding/json" "fmt" "strconv" @@ -268,6 +270,11 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na oauthMemoryRequest = str } + cookieSecret, err := generateCookieSecret() + if err != nil { + return corev1.Container{}, err + } + return corev1.Container{ Name: "oauth-proxy", Args: []string{ @@ -277,7 +284,7 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na `--upstream=http://localhost:` + upstreamPort, `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, - `--cookie-secret=SECRET`, + `--cookie-secret=` + cookieSecret, `--openshift-delegate-urls={"/": {"namespace": "` + namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + isvc + `", "verb": "get"}}`, `--openshift-sar={"namespace": "` + namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + isvc + `", "verb": "get"}`, `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, @@ -336,6 +343,15 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na }, nil } +func generateCookieSecret() (string, error) { + secret := make([]byte, 32) + _, err := rand.Read(secret) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(secret), nil +} + // checkDeploymentExist checks if the deployment exists? func (r *DeploymentReconciler) checkDeploymentExist(client kclient.Client, deployment *appsv1.Deployment) (constants.CheckResultType, *appsv1.Deployment, error) { // get deployment diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go index e8584b0a5ee..6af045e0c01 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go @@ -65,10 +65,14 @@ func NewRawKubeReconciler(client client.Client, if workerPodSpec != nil { multiNodeEnabled = true } + depl, err := deployment.NewDeploymentReconciler(client, clientset, scheme, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec) + if err != nil { + return nil, err + } return &RawKubeReconciler{ client: client, scheme: scheme, - Deployment: deployment.NewDeploymentReconciler(client, clientset, scheme, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec), + Deployment: depl, Service: service.NewServiceReconciler(client, scheme, componentMeta, componentExt, podSpec, multiNodeEnabled), Scaler: as, URL: url, From a5e96434060d58e18e124bc6543e65c8975a6a60 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 20 Nov 2024 20:23:37 -0500 Subject: [PATCH 14/20] test and other fixes Signed-off-by: Vedant Mahabaleshwarkar --- pkg/apis/serving/v1beta1/configmap.go | 9 + pkg/constants/constants.go | 7 +- .../rawkube_controller_test.go | 93 +++++-- .../deployment/deployment_reconciler.go | 42 ++- .../deployment/deployment_reconciler_test.go | 6 +- .../ingress/kube_ingress_reconciler.go | 19 +- .../reconcilers/raw/raw_kube_reconciler.go | 6 + .../v1beta1/inferenceservice/suite_test.go | 3 + .../v1beta1/inferenceservice/utils/utils.go | 17 ++ test/crds/route.openshift.io_routes.yaml | 256 ++++++++++++++++++ test/e2e/common/utils.py | 22 +- test/e2e/predictor/test_raw_deployment.py | 44 --- 12 files changed, 397 insertions(+), 127 deletions(-) create mode 100644 test/crds/route.openshift.io_routes.yaml diff --git a/pkg/apis/serving/v1beta1/configmap.go b/pkg/apis/serving/v1beta1/configmap.go index 27d7770bf33..24ae8aa7ea9 100644 --- a/pkg/apis/serving/v1beta1/configmap.go +++ b/pkg/apis/serving/v1beta1/configmap.go @@ -79,6 +79,15 @@ type IngressConfig struct { DisableIngressCreation bool `json:"disableIngressCreation,omitempty"` } +// +kubebuilder:object:generate=false +type OauthConfig struct { + Image string `json:"image"` + CpuLimit string `json:"cpuLimit"` + CpuRequest string `json:"cpuRequest"` + MemoryLimit string `json:"memoryLimit"` + MemoryRequest string `json:"memoryRequest"` +} + // +kubebuilder:object:generate=false type DeployConfig struct { DefaultDeploymentMode string `json:"defaultDeploymentMode,omitempty"` diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 11cb76d4a2e..70df2ac2f66 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -451,7 +451,12 @@ const ( // opendatahub rawDeployment Auth const ( - OauthProxyPort = 8443 + OauthProxyPort = 8443 + OauthProxyResourceMemoryLimit = "128Mi" + OauthProxyResourceCPULimit = "200m" + OauthProxyResourceMemoryRequest = "64Mi" + OauthProxyResourceCPURequest = "100m" + OauthProxyImage = "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:234af927030921ab8f7333f61f967b4b4dee37a1b3cf85689e9e63240dd62800" ) type ProtocolVersion int diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index 997a54cd8ad..e92f2bfd5d0 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -19,6 +19,7 @@ package inferenceservice import ( "context" "fmt" + routev1 "github.com/openshift/api/route/v1" "time" "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" @@ -40,6 +41,7 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" v1 "k8s.io/api/core/v1" + v1beta1utils "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/utils" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -2243,28 +2245,9 @@ var _ = Describe("v1beta1 inference service controller", func() { }) Context("When creating an inferenceservice with raw kube predictor and ODH auth enabled", func() { configs := map[string]string{ - "explainers": `{ - "alibi": { - "image": "kserve/alibi-explainer", - "defaultImageVersion": "latest" - } - }`, - "ingress": `{ - "ingressGateway": "knative-serving/knative-ingress-gateway", - "ingressService": "test-destination", - "localGateway": "knative-serving/knative-local-gateway", - "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local" - }`, - "storageInitializer": `{ - "image" : "kserve/storage-initializer:latest", - "memoryRequest": "100Mi", - "memoryLimit": "1Gi", - "cpuRequest": "100m", - "cpuLimit": "1", - "CaBundleConfigMapName": "", - "caBundleVolumeMountPath": "/etc/ssl/custom-certs", - "enableDirectPvcVolumeMount": false - }`, + "oauthProxy": `{"image": "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:234af927030921ab8f7333f61f967b4b4dee37a1b3cf85689e9e63240dd62800", "memoryRequest": "64Mi", "memoryLimit": "128Mi", "cpuRequest": "100m", "cpuLimit": "200m"}`, + "ingress": `{"ingressGateway": "knative-serving/knative-ingress-gateway", "ingressService": "test-destination", "localGateway": "knative-serving/knative-local-gateway", "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local"}`, + "storageInitializer": `{"image": "kserve/storage-initializer:latest", "memoryRequest": "100Mi", "memoryLimit": "1Gi", "cpuRequest": "100m", "cpuLimit": "1", "CaBundleConfigMapName": "", "caBundleVolumeMountPath": "/etc/ssl/custom-certs", "enableDirectPvcVolumeMount": false}`, } It("Should have ingress/service/deployment/hpa created", func() { @@ -2444,7 +2427,8 @@ var _ = Describe("v1beta1 inference service controller", func() { `--upstream=http://localhost:8080`, `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, - `--cookie-secret=SECRET`, + // omit cookie secret arg in unit test as it is generated randomly + //`--cookie-secret=SECRET`, `--openshift-delegate-urls={"/": {"namespace": "` + serviceKey.Namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + serviceName + `", "verb": "get"}}`, `--openshift-sar={"namespace": "` + serviceKey.Namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + serviceName + `", "verb": "get"}`, `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, @@ -2546,7 +2530,10 @@ var _ = Describe("v1beta1 inference service controller", func() { ProgressDeadlineSeconds: &progressDeadlineSeconds, }, } - Expect(actualDeployment.Spec).To(gomega.Equal(expectedDeployment.Spec)) + // remove the cookie-secret arg from the generated deployment for comparison + cleanedDep := actualDeployment.DeepCopy() + actualDep := v1beta1utils.RemoveCookieSecretArg(*cleanedDep) + Expect(actualDep.Spec).To(gomega.Equal(expectedDeployment.Spec)) //check service actualService := &v1.Service{} @@ -2583,6 +2570,56 @@ var _ = Describe("v1beta1 inference service controller", func() { actualService.Spec.InternalTrafficPolicy = nil Expect(actualService.Spec).To(gomega.Equal(expectedService.Spec)) + route := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + Labels: map[string]string{ + "inferenceservice-name": serviceName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "serving.kserve.io/v1beta1", + Kind: "InferenceService", + Name: serviceKey.Name, + UID: isvc.GetUID(), + Controller: proto.Bool(true), + BlockOwnerDeletion: proto.Bool(true), + }, + }, + }, + Spec: routev1.RouteSpec{ + Host: "raw-auth-default.example.com", + To: routev1.RouteTargetReference{ + Kind: "Service", + Name: predictorServiceKey.Name, + Weight: proto.Int32(100), + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromInt(8443), + }, + TLS: &routev1.TLSConfig{ + Termination: routev1.TLSTerminationReencrypt, + InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect, + }, + WildcardPolicy: routev1.WildcardPolicyNone, + }, + Status: routev1.RouteStatus{ + Ingress: []routev1.RouteIngress{ + { + Host: "raw-auth-default.example.com", + Conditions: []routev1.RouteIngressCondition{ + { + Type: routev1.RouteAdmitted, + Status: v1.ConditionTrue, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(context.TODO(), route)).Should(Succeed()) + //check isvc status updatedDeployment := actualDeployment.DeepCopy() updatedDeployment.Status.Conditions = []appsv1.DeploymentCondition{ @@ -2612,21 +2649,21 @@ var _ = Describe("v1beta1 inference service controller", func() { }, }, URL: &apis.URL{ - Scheme: "http", + Scheme: "https", Host: "raw-auth-default.example.com", }, Address: &duckv1.Addressable{ URL: &apis.URL{ Scheme: "http", - Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local", serviceKey.Name, serviceKey.Namespace), + Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local:8443", serviceKey.Name, serviceKey.Namespace), }, }, Components: map[v1beta1.ComponentType]v1beta1.ComponentStatusSpec{ v1beta1.PredictorComponent: { LatestCreatedRevision: "", URL: &apis.URL{ - Scheme: "http", - Host: "raw-auth-predictor-default.example.com", + Scheme: "https", + Host: "raw-auth-predictor-default.example.com:8443", }, }, }, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index e1ec4fce888..5fee38fcd6e 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -31,6 +31,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" + v1beta1utils "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/utils" "github.com/kserve/kserve/pkg/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -57,6 +58,7 @@ type DeploymentReconciler struct { const ( tlsVolumeName = "proxy-tls" + oauthProxy = "oauthProxy" ) func NewDeploymentReconciler(client kclient.Client, @@ -135,7 +137,6 @@ func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.Ob addGPUResourceToDeployment(workerDeployment, constants.WorkerContainerName, tensorParallelSize) deploymentList = append(deploymentList, workerDeployment) } - return deploymentList, nil } @@ -146,12 +147,12 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me podMetadata.Labels["app"] = constants.GetRawServiceLabel(componentMeta.Name) setDefaultPodSpec(podSpec) var isvcname string + var upstreamPort string if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { isvcname = val } else { isvcname = componentMeta.Name } - var upstreamPort string if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { if componentExt != nil && componentExt.Batcher != nil { upstreamPort = constants.InferenceServiceDefaultAgentPortStr @@ -241,34 +242,21 @@ func GetKServeContainerPort(podSpec *corev1.PodSpec) string { } func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string) (corev1.Container, error) { - var oauthImage string - var oauthCpuLimit string - var oauthCpuRequest string - var oauthMemoryLimit string - var oauthMemoryRequest string - inferenceServiceConfigMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) + + configMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) if err != nil { return corev1.Container{}, err } - var oauthData map[string]interface{} - if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &oauthData); err != nil { + oauthProxyJSON := strings.TrimSpace(configMap.Data["oauthProxy"]) + oauthProxyConfig := v1beta1.OauthConfig{} + if err := json.Unmarshal([]byte(oauthProxyJSON), &oauthProxyConfig); err != nil { return corev1.Container{}, err } - if str, ok := oauthData["image"].(string); ok { - oauthImage = str - } - if str, ok := oauthData["cpuLimit"].(string); ok { - oauthCpuLimit = str - } - if str, ok := oauthData["memoryLimit"].(string); ok { - oauthMemoryLimit = str - } - if str, ok := oauthData["cpuRequest"].(string); ok { - oauthCpuRequest = str - } - if str, ok := oauthData["memoryRequest"].(string); ok { - oauthMemoryRequest = str - } + oauthImage := oauthProxyConfig.Image + oauthMemoryRequest := oauthProxyConfig.MemoryRequest + oauthMemoryLimit := oauthProxyConfig.MemoryLimit + oauthCpuRequest := oauthProxyConfig.CpuRequest + oauthCpuLimit := oauthProxyConfig.CpuLimit cookieSecret, err := generateCookieSecret() if err != nil { @@ -375,7 +363,9 @@ func (r *DeploymentReconciler) checkDeploymentExist(client kclient.Client, deplo log.Error(err, "Failed to perform dry-run update of deployment", "Deployment", deployment.Name) return constants.CheckResultUnknown, nil, err } - if diff, err := kmp.SafeDiff(deployment.Spec, existingDeployment.Spec, ignoreFields); err != nil { + processedExistingDep := v1beta1utils.RemoveCookieSecretArg(*existingDeployment) + processedNewDep := v1beta1utils.RemoveCookieSecretArg(*deployment) + if diff, err := kmp.SafeDiff(processedNewDep.Spec, processedExistingDep.Spec, ignoreFields); err != nil { return constants.CheckResultUnknown, nil, err } else if diff != "" { log.Info("Deployment Updated", "Diff", diff) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go index ac12c7eb308..49149bf454e 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go @@ -393,7 +393,7 @@ func TestCreateDefaultDeployment(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := createRawDeployment(clientset, tt.args.objectMeta, tt.args.workerObjectMeta, tt.args.componentExt, tt.args.podSpec, tt.args.workerPodSpec) + got, _ := createRawDeployment(clientset, tt.args.objectMeta, tt.args.workerObjectMeta, tt.args.componentExt, tt.args.podSpec, tt.args.workerPodSpec) for i, deploy := range got { if diff := cmp.Diff(tt.expected[i], deploy, cmpopts.IgnoreFields(appsv1.Deployment{}, "Spec.Template.Spec.SecurityContext"), cmpopts.IgnoreFields(appsv1.Deployment{}, "Spec.Template.Spec.RestartPolicy"), @@ -461,7 +461,7 @@ func TestCreateDefaultDeployment(t *testing.T) { ttExpected := getDefaultExpectedDeployment() // update objectMeta using modify func - got := createRawDeployment(clientset, ttArgs.objectMeta, ttArgs.workerObjectMeta, ttArgs.componentExt, tt.modifyArgs(ttArgs).podSpec, tt.modifyArgs(ttArgs).workerPodSpec) + got, _ := createRawDeployment(clientset, ttArgs.objectMeta, ttArgs.workerObjectMeta, ttArgs.componentExt, tt.modifyArgs(ttArgs).podSpec, tt.modifyArgs(ttArgs).workerPodSpec) // update expected value using modifyExpected func expected := tt.modifyExpected(ttExpected) @@ -764,7 +764,7 @@ func TestCreateDefaultDeployment(t *testing.T) { ttExpected := getDefaultExpectedDeployment() // update objectMeta using modify func - got := createRawDeployment(clientset, tt.modifyObjectMetaArgs(ttArgs).objectMeta, tt.modifyWorkerObjectMetaArgs(ttArgs).workerObjectMeta, ttArgs.componentExt, tt.modifyPodSpecArgs(ttArgs).podSpec, tt.modifyWorkerPodSpecArgs(ttArgs).workerPodSpec) + got, _ := createRawDeployment(clientset, tt.modifyObjectMetaArgs(ttArgs).objectMeta, tt.modifyWorkerObjectMetaArgs(ttArgs).workerObjectMeta, ttArgs.componentExt, tt.modifyPodSpecArgs(ttArgs).podSpec, tt.modifyWorkerPodSpecArgs(ttArgs).workerPodSpec) // update expected value using modifyExpected func expected := tt.modifyExpected(ttExpected) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index ffb4e987437..5b686cfee62 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -36,6 +36,7 @@ import ( "knative.dev/pkg/network" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "strconv" ) // RawIngressReconciler reconciles the kubernetes ingress @@ -56,7 +57,7 @@ func NewRawIngressReconciler(client client.Client, } func createRawURL(isvc *v1beta1.InferenceService, - ingressConfig *v1beta1.IngressConfig) (*knapis.URL, error) { + ingressConfig *v1beta1.IngressConfig, authEnabled bool) (*knapis.URL, error) { var err error url := &knapis.URL{} url.Scheme = ingressConfig.UrlScheme @@ -64,7 +65,9 @@ func createRawURL(isvc *v1beta1.InferenceService, if err != nil { return nil, err } - + if authEnabled { + url.Host += ":" + strconv.Itoa(constants.OauthProxyPort) + } return url, nil } @@ -349,7 +352,11 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { return err } } - isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig) + authEnabled := false + if val, ok := isvc.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + authEnabled = true + } + isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig, authEnabled) if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ODHRouteEnabled { routeUrl, err := getRouteURLIfExists(r.client, isvc) if err != nil { @@ -362,9 +369,13 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { if err != nil { return err } + internalHost := getRawServiceHost(isvc, r.client) + if authEnabled { + internalHost += ":8443" + } isvc.Status.Address = &duckv1.Addressable{ URL: &apis.URL{ - Host: getRawServiceHost(isvc, r.client), + Host: internalHost, Scheme: r.ingressConfig.UrlScheme, Path: "", }, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go index 6af045e0c01..fea20615d26 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go @@ -18,6 +18,7 @@ package raw import ( "fmt" + "github.com/kserve/kserve/pkg/constants" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -92,6 +93,11 @@ func createRawURL(clientset kubernetes.Interface, metadata metav1.ObjectMeta) (* return nil, fmt.Errorf("failed creating host name: %w", err) } + if val, ok := metadata.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + url.Scheme = "https" + url.Host += ":8443" + } + return url, nil } diff --git a/pkg/controller/v1beta1/inferenceservice/suite_test.go b/pkg/controller/v1beta1/inferenceservice/suite_test.go index 19fe37294e3..058ce3f7d54 100644 --- a/pkg/controller/v1beta1/inferenceservice/suite_test.go +++ b/pkg/controller/v1beta1/inferenceservice/suite_test.go @@ -41,6 +41,7 @@ import ( "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" pkgtest "github.com/kserve/kserve/pkg/testing" + routev1 "github.com/openshift/api/route/v1" ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -81,6 +82,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = netv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = routev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils.go b/pkg/controller/v1beta1/inferenceservice/utils/utils.go index 7c697009159..831b86ae5ce 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "html/template" + appsv1 "k8s.io/api/apps/v1" "regexp" "sort" "strings" @@ -441,3 +442,19 @@ func MergeServingRuntimeAndInferenceServiceSpecs(srContainers []v1.Container, is } return containerIndexInSR, mergedContainer, mergedPodSpec, nil } + +func RemoveCookieSecretArg(deployment appsv1.Deployment) *appsv1.Deployment { + dep := deployment.DeepCopy() + for i, container := range dep.Spec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + var newArgs []string + for _, arg := range container.Args { + if !strings.Contains(arg, "cookie-secret") { + newArgs = append(newArgs, arg) + } + } + dep.Spec.Template.Spec.Containers[i].Args = newArgs + } + } + return dep +} diff --git a/test/crds/route.openshift.io_routes.yaml b/test/crds/route.openshift.io_routes.yaml new file mode 100644 index 00000000000..314ff2fc410 --- /dev/null +++ b/test/crds/route.openshift.io_routes.yaml @@ -0,0 +1,256 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: routes.route.openshift.io +spec: + group: route.openshift.io + names: + kind: Route + listKind: RouteList + plural: routes + singular: route + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: "A route allows developers to expose services through an HTTP(S) + aware load balancing and proxy layer via a public DNS entry. The route may + further specify TLS options and a certificate, or specify a public CNAME + that the router should also accept for HTTP and HTTPS traffic. An administrator + typically configures their router to be visible outside the cluster firewall, + and may also add additional security, caching, or traffic controls on the + service content. Routers usually talk directly to the service endpoints. + \n Once a route is created, the `host` field may not be changed. Generally, + routers use the oldest route with a given host when resolving conflicts. + \n Routers are subject to additional customization and may support additional + controls via the annotations field. \n Because administrators may configure + multiple routers, the route status field is used to return information to + clients about the names and states of the route under each router. If a + client chooses a duplicate name, for instance, the route status conditions + are used to indicate the route cannot be chosen." + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: spec is the desired state of the route + properties: + alternateBackends: + description: alternateBackends allows up to 3 additional backends + to be assigned to the route. Only the Service kind is allowed, and + it will be defaulted to Service. Use the weight field in RouteTargetReference + object to specify relative preference. + items: + description: RouteTargetReference specifies the target that resolve + into endpoints. Only the 'Service' kind is allowed. Use 'weight' + field to emphasize one over others. + properties: + kind: + description: The kind of target that the route is referring + to. Currently, only 'Service' is allowed + type: string + name: + description: name of the service/target that is being referred + to. e.g. name of the service + type: string + weight: + description: weight as an integer between 0 and 256, default + 1, that specifies the target's relative weight against other + target reference objects. 0 suppresses requests to this backend. + format: int32 + type: integer + required: + - kind + - name + - weight + type: object + type: array + host: + description: host is an alias/DNS that points to the service. Optional. + If not specified a route name will typically be automatically chosen. + Must follow DNS952 subdomain conventions. + type: string + path: + description: Path that the router watches for, to route traffic for + to the service. Optional + type: string + port: + description: If specified, the port to be used by the router. Most + routers will use all endpoints exposed by the service by default + - set this value to instruct routers which port to use. + properties: + targetPort: + anyOf: + - type: integer + - type: string + description: The target port on pods selected by the service this + route points to. If this is a string, it will be looked up as + a named port in the target endpoints port list. Required + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + tls: + description: The tls field provides the ability to configure certificates + and termination for the route. + properties: + caCertificate: + description: caCertificate provides the cert authority certificate + contents + type: string + certificate: + description: certificate provides certificate contents + type: string + destinationCACertificate: + description: destinationCACertificate provides the contents of + the ca certificate of the final destination. When using reencrypt + termination this file should be provided in order to have routers + use it for health checks on the secure connection. If this field + is not specified, the router may provide its own destination + CA and perform hostname validation using the short service name + (service.namespace.svc), which allows infrastructure generated + certificates to automatically verify. + type: string + insecureEdgeTerminationPolicy: + description: "insecureEdgeTerminationPolicy indicates the desired + behavior for insecure connections to a route. While each router + may make its own decisions on which ports to expose, this is + normally port 80. \n * Allow - traffic is sent to the server + on the insecure port (default) * Disable - no traffic is allowed + on the insecure port. * Redirect - clients are redirected to + the secure port." + type: string + key: + description: key provides key file contents + type: string + termination: + description: termination indicates termination type. + type: string + required: + - termination + type: object + to: + description: to is an object the route should use as the primary backend. + Only the Service kind is allowed, and it will be defaulted to Service. + If the weight field (0-256 default 1) is set to zero, no traffic + will be sent to this backend. + properties: + kind: + description: The kind of target that the route is referring to. + Currently, only 'Service' is allowed + type: string + name: + description: name of the service/target that is being referred + to. e.g. name of the service + type: string + weight: + description: weight as an integer between 0 and 256, default 1, + that specifies the target's relative weight against other target + reference objects. 0 suppresses requests to this backend. + format: int32 + type: integer + required: + - kind + - name + - weight + type: object + wildcardPolicy: + description: Wildcard policy if any for the route. Currently only + 'Subdomain' or 'None' is allowed. + type: string + required: + - host + - to + type: object + status: + description: status is the current state of the route + properties: + ingress: + description: ingress describes the places where the route may be exposed. + The list of ingress points may contain duplicate Host or RouterName + values. Routes are considered live once they are `Ready` + items: + description: RouteIngress holds information about the places where + a route is exposed. + properties: + conditions: + description: Conditions is the state of the route, may be empty. + items: + description: RouteIngressCondition contains details for the + current condition of this route on a particular router. + properties: + lastTransitionTime: + description: RFC 3339 date and time when this condition + last transitioned + format: date-time + type: string + message: + description: Human readable message indicating details + about last transition. + type: string + reason: + description: (brief) reason for the condition's last transition, + and is usually a machine and human readable constant + type: string + status: + description: Status is the status of the condition. Can + be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. Currently + only Ready. + type: string + required: + - status + - type + type: object + type: array + host: + description: Host is the host string under which the route is + exposed; this value is required + type: string + routerCanonicalHostname: + description: CanonicalHostname is the external host name for + the router that can be used as a CNAME for the host requested + for this route. This value is optional and may not be set + in all cases. + type: string + routerName: + description: Name is a name chosen by the router to identify + itself; this value is required + type: string + wildcardPolicy: + description: Wildcard policy is the wildcard policy that was + allowed where this route is exposed. + type: string + type: object + type: array + required: + - ingress + type: object + required: + - spec + - status + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/test/e2e/common/utils.py b/test/e2e/common/utils.py index 5cee6b34864..08fe08d812a 100644 --- a/test/e2e/common/utils.py +++ b/test/e2e/common/utils.py @@ -22,7 +22,7 @@ import portforward import requests -from kubernetes import client as k8s_client, config +from kubernetes import client as k8s_client from orjson import orjson from kserve import KServeClient, InferResponse, InferRequest @@ -30,8 +30,6 @@ from kserve.inference_client import InferenceGRPCClient, InferenceRESTClient from kserve.protocol.grpc import grpc_predict_v2_pb2 as pb from kserve.logging import trace_logger as logger -from openshift.dynamic import DynamicClient - KSERVE_NAMESPACE = "kserve" KSERVE_TEST_NAMESPACE = "kserve-ci-e2e-test" @@ -304,24 +302,6 @@ def get_isvc_endpoint(isvc): cluster_ip = get_cluster_ip() return scheme, cluster_ip, host, path -def isvc_route_ready(isvc): - # Load Kubernetes config and initialize OpenShift client - k8s_client_config = config.load_kube_config(os.environ.get("KUBECONFIG", "~/.kube/config")) - dyn_client = DynamicClient(k8s_client.ApiClient(k8s_client_config)) - - # Define the namespace and route name - namespace = isvc['metadata']['namespace'] - route_name = isvc['metadata']['name'] - - # Access the OpenShift routes API - route_resource = dyn_client.resources.get(api_version='route.openshift.io/v1', kind='Route') - - try: - # Attempt to get the route by name - route = route_resource.get(name=route_name, namespace=namespace) - return True # Route exists - except k8s_client.exceptions.ApiException as e: - return False def generate( service_name, diff --git a/test/e2e/predictor/test_raw_deployment.py b/test/e2e/predictor/test_raw_deployment.py index 9fafea762d2..db9541c2ef8 100644 --- a/test/e2e/predictor/test_raw_deployment.py +++ b/test/e2e/predictor/test_raw_deployment.py @@ -35,7 +35,6 @@ from ..common.utils import KSERVE_TEST_NAMESPACE, predict_grpc from ..common.utils import predict_isvc -from ..common.utils import isvc_route_ready api_version = constants.KSERVE_V1BETA1 @@ -120,49 +119,6 @@ async def test_raw_deployment_runtime_kserve(rest_v1_client): assert res["predictions"] == [1, 1] kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) -@pytest.mark.raw -@pytest.mark.asyncio(scope="session") -async def test_raw_deployment_runtime_kserve_route_auth(rest_v1_client): - service_name = "raw-sklearn-runtime" - annotations = dict() - labels = dict() - annotations["serving.kserve.io/deploymentMode"] = "RawDeployment" - labels["networking.kserve.io/visibility"] = "exposed" - predictor = V1beta1PredictorSpec( - min_replicas=1, - model=V1beta1ModelSpec( - model_format=V1beta1ModelFormat( - name="sklearn", - ), - storage_uri="gs://kfserving-examples/models/sklearn/1.0/model", - resources=V1ResourceRequirements( - requests={"cpu": "50m", "memory": "128Mi"}, - limits={"cpu": "100m", "memory": "256Mi"}, - ), - ), - ) - - isvc = V1beta1InferenceService( - api_version=constants.KSERVE_V1BETA1, - kind=constants.KSERVE_KIND, - metadata=client.V1ObjectMeta( - name=service_name, - namespace=KSERVE_TEST_NAMESPACE, - annotations=annotations, - labels = labels - ), - spec=V1beta1InferenceServiceSpec(predictor=predictor), - ) - - kserve_client = KServeClient( - config_file=os.environ.get("KUBECONFIG", "~/.kube/config") - ) - kserve_client.create(isvc) - kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) - assert isvc_route_ready(isvc) - res = await predict_isvc(rest_v1_client, service_name, "./data/iris_input.json") - assert res["predictions"] == [1, 1] - kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) @pytest.mark.grpc @pytest.mark.raw From efbfed139eb28579833b6b2f3b210eba2e9c4ed0 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Thu, 21 Nov 2024 08:57:03 -0500 Subject: [PATCH 15/20] fix lint issues Signed-off-by: Vedant Mahabaleshwarkar --- .../v1beta1/inferenceservice/rawkube_controller_test.go | 3 ++- .../reconcilers/deployment/deployment_reconciler.go | 3 +-- .../reconcilers/ingress/kube_ingress_reconciler.go | 3 ++- .../inferenceservice/reconcilers/raw/raw_kube_reconciler.go | 1 + pkg/controller/v1beta1/inferenceservice/utils/utils.go | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index e92f2bfd5d0..47a97d15dc0 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -19,7 +19,7 @@ package inferenceservice import ( "context" "fmt" - routev1 "github.com/openshift/api/route/v1" + "time" "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" @@ -42,6 +42,7 @@ import ( v1 "k8s.io/api/core/v1" v1beta1utils "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/utils" + routev1 "github.com/openshift/api/route/v1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 5fee38fcd6e..88be25092f9 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -154,6 +154,7 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me isvcname = componentMeta.Name } if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + //nolint:gocritic if componentExt != nil && componentExt.Batcher != nil { upstreamPort = constants.InferenceServiceDefaultAgentPortStr } else if componentExt != nil && componentExt.Logger != nil { @@ -240,9 +241,7 @@ func GetKServeContainerPort(podSpec *corev1.PodSpec) string { } return "" } - func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string) (corev1.Container, error) { - configMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) if err != nil { return corev1.Container{}, err diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 5b686cfee62..096911dcbc3 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -19,6 +19,8 @@ package ingress import ( "context" "fmt" + "strconv" + v1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" "github.com/kserve/kserve/pkg/utils" @@ -36,7 +38,6 @@ import ( "knative.dev/pkg/network" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "strconv" ) // RawIngressReconciler reconciles the kubernetes ingress diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go index fea20615d26..4d96688659c 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go @@ -18,6 +18,7 @@ package raw import ( "fmt" + "github.com/kserve/kserve/pkg/constants" appsv1 "k8s.io/api/apps/v1" diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils.go b/pkg/controller/v1beta1/inferenceservice/utils/utils.go index 831b86ae5ce..dd04f672518 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils.go @@ -22,11 +22,12 @@ import ( "encoding/json" "fmt" "html/template" - appsv1 "k8s.io/api/apps/v1" "regexp" "sort" "strings" + appsv1 "k8s.io/api/apps/v1" + "github.com/pkg/errors" goerrors "github.com/pkg/errors" v1 "k8s.io/api/core/v1" From 4160784556e7fecebde187a6cd9917e4d493673a Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 27 Nov 2024 09:33:33 -0500 Subject: [PATCH 16/20] address latest feedback Signed-off-by: Vedant Mahabaleshwarkar --- .../v1beta1/inferenceservice/controller.go | 3 --- .../inferenceservice/rawkube_controller_test.go | 4 ++-- .../reconcilers/deployment/deployment_reconciler.go | 12 ++++++++---- .../reconcilers/raw/raw_kube_reconciler.go | 8 -------- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index 9b625f3f344..7b01742acb8 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -250,9 +250,6 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req reconciler := ingress.NewIngressReconciler(r.Client, r.Clientset, r.Scheme, ingressConfig) r.Log.Info("Reconciling ingress for inference service", "isvc", isvc.Name) if err := reconciler.Reconcile(isvc); err != nil { - if apierr.IsNotFound(err) { - return reconcile.Result{Requeue: true}, errors.Wrapf(err, "ISVC Route not found. Retrying") - } return reconcile.Result{}, errors.Wrapf(err, "fails to reconcile ingress") } } diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index 47a97d15dc0..6baa2035fd7 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -2663,8 +2663,8 @@ var _ = Describe("v1beta1 inference service controller", func() { v1beta1.PredictorComponent: { LatestCreatedRevision: "", URL: &apis.URL{ - Scheme: "https", - Host: "raw-auth-predictor-default.example.com:8443", + Scheme: "http", + Host: "raw-auth-predictor-default.example.com", }, }, }, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 88be25092f9..1a3fd91f55f 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -154,12 +154,12 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me isvcname = componentMeta.Name } if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { - //nolint:gocritic - if componentExt != nil && componentExt.Batcher != nil { + switch { + case componentExt != nil && componentExt.Batcher != nil: upstreamPort = constants.InferenceServiceDefaultAgentPortStr - } else if componentExt != nil && componentExt.Logger != nil { + case componentExt != nil && componentExt.Logger != nil: upstreamPort = constants.InferenceServiceDefaultAgentPortStr - } else { + default: upstreamPort = GetKServeContainerPort(podSpec) if upstreamPort == "" { upstreamPort = constants.InferenceServiceDefaultHttpPort @@ -251,6 +251,10 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na if err := json.Unmarshal([]byte(oauthProxyJSON), &oauthProxyConfig); err != nil { return corev1.Container{}, err } + if oauthProxyConfig.Image == "" || oauthProxyConfig.MemoryRequest == "" || oauthProxyConfig.MemoryLimit == "" || + oauthProxyConfig.CpuRequest == "" || oauthProxyConfig.CpuLimit == "" { + return corev1.Container{}, fmt.Errorf("one or more oauthProxyConfig fields are empty") + } oauthImage := oauthProxyConfig.Image oauthMemoryRequest := oauthProxyConfig.MemoryRequest oauthMemoryLimit := oauthProxyConfig.MemoryLimit diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go index 4d96688659c..64554f1a24f 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/raw/raw_kube_reconciler.go @@ -19,8 +19,6 @@ package raw import ( "fmt" - "github.com/kserve/kserve/pkg/constants" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,12 +91,6 @@ func createRawURL(clientset kubernetes.Interface, metadata metav1.ObjectMeta) (* if err != nil { return nil, fmt.Errorf("failed creating host name: %w", err) } - - if val, ok := metadata.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { - url.Scheme = "https" - url.Host += ":8443" - } - return url, nil } From a2d65626efdeb214757e7548197ff766ff4ac051 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Wed, 27 Nov 2024 09:37:02 -0500 Subject: [PATCH 17/20] missed import sort Signed-off-by: Vedant Mahabaleshwarkar --- .../reconcilers/deployment/deployment_reconciler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go index 49149bf454e..4bbce411c4c 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go @@ -16,7 +16,6 @@ limitations under the License. package deployment import ( - "k8s.io/client-go/kubernetes" "strings" "testing" @@ -31,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" testclient "k8s.io/client-go/kubernetes/fake" ) From 8a3c76bb8c400e70d698c20f327a83753c67619b Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Mon, 2 Dec 2024 14:58:57 -0500 Subject: [PATCH 18/20] address more feedback Signed-off-by: Vedant Mahabaleshwarkar --- .../rawkube_controller_test.go | 3 +- .../deployment/deployment_reconciler.go | 72 ++++++++++--------- .../deployment/deployment_reconciler_test.go | 10 +++ .../ingress/kube_ingress_reconciler.go | 3 +- .../reconcilers/service/service_reconciler.go | 2 +- .../v1beta1/inferenceservice/utils/utils.go | 2 + 6 files changed, 56 insertions(+), 36 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index 6baa2035fd7..0b1e9c23ea5 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -1375,7 +1375,7 @@ var _ = Describe("v1beta1 inference service controller", func() { actualIngress := &netv1.Ingress{} predictorIngressKey := types.NamespacedName{Name: serviceKey.Name, Namespace: serviceKey.Namespace} - Eventually(func() error { return k8sClient.Get(context.TODO(), predictorIngressKey, actualIngress) }, timeout). + Consistently(func() error { return k8sClient.Get(context.TODO(), predictorIngressKey, actualIngress) }, timeout). ShouldNot(Succeed()) }) }) @@ -2432,7 +2432,6 @@ var _ = Describe("v1beta1 inference service controller", func() { //`--cookie-secret=SECRET`, `--openshift-delegate-urls={"/": {"namespace": "` + serviceKey.Namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + serviceName + `", "verb": "get"}}`, `--openshift-sar={"namespace": "` + serviceKey.Namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + serviceName + `", "verb": "get"}`, - `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, }, Ports: []v1.ContainerPort{ { diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 1a3fd91f55f..3afbb01b82b 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -109,7 +109,10 @@ func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.Ob defaultDeployment, err := createRawDefaultDeployment(clientset, componentMeta, componentExt, podSpec) if err != nil { - return nil, err + // the calling function will ignore the deploymentlist if an error is also returned + // This is required for the deployment_reconciler_tests + deploymentList = append(deploymentList, defaultDeployment) + return deploymentList, err } if multiNodeEnabled { // Use defaut value(1) if tensor-parallel-size is not set (gpu count) @@ -146,6 +149,26 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me podMetadata := componentMeta podMetadata.Labels["app"] = constants.GetRawServiceLabel(componentMeta.Name) setDefaultPodSpec(podSpec) + + deployment := &appsv1.Deployment{ + ObjectMeta: componentMeta, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": constants.GetRawServiceLabel(componentMeta.Name), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: podMetadata, + Spec: *podSpec, + }, + }, + } + if componentExt.DeploymentStrategy != nil { + deployment.Spec.Strategy = *componentExt.DeploymentStrategy + } + setDefaultDeploymentSpec(&deployment.Spec) + var isvcname string var upstreamPort string if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { @@ -167,9 +190,12 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me } oauthProxyContainer, err := generateOauthProxyContainer(clientset, isvcname, componentMeta.Namespace, upstreamPort) if err != nil { - return nil, err + // return the deployment without the oauth proxy container if there was an error + // This is required for the deployment_reconciler_tests + return deployment, err } - podSpec.Containers = append(podSpec.Containers, oauthProxyContainer) + updatedPodSpec := podSpec.DeepCopy() + updatedPodSpec.Containers = append(updatedPodSpec.Containers, *oauthProxyContainer) tlsSecretVolume := corev1.Volume{ Name: tlsVolumeName, VolumeSource: corev1.VolumeSource{ @@ -179,26 +205,9 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me }, }, } - podSpec.Volumes = append(podSpec.Volumes, tlsSecretVolume) + updatedPodSpec.Volumes = append(updatedPodSpec.Volumes, tlsSecretVolume) + deployment.Spec.Template.Spec = *updatedPodSpec } - deployment := &appsv1.Deployment{ - ObjectMeta: componentMeta, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": constants.GetRawServiceLabel(componentMeta.Name), - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: podMetadata, - Spec: *podSpec, - }, - }, - } - if componentExt.DeploymentStrategy != nil { - deployment.Spec.Strategy = *componentExt.DeploymentStrategy - } - setDefaultDeploymentSpec(&deployment.Spec) return deployment, nil } func createRawWorkerDeployment(componentMeta metav1.ObjectMeta, @@ -241,19 +250,19 @@ func GetKServeContainerPort(podSpec *corev1.PodSpec) string { } return "" } -func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string) (corev1.Container, error) { - configMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) +func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string) (*corev1.Container, error) { + isvcConfigMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) if err != nil { - return corev1.Container{}, err + return nil, err } - oauthProxyJSON := strings.TrimSpace(configMap.Data["oauthProxy"]) + oauthProxyJSON := strings.TrimSpace(isvcConfigMap.Data["oauthProxy"]) oauthProxyConfig := v1beta1.OauthConfig{} if err := json.Unmarshal([]byte(oauthProxyJSON), &oauthProxyConfig); err != nil { - return corev1.Container{}, err + return nil, err } if oauthProxyConfig.Image == "" || oauthProxyConfig.MemoryRequest == "" || oauthProxyConfig.MemoryLimit == "" || oauthProxyConfig.CpuRequest == "" || oauthProxyConfig.CpuLimit == "" { - return corev1.Container{}, fmt.Errorf("one or more oauthProxyConfig fields are empty") + return nil, fmt.Errorf("one or more oauthProxyConfig fields are empty") } oauthImage := oauthProxyConfig.Image oauthMemoryRequest := oauthProxyConfig.MemoryRequest @@ -263,13 +272,13 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na cookieSecret, err := generateCookieSecret() if err != nil { - return corev1.Container{}, err + return nil, err } - return corev1.Container{ + return &corev1.Container{ Name: "oauth-proxy", Args: []string{ - `--https-address=:8443`, + `--https-address=:` + strconv.Itoa(constants.OauthProxyPort), `--provider=openshift`, `--skip-provider-button`, `--upstream=http://localhost:` + upstreamPort, @@ -278,7 +287,6 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na `--cookie-secret=` + cookieSecret, `--openshift-delegate-urls={"/": {"namespace": "` + namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + isvc + `", "verb": "get"}}`, `--openshift-sar={"namespace": "` + namespace + `", "resource": "inferenceservices", "group": "serving.kserve.io", "name": "` + isvc + `", "verb": "get"}`, - `--skip-auth-regex="(^/metrics|^/apis/v1beta1/healthz)"`, }, Image: oauthImage, Ports: []corev1.ContainerPort{ diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go index 4bbce411c4c..0cb90d49875 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go @@ -394,6 +394,10 @@ func TestCreateDefaultDeployment(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, _ := createRawDeployment(clientset, tt.args.objectMeta, tt.args.workerObjectMeta, tt.args.componentExt, tt.args.podSpec, tt.args.workerPodSpec) + if len(got) == 0 { + t.Errorf("Got empty deployment") + } + for i, deploy := range got { if diff := cmp.Diff(tt.expected[i], deploy, cmpopts.IgnoreFields(appsv1.Deployment{}, "Spec.Template.Spec.SecurityContext"), cmpopts.IgnoreFields(appsv1.Deployment{}, "Spec.Template.Spec.RestartPolicy"), @@ -462,6 +466,9 @@ func TestCreateDefaultDeployment(t *testing.T) { // update objectMeta using modify func got, _ := createRawDeployment(clientset, ttArgs.objectMeta, ttArgs.workerObjectMeta, ttArgs.componentExt, tt.modifyArgs(ttArgs).podSpec, tt.modifyArgs(ttArgs).workerPodSpec) + if len(got) == 0 { + t.Errorf("Got empty deployment") + } // update expected value using modifyExpected func expected := tt.modifyExpected(ttExpected) @@ -765,6 +772,9 @@ func TestCreateDefaultDeployment(t *testing.T) { // update objectMeta using modify func got, _ := createRawDeployment(clientset, tt.modifyObjectMetaArgs(ttArgs).objectMeta, tt.modifyWorkerObjectMetaArgs(ttArgs).workerObjectMeta, ttArgs.componentExt, tt.modifyPodSpecArgs(ttArgs).podSpec, tt.modifyWorkerPodSpecArgs(ttArgs).workerPodSpec) + if len(got) == 0 { + t.Errorf("Got empty deployment") + } // update expected value using modifyExpected func expected := tt.modifyExpected(ttExpected) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 096911dcbc3..16c64644ab7 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -267,6 +267,7 @@ func semanticIngressEquals(desired, existing *netv1.Ingress) bool { return equality.Semantic.DeepEqual(desired.Spec, existing.Spec) } +// Check for route created by odh-model-controller. If the route is found, use it as the isvc URL func getRouteURLIfExists(cli client.Client, isvc *v1beta1.InferenceService) (*apis.URL, error) { foundRoute := false routeReady := false @@ -372,7 +373,7 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { } internalHost := getRawServiceHost(isvc, r.client) if authEnabled { - internalHost += ":8443" + internalHost += ":" + strconv.Itoa(constants.OauthProxyPort) } isvc.Status.Address = &duckv1.Addressable{ URL: &apis.URL{ diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go index d9d31b47791..084fcb51d1d 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/service/service_reconciler.go @@ -110,7 +110,7 @@ func createDefaultSvc(componentMeta metav1.ObjectMeta, componentExt *v1beta1.Com }, Protocol: container.Ports[0].Protocol, } - if servicePort.Name == "" { + if len(servicePort.Name) == 0 { servicePort.Name = "http" } servicePorts = append(servicePorts, servicePort) diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils.go b/pkg/controller/v1beta1/inferenceservice/utils/utils.go index dd04f672518..d6ec29cf7a4 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils.go @@ -444,6 +444,8 @@ func MergeServingRuntimeAndInferenceServiceSpecs(srContainers []v1.Container, is return containerIndexInSR, mergedContainer, mergedPodSpec, nil } +// Since the "cookie-secret" arg in the oauth proxy container is generated randomly, +// we exclude that arg while comparing existing and desired deployment specs func RemoveCookieSecretArg(deployment appsv1.Deployment) *appsv1.Deployment { dep := deployment.DeepCopy() for i, container := range dep.Spec.Template.Spec.Containers { From 0aa3eeefd37757912d1760e5cc3a3ad255608e59 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Thu, 5 Dec 2024 20:11:10 -0500 Subject: [PATCH 19/20] bug fix Signed-off-by: Vedant Mahabaleshwarkar --- .../odh/inferenceservice-config-patch.yaml | 2 +- pkg/constants/constants.go | 1 + .../rawkube_controller_test.go | 11 +-- .../deployment/deployment_reconciler.go | 55 ++++++++++--- .../deployment/deployment_reconciler_test.go | 17 ++-- .../ingress/kube_ingress_reconciler.go | 80 ++++++------------- .../v1beta1/inferenceservice/utils/utils.go | 49 ++++++++++++ 7 files changed, 134 insertions(+), 81 deletions(-) diff --git a/config/overlays/odh/inferenceservice-config-patch.yaml b/config/overlays/odh/inferenceservice-config-patch.yaml index f1e81bfe517..1da7fa9fa4b 100644 --- a/config/overlays/odh/inferenceservice-config-patch.yaml +++ b/config/overlays/odh/inferenceservice-config-patch.yaml @@ -11,7 +11,7 @@ data: "memoryRequest": "64Mi", "memoryLimit": "128Mi", "cpuRequest": "100m", - "cpuLimit": "200m", + "cpuLimit": "200m" } storageInitializer: |- { diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 70df2ac2f66..e1b1e80d0b5 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -457,6 +457,7 @@ const ( OauthProxyResourceMemoryRequest = "64Mi" OauthProxyResourceCPURequest = "100m" OauthProxyImage = "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:234af927030921ab8f7333f61f967b4b4dee37a1b3cf85689e9e63240dd62800" + DefaultServiceAccount = "default" ) type ProtocolVersion int diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index 0b1e9c23ea5..41a3a95381a 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -417,7 +417,7 @@ var _ = Describe("v1beta1 inference service controller", func() { }, URL: &apis.URL{ Scheme: "http", - Host: "raw-foo-default.example.com", + Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local", serviceKey.Name, serviceKey.Namespace), }, Address: &duckv1.Addressable{ URL: &apis.URL{ @@ -832,7 +832,7 @@ var _ = Describe("v1beta1 inference service controller", func() { }, URL: &apis.URL{ Scheme: "http", - Host: "raw-foo-customized-default.example.com", + Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local", serviceKey.Name, serviceKey.Namespace), }, Address: &duckv1.Addressable{ URL: &apis.URL{ @@ -1238,7 +1238,7 @@ var _ = Describe("v1beta1 inference service controller", func() { }, URL: &apis.URL{ Scheme: "http", - Host: "raw-foo-2-default.example.com", + Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local", serviceKey.Name, serviceKey.Namespace), }, Address: &duckv1.Addressable{ URL: &apis.URL{ @@ -1715,7 +1715,7 @@ var _ = Describe("v1beta1 inference service controller", func() { }, URL: &apis.URL{ Scheme: "http", - Host: fmt.Sprintf("%s-default.example.com", serviceName), + Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local", serviceKey.Name, serviceKey.Namespace), }, Address: &duckv1.Addressable{ URL: &apis.URL{ @@ -2148,7 +2148,7 @@ var _ = Describe("v1beta1 inference service controller", func() { }, URL: &apis.URL{ Scheme: "http", - Host: fmt.Sprintf("%s.%s.%s", serviceName, serviceKey.Namespace, domain), + Host: fmt.Sprintf("%s-predictor.%s.svc.cluster.local", serviceKey.Name, serviceKey.Namespace), }, Address: &duckv1.Addressable{ URL: &apis.URL{ @@ -2425,6 +2425,7 @@ var _ = Describe("v1beta1 inference service controller", func() { `--https-address=:8443`, `--provider=openshift`, `--skip-provider-button`, + `--openshift-service-account=default`, `--upstream=http://localhost:8080`, `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go index 3afbb01b82b..ca0c1afb702 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler.go @@ -68,7 +68,7 @@ func NewDeploymentReconciler(client kclient.Client, workerComponentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) (*DeploymentReconciler, error) { - deploymentList, err := createRawDeployment(clientset, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec) + deploymentList, err := createRawDeploymentODH(clientset, componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec) if err != nil { return nil, err } @@ -79,7 +79,26 @@ func NewDeploymentReconciler(client kclient.Client, componentExt: componentExt, }, nil } -func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, + +func createRawDeploymentODH(clientset kubernetes.Interface, componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, + componentExt *v1beta1.ComponentExtensionSpec, + podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) ([]*appsv1.Deployment, error) { + deploymentList, err := createRawDeployment(componentMeta, workerComponentMeta, componentExt, podSpec, workerPodSpec) + if err != nil { + return nil, err + } + if val, ok := componentMeta.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { + for _, deployment := range deploymentList { + err := addOauthContainerToDeployment(clientset, deployment, componentMeta, componentExt, podSpec) + if err != nil { + return nil, err + } + } + } + return deploymentList, nil +} + +func createRawDeployment(componentMeta metav1.ObjectMeta, workerComponentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec, workerPodSpec *corev1.PodSpec) ([]*appsv1.Deployment, error) { var deploymentList []*appsv1.Deployment @@ -107,12 +126,9 @@ func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.Ob } } - defaultDeployment, err := createRawDefaultDeployment(clientset, componentMeta, componentExt, podSpec) + defaultDeployment, err := createRawDefaultDeployment(componentMeta, componentExt, podSpec) if err != nil { - // the calling function will ignore the deploymentlist if an error is also returned - // This is required for the deployment_reconciler_tests - deploymentList = append(deploymentList, defaultDeployment) - return deploymentList, err + return nil, err } if multiNodeEnabled { // Use defaut value(1) if tensor-parallel-size is not set (gpu count) @@ -143,7 +159,7 @@ func createRawDeployment(clientset kubernetes.Interface, componentMeta metav1.Ob return deploymentList, nil } -func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta metav1.ObjectMeta, +func createRawDefaultDeployment(componentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec) (*appsv1.Deployment, error) { podMetadata := componentMeta @@ -168,9 +184,14 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me deployment.Spec.Strategy = *componentExt.DeploymentStrategy } setDefaultDeploymentSpec(&deployment.Spec) + return deployment, nil +} +func addOauthContainerToDeployment(clientset kubernetes.Interface, deployment *appsv1.Deployment, componentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, + podSpec *corev1.PodSpec) error { var isvcname string var upstreamPort string + var sa string if val, ok := componentMeta.Labels[constants.InferenceServiceLabel]; ok { isvcname = val } else { @@ -188,13 +209,19 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me upstreamPort = constants.InferenceServiceDefaultHttpPort } } - oauthProxyContainer, err := generateOauthProxyContainer(clientset, isvcname, componentMeta.Namespace, upstreamPort) + if podSpec.ServiceAccountName == "" { + sa = constants.DefaultServiceAccount + } else { + sa = podSpec.ServiceAccountName + } + oauthProxyContainer, err := generateOauthProxyContainer(clientset, isvcname, componentMeta.Namespace, upstreamPort, sa) if err != nil { // return the deployment without the oauth proxy container if there was an error // This is required for the deployment_reconciler_tests - return deployment, err + return err } - updatedPodSpec := podSpec.DeepCopy() + updatedPodSpec := deployment.Spec.Template.Spec.DeepCopy() + // updatedPodSpec := podSpec.DeepCopy() updatedPodSpec.Containers = append(updatedPodSpec.Containers, *oauthProxyContainer) tlsSecretVolume := corev1.Volume{ Name: tlsVolumeName, @@ -208,8 +235,9 @@ func createRawDefaultDeployment(clientset kubernetes.Interface, componentMeta me updatedPodSpec.Volumes = append(updatedPodSpec.Volumes, tlsSecretVolume) deployment.Spec.Template.Spec = *updatedPodSpec } - return deployment, nil + return nil } + func createRawWorkerDeployment(componentMeta metav1.ObjectMeta, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec, predictorName string, replicas int32) *appsv1.Deployment { @@ -250,7 +278,7 @@ func GetKServeContainerPort(podSpec *corev1.PodSpec) string { } return "" } -func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string) (*corev1.Container, error) { +func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, namespace string, upstreamPort string, sa string) (*corev1.Container, error) { isvcConfigMap, err := clientset.CoreV1().ConfigMaps(constants.KServeNamespace).Get(context.TODO(), constants.InferenceServiceConfigMapName, metav1.GetOptions{}) if err != nil { return nil, err @@ -281,6 +309,7 @@ func generateOauthProxyContainer(clientset kubernetes.Interface, isvc string, na `--https-address=:` + strconv.Itoa(constants.OauthProxyPort), `--provider=openshift`, `--skip-provider-button`, + `--openshift-service-account=` + sa, `--upstream=http://localhost:` + upstreamPort, `--tls-cert=/etc/tls/private/tls.crt`, `--tls-key=/etc/tls/private/tls.key`, diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go index 0cb90d49875..c2f84f765e4 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/deployment/deployment_reconciler_test.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" - testclient "k8s.io/client-go/kubernetes/fake" ) func TestCreateDefaultDeployment(t *testing.T) { @@ -44,7 +43,6 @@ func TestCreateDefaultDeployment(t *testing.T) { podSpec *corev1.PodSpec workerPodSpec *corev1.PodSpec } - clientset := testclient.NewClientset() testInput := map[string]args{ "defaultDeployment": { objectMeta: metav1.ObjectMeta{ @@ -393,7 +391,10 @@ func TestCreateDefaultDeployment(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, _ := createRawDeployment(clientset, tt.args.objectMeta, tt.args.workerObjectMeta, tt.args.componentExt, tt.args.podSpec, tt.args.workerPodSpec) + got, err := createRawDeployment(tt.args.objectMeta, tt.args.workerObjectMeta, tt.args.componentExt, tt.args.podSpec, tt.args.workerPodSpec) + if err != nil { + t.Error(err) + } if len(got) == 0 { t.Errorf("Got empty deployment") } @@ -465,7 +466,10 @@ func TestCreateDefaultDeployment(t *testing.T) { ttExpected := getDefaultExpectedDeployment() // update objectMeta using modify func - got, _ := createRawDeployment(clientset, ttArgs.objectMeta, ttArgs.workerObjectMeta, ttArgs.componentExt, tt.modifyArgs(ttArgs).podSpec, tt.modifyArgs(ttArgs).workerPodSpec) + got, err := createRawDeployment(ttArgs.objectMeta, ttArgs.workerObjectMeta, ttArgs.componentExt, tt.modifyArgs(ttArgs).podSpec, tt.modifyArgs(ttArgs).workerPodSpec) + if err != nil { + t.Error(err) + } if len(got) == 0 { t.Errorf("Got empty deployment") } @@ -771,7 +775,10 @@ func TestCreateDefaultDeployment(t *testing.T) { ttExpected := getDefaultExpectedDeployment() // update objectMeta using modify func - got, _ := createRawDeployment(clientset, tt.modifyObjectMetaArgs(ttArgs).objectMeta, tt.modifyWorkerObjectMetaArgs(ttArgs).workerObjectMeta, ttArgs.componentExt, tt.modifyPodSpecArgs(ttArgs).podSpec, tt.modifyWorkerPodSpecArgs(ttArgs).workerPodSpec) + got, err := createRawDeployment(tt.modifyObjectMetaArgs(ttArgs).objectMeta, tt.modifyWorkerObjectMetaArgs(ttArgs).workerObjectMeta, ttArgs.componentExt, tt.modifyPodSpecArgs(ttArgs).podSpec, tt.modifyWorkerPodSpecArgs(ttArgs).workerPodSpec) + if err != nil { + t.Error(err) + } if len(got) == 0 { t.Errorf("Got empty deployment") } diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 16c64644ab7..62d229a13d6 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -23,8 +23,8 @@ import ( v1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" + v1beta1utils "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/utils" "github.com/kserve/kserve/pkg/utils" - routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -72,6 +72,27 @@ func createRawURL(isvc *v1beta1.InferenceService, return url, nil } +func createRawURLODH(client client.Client, isvc *v1beta1.InferenceService, authEnabled bool) (*knapis.URL, error) { + url := &knapis.URL{} + if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ODHRouteEnabled { + var err error + url, err = v1beta1utils.GetRouteURLIfExists(client, isvc.ObjectMeta, isvc.Name) + if err != nil { + return nil, err + } + } else { + url = &apis.URL{ + Host: getRawServiceHost(isvc, client), + Scheme: "http", + Path: "", + } + if authEnabled { + url.Host += ":" + strconv.Itoa(constants.OauthProxyPort) + } + } + return url, nil +} + func getRawServiceHost(isvc *v1beta1.InferenceService, client client.Client) string { existingService := &corev1.Service{} if isvc.Spec.Transformer != nil { @@ -267,52 +288,6 @@ func semanticIngressEquals(desired, existing *netv1.Ingress) bool { return equality.Semantic.DeepEqual(desired.Spec, existing.Spec) } -// Check for route created by odh-model-controller. If the route is found, use it as the isvc URL -func getRouteURLIfExists(cli client.Client, isvc *v1beta1.InferenceService) (*apis.URL, error) { - foundRoute := false - routeReady := false - route := &routev1.Route{} - err := cli.Get(context.TODO(), types.NamespacedName{Name: isvc.Name, Namespace: isvc.Namespace}, route) - if err != nil { - return nil, err - } - - // Check if the route is owned by the InferenceService - for _, ownerRef := range route.OwnerReferences { - if ownerRef.UID == isvc.UID { - foundRoute = true - } - } - - // Check if the route is admitted - for _, ingress := range route.Status.Ingress { - for _, condition := range ingress.Conditions { - if condition.Type == "Admitted" && condition.Status == "True" { - routeReady = true - } - } - } - - if !(foundRoute && routeReady) { - return nil, fmt.Errorf("route %s/%s not found or not ready", isvc.Namespace, isvc.Name) - } - - // Construct the URL - host := route.Spec.Host - scheme := "http" - if route.Spec.TLS != nil && route.Spec.TLS.Termination != "" { - scheme = "https" - } - - // Create the URL as an apis.URL object - routeURL := &apis.URL{ - Scheme: scheme, - Host: host, - } - - return routeURL, nil -} - func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { var err error isInternal := false @@ -358,16 +333,7 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { if val, ok := isvc.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { authEnabled = true } - isvc.Status.URL, err = createRawURL(isvc, r.ingressConfig, authEnabled) - if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ODHRouteEnabled { - routeUrl, err := getRouteURLIfExists(r.client, isvc) - if err != nil { - return err - } - if routeUrl != nil { - isvc.Status.URL = routeUrl - } - } + isvc.Status.URL, err = createRawURLODH(r.client, isvc, authEnabled) if err != nil { return err } diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils.go b/pkg/controller/v1beta1/inferenceservice/utils/utils.go index d6ec29cf7a4..38f68c38b41 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils.go @@ -26,7 +26,10 @@ import ( "sort" "strings" + routev1 "github.com/openshift/api/route/v1" appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "knative.dev/pkg/apis" "github.com/pkg/errors" goerrors "github.com/pkg/errors" @@ -461,3 +464,49 @@ func RemoveCookieSecretArg(deployment appsv1.Deployment) *appsv1.Deployment { } return dep } + +// Check for route created by odh-model-controller. If the route is found, use it as the isvc URL +func GetRouteURLIfExists(cli client.Client, metadata metav1.ObjectMeta, isvcName string) (*apis.URL, error) { + foundRoute := false + routeReady := false + route := &routev1.Route{} + err := cli.Get(context.TODO(), types.NamespacedName{Name: isvcName, Namespace: metadata.Namespace}, route) + if err != nil { + return nil, err + } + + // Check if the route is owned by the InferenceService + for _, ownerRef := range route.OwnerReferences { + if ownerRef.UID == metadata.UID { + foundRoute = true + } + } + + // Check if the route is admitted + for _, ingress := range route.Status.Ingress { + for _, condition := range ingress.Conditions { + if condition.Type == "Admitted" && condition.Status == "True" { + routeReady = true + } + } + } + + if !(foundRoute && routeReady) { + return nil, fmt.Errorf("route %s/%s not found or not ready", metadata.Namespace, metadata.Name) + } + + // Construct the URL + host := route.Spec.Host + scheme := "http" + if route.Spec.TLS != nil && route.Spec.TLS.Termination != "" { + scheme = "https" + } + + // Create the URL as an apis.URL object + routeURL := &apis.URL{ + Scheme: scheme, + Host: host, + } + + return routeURL, nil +} From 96f4b7dfc18c31e7876db82dfd26c84f7c48f4f3 Mon Sep 17 00:00:00 2001 From: Vedant Mahabaleshwarkar Date: Fri, 6 Dec 2024 13:51:22 -0500 Subject: [PATCH 20/20] fix lint error Signed-off-by: Vedant Mahabaleshwarkar --- .../ingress/kube_ingress_reconciler.go | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go index 62d229a13d6..5bf7da76d53 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/kube_ingress_reconciler.go @@ -57,22 +57,21 @@ func NewRawIngressReconciler(client client.Client, }, nil } -func createRawURL(isvc *v1beta1.InferenceService, - ingressConfig *v1beta1.IngressConfig, authEnabled bool) (*knapis.URL, error) { - var err error - url := &knapis.URL{} - url.Scheme = ingressConfig.UrlScheme - url.Host, err = GenerateDomainName(isvc.Name, isvc.ObjectMeta, ingressConfig) - if err != nil { - return nil, err - } - if authEnabled { - url.Host += ":" + strconv.Itoa(constants.OauthProxyPort) - } - return url, nil -} +func createRawURL(client client.Client, isvc *v1beta1.InferenceService, authEnabled bool) (*knapis.URL, error) { + // upstream implementation + // var err error + // url := &knapis.URL{} + // url.Scheme = ingressConfig.UrlScheme + // url.Host, err = GenerateDomainName(isvc.Name, isvc.ObjectMeta, ingressConfig) + // if err != nil { + // return nil, err + // } + // if authEnabled { + // url.Host += ":" + strconv.Itoa(constants.OauthProxyPort) + // } + // return url, nil -func createRawURLODH(client client.Client, isvc *v1beta1.InferenceService, authEnabled bool) (*knapis.URL, error) { + // ODH changes url := &knapis.URL{} if val, ok := isvc.Labels[constants.NetworkVisibility]; ok && val == constants.ODHRouteEnabled { var err error @@ -333,7 +332,7 @@ func (r *RawIngressReconciler) Reconcile(isvc *v1beta1.InferenceService) error { if val, ok := isvc.Labels[constants.ODHKserveRawAuth]; ok && val == "true" { authEnabled = true } - isvc.Status.URL, err = createRawURLODH(r.client, isvc, authEnabled) + isvc.Status.URL, err = createRawURL(r.client, isvc, authEnabled) if err != nil { return err }