From 49ec883b03941ad3d62150dcc1de060f46e57699 Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 14:39:59 +0300 Subject: [PATCH 01/16] Add the Cluster Autoscaler --- platform/TEMPLATING.md | 3 +- .../core-services/30-cluster-autoscaler.yaml | 28 ++ .../cluster-autoscaler/application.yaml | 251 ++++++++++++++++++ platform/terraform/hosting_provider/output.tf | 4 + platform/terraform/modules/cloud_aws/iam.tf | 15 ++ tools/cli/commands/setup.py | 1 + 6 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml diff --git a/platform/TEMPLATING.md b/platform/TEMPLATING.md index 2b227264..699aa81d 100644 --- a/platform/TEMPLATING.md +++ b/platform/TEMPLATING.md @@ -35,6 +35,7 @@ IAM roles for core components created during the setup process - `EXTERNAL_DNS_IAM_ROLE_RN` - External DNS IAM role for a K8s service account - `IAC_PR_AUTOMATION_IAM_ROLE_RN` - IaC PR automation IAM role for a K8s service account - `SECRET_MANAGER_IAM_ROLE_RN` - Secrets Manager (Vault) IAM role for a K8s service account +- `CLUSTER_AUTOSCALER_IAM_ROLE_RN` - Cluster Autoscaler IAM role for a K8s service account ### Ingress @@ -124,4 +125,4 @@ OIDC provider configuration Note!: URL does not contain protocol prefix - `REGISTRY_ROBO_USER_AUTH` - Registry (Harbor) auth string - `REGISTRY_ROBO_USER_PASSWORD` - Registry (Harbor) machine user password - `TF_BACKEND_STORAGE_NAME` - Terraform state storage backend location -- `VAULT_ROOT_TOKEN` - Secrets Manager (Vault) root access token \ No newline at end of file +- `VAULT_ROOT_TOKEN` - Secrets Manager (Vault) root access token diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml new file mode 100644 index 00000000..a96724af --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cluster-auroscaler-components + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: '35' +spec: + project: core + source: + repoURL: + path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/cluster-auroscaler + targetRevision: HEAD + destination: + server: https://kubernetes.default.svc + namespace: cluster-auroscaler + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=false + retry: + limit: 10 + backoff: + duration: 30s + maxDuration: 15m0s + factor: 2 diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml new file mode 100644 index 00000000..c20271fa --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml @@ -0,0 +1,251 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cluster-autoscaler + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + project: core + source: + repoURL: https://kubernetes.github.io/autoscaler + chart: cluster-autoscaler + targetRevision: 9.34.0 + helm: + values: |- + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + # affinity -- Affinity for pod assignment + affinity: {} + + # additionalLabels -- Labels to add to each object of the chart. + additionalLabels: {} + + autoDiscovery: + # cloudProviders `aws`, `gce`, `azure`, `magnum`, `clusterapi` and `oci` are supported by auto-discovery at this time + # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup + + # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. + # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=azure`, using tags defined in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md#auto-discovery-setup. + # Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. + # Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. + # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. + clusterName: + + # autoDiscovery.tags -- ASG tags to match, run through `tpl`. + tags: + - k8s.io/cluster-autoscaler/enabled + - k8s.io/cluster-autoscaler/ + # - kubernetes.io/cluster/{{ .Values.autoDiscovery.clusterName }} + + # autoDiscovery.roles -- Magnum node group roles to match. + roles: + - worker + + # awsRegion -- AWS region (required if `cloudProvider=aws`) + awsRegion: + + # cloudProvider -- The cloud provider where the autoscaler runs. + # Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. + # `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. + # `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. + cloudProvider: + + # containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) + containerSecurityContext: + capabilities: + drop: + - ALL + + deployment: + # deployment.annotations -- Annotations to add to the Deployment object. + annotations: {} + + # dnsPolicy -- Defaults to `ClusterFirst`. Valid values are: + # `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. + # If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. + dnsPolicy: ClusterFirst + + # envFromConfigMap -- ConfigMap name to use as envFrom. + envFromConfigMap: "" + + # envFromSecret -- Secret name to use as envFrom. + envFromSecret: "" + + # extraArgs -- Additional container arguments. + # Refer to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca for the full list of cluster autoscaler + # parameters and their default values. + # Everything after the first _ will be ignored allowing the use of multi-string arguments. + extraArgs: + logtostderr: true + stderrthreshold: info + v: 4 + # write-status-configmap: true + # status-config-map-name: cluster-autoscaler-status + # leader-elect: true + # leader-elect-resource-lock: endpoints + # skip-nodes-with-local-storage: true + # expander: random + # scale-down-enabled: true + # balance-similar-node-groups: true + # min-replica-count: 0 + # scale-down-utilization-threshold: 0.5 + # scale-down-non-empty-candidates-count: 30 + # max-node-provision-time: 15m0s + # scan-interval: 10s + # scale-down-delay-after-add: 10m + # scale-down-delay-after-delete: 0s + # scale-down-delay-after-failure: 3m + # scale-down-unneeded-time: 10m + # skip-nodes-with-system-pods: true + # balancing-ignore-label_1: first-label-to-ignore + # balancing-ignore-label_2: second-label-to-ignore + + # hostNetwork -- Whether to expose network interfaces of the host machine to pods. + hostNetwork: false + + # kwokConfigMapName -- configmap for configuring kwok provider + kwokConfigMapName: "kwok-provider-config" + + # nodeSelector -- Node labels for pod assignment. Ref: https://kubernetes.io/docs/user-guide/node-selection/. + nodeSelector: {} + + # podAnnotations -- Annotations to add to each pod. + podAnnotations: {} + + # podDisruptionBudget -- Pod disruption budget. + podDisruptionBudget: + maxUnavailable: 1 + # minAvailable: 2 + + # podLabels -- Labels to add to each pod. + podLabels: {} + + # priorityClassName -- priorityClassName + priorityClassName: "system-cluster-critical" + + ## Custom PrometheusRule to be defined + ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart + ## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions + prometheusRule: + # prometheusRule.enabled -- If true, creates a Prometheus Operator PrometheusRule. + enabled: true + # prometheusRule.additionalLabels -- Additional labels to be set in metadata. + additionalLabels: {} + # prometheusRule.namespace -- Namespace which Prometheus is running in. + namespace: monitoring + # prometheusRule.interval -- How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). + interval: null + # prometheusRule.rules -- Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). + rules: [] + + rbac: + # rbac.create -- If `true`, create and use RBAC resources. + create: true + # rbac.pspEnabled -- If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. + # Must be used with `rbac.create` set to `true`. + pspEnabled: false + # rbac.clusterScoped -- if set to false will only provision RBAC to alter resources in the current namespace. Most useful for Cluster-API + clusterScoped: true + serviceAccount: + # rbac.serviceAccount.annotations -- Additional Service Account annotations. + annotations: + : “” + # rbac.serviceAccount.create -- If `true` and `rbac.create` is also true, a Service Account will be created. + create: true + # rbac.serviceAccount.name -- The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. + name: + cluster-autoscaler + # rbac.serviceAccount.automountServiceAccountToken -- Automount API credentials for a Service Account. + automountServiceAccountToken: true + + # replicaCount -- Desired number of pods + replicaCount: 1 + + # resources -- Pod resource requests and limits. + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + + # revisionHistoryLimit -- The number of revisions to keep. + revisionHistoryLimit: 10 + + # securityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + + service: + # service.create -- If `true`, a Service will be created. + create: true + # service.annotations -- Annotations to add to service + annotations: {} + # service.labels -- Labels to add to service + labels: {} + # service.externalIPs -- List of IP addresses at which the service is available. Ref: https://kubernetes.io/docs/user-guide/services/#external-ips. + externalIPs: [] + + # service.loadBalancerIP -- IP address to assign to load balancer (if supported). + loadBalancerIP: "" + # service.loadBalancerSourceRanges -- List of IP CIDRs allowed access to load balancer (if supported). + loadBalancerSourceRanges: [] + # service.servicePort -- Service port to expose. + servicePort: 8085 + # service.portName -- Name for service port. + portName: http + # service.type -- Type of service to create. + type: ClusterIP + + ## Are you using Prometheus Operator? + serviceMonitor: + # serviceMonitor.enabled -- If true, creates a Prometheus Operator ServiceMonitor. + enabled: true + # serviceMonitor.interval -- Interval that Prometheus scrapes Cluster Autoscaler metrics. + interval: 10s + # serviceMonitor.namespace -- Namespace which Prometheus is running in. + namespace: monitoring + ## [Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#prometheus-operator-1) + ## [Kube Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#exporters) + # serviceMonitor.selector -- Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install. + selector: + release: prometheus-operator + # serviceMonitor.path -- The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard) + path: /metrics + # serviceMonitor.annotations -- Annotations to add to service monitor + annotations: {} + ## [RelabelConfig](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig) + # serviceMonitor.metricRelabelings -- MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: {} + + # tolerations -- List of node taints to tolerate (requires Kubernetes >= 1.6). + tolerations: [] + + # topologySpreadConstraints -- You can use topology spread constraints to control how Pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains. (requires Kubernetes >= 1.19). + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: cluster-autoscaler + + # updateStrategy -- [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) + updateStrategy: {} + # rollingUpdate: + # maxSurge: 1 + # maxUnavailable: 0 + # type: RollingUpdate + + destination: + server: https://kubernetes.default.svc + namespace: cluster-autoscaler + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index 5010689d..67e21f49 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -25,6 +25,10 @@ output "secret_manager_role" { value = module.hosting-provider.secret_manager_irsa_role description = "Secrets Manager IAM role for a K8s service account" } +output "cluster_autoscaler_role" { + value = module.hosting-provider.cluster_autoscaler_irsa_role + description = "Secrets Manager IAM role for a K8s service account" +} # cluster output "cluster_endpoint" { diff --git a/platform/terraform/modules/cloud_aws/iam.tf b/platform/terraform/modules/cloud_aws/iam.tf index e435b78f..fed038dc 100644 --- a/platform/terraform/modules/cloud_aws/iam.tf +++ b/platform/terraform/modules/cloud_aws/iam.tf @@ -130,3 +130,18 @@ module "secret_manager_irsa_role" { } +# Cluster Autoscaler +module "cluster_autoscaler_irsa_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + + role_name = "${local.name}-cluster-autoscaler" + attach_cluster_autoscaler_policy = true + cluster_autoscaler_cluster_names = [module.eks.cluster_name] + + oidc_providers = { + ex = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["cluster-autoscaler:cluster-autoscaler"] + } + } +} diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index eb57f7a5..e800af82 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -335,6 +335,7 @@ def setup( p.parameters[""] = hp_out["cert_manager_role"] p.parameters[""] = hp_out["external_dns_role"] p.parameters[""] = hp_out["secret_manager_role"] + p.parameters[""] = hp_out["cluster_autoscaler_role"] # cluster p.internals["CC_CLUSTER_ENDPOINT"] = hp_out["cluster_endpoint"] p.internals["CC_CLUSTER_CA_CERT_DATA"] = hp_out["cluster_certificate_authority_data"] From a27f7de467f147d9183f0b83c08d966956fcbd8e Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 15:17:40 +0300 Subject: [PATCH 02/16] Add the cluster autoscaler to the cloud_aws provider --- platform/terraform/modules/cloud_aws/outputs.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platform/terraform/modules/cloud_aws/outputs.tf b/platform/terraform/modules/cloud_aws/outputs.tf index deaf47cd..0ac99194 100644 --- a/platform/terraform/modules/cloud_aws/outputs.tf +++ b/platform/terraform/modules/cloud_aws/outputs.tf @@ -53,6 +53,10 @@ output "secret_manager_irsa_role" { description = "AWS Secretsmanager IAM Role ARN" value = module.secret_manager_irsa_role.iam_role_arn } +output "cluster_autoscaler_irsa_role" { + description = "AWS Secretsmanager IAM Role ARN" + value = module.cluster_autoscaler_irsa_role.iam_role_arn +} ################################################################################ # Cluster From 4b68488b892593a5a1eaf1dcf558c854297002a9 Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 15:41:37 +0300 Subject: [PATCH 03/16] Change the Cluster Autoscaler component path --- .../cc-cluster/core-services/30-cluster-autoscaler.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml index a96724af..e815c91f 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml @@ -9,7 +9,7 @@ spec: project: core source: repoURL: - path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/cluster-auroscaler + path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-auroscaler targetRevision: HEAD destination: server: https://kubernetes.default.svc From b6abec4e1e485ea6bb81e04376d47fb67edf80ed Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 15:42:25 +0300 Subject: [PATCH 04/16] Change the Cluster Autoscaler component path --- .../cc-cluster/core-services/30-cluster-autoscaler.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml index e815c91f..3396c13d 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml @@ -9,7 +9,7 @@ spec: project: core source: repoURL: - path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-auroscaler + path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler targetRevision: HEAD destination: server: https://kubernetes.default.svc From 080a6f6ff3371171035d4bffb48a763bc05938b1 Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 15:43:47 +0300 Subject: [PATCH 05/16] Change the syn-vawe to 30 --- .../cc-cluster/core-services/30-cluster-autoscaler.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml index 3396c13d..71c9616e 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml @@ -4,7 +4,7 @@ metadata: name: cluster-auroscaler-components namespace: argocd annotations: - argocd.argoproj.io/sync-wave: '35' + argocd.argoproj.io/sync-wave: '30' spec: project: core source: From 894df313c7e60e41a4b51da1b1af57372e7b6ca8 Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 15:44:16 +0300 Subject: [PATCH 06/16] Change the syn-vawe to 30 --- .../cc-cluster/core-services/30-cluster-autoscaler.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml index 71c9616e..91d48071 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: cluster-auroscaler-components + name: cluster-autoscaler-components namespace: argocd annotations: argocd.argoproj.io/sync-wave: '30' @@ -13,7 +13,7 @@ spec: targetRevision: HEAD destination: server: https://kubernetes.default.svc - namespace: cluster-auroscaler + namespace: cluster-autoscaler syncPolicy: automated: prune: true From 4572a441f92d76e5922be0c70467790f84dce415 Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 17:27:40 +0300 Subject: [PATCH 07/16] Change the ARN for Cluster Autoscaler. Fix typo --- .../components/kube-system/cluster-autoscaler/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml index c20271fa..5c4d2294 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml @@ -149,7 +149,7 @@ spec: serviceAccount: # rbac.serviceAccount.annotations -- Additional Service Account annotations. annotations: - : “” + : "" # rbac.serviceAccount.create -- If `true` and `rbac.create` is also true, a Service Account will be created. create: true # rbac.serviceAccount.name -- The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. From f7bb51eea75afb0a6013d0b8b1e856b3387d3ea4 Mon Sep 17 00:00:00 2001 From: Alexander Akilin Date: Fri, 22 Dec 2023 17:36:21 +0300 Subject: [PATCH 08/16] Add the Prometheus rule additional rules --- .../components/kube-system/cluster-autoscaler/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml index 5c4d2294..09da8560 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml @@ -130,7 +130,7 @@ spec: # prometheusRule.enabled -- If true, creates a Prometheus Operator PrometheusRule. enabled: true # prometheusRule.additionalLabels -- Additional labels to be set in metadata. - additionalLabels: {} + additionalLabels: ClusterAutoscalerRules # prometheusRule.namespace -- Namespace which Prometheus is running in. namespace: monitoring # prometheusRule.interval -- How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). From 4b3ef82c8b20d29558011c4a4e183224c9201e18 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Wed, 3 Jan 2024 21:58:06 +0100 Subject: [PATCH 09/16] feat: add sa for autoscaler on azure --- .../core-services/30-cluster-autoscaler.yaml | 2 +- platform/terraform/modules/cloud_aws/outputs.tf | 2 +- platform/terraform/modules/cloud_azure/aks.tf | 7 +++---- platform/terraform/modules/cloud_azure/main.tf | 1 + platform/terraform/modules/cloud_azure/outputs.tf | 5 +++++ .../modules/cloud_azure/service_accounts.tf | 14 ++++++++++++++ 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml index 91d48071..37ccbf65 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/30-cluster-autoscaler.yaml @@ -19,7 +19,7 @@ spec: prune: true selfHeal: true syncOptions: - - CreateNamespace=false + - CreateNamespace=true retry: limit: 10 backoff: diff --git a/platform/terraform/modules/cloud_aws/outputs.tf b/platform/terraform/modules/cloud_aws/outputs.tf index 0ac99194..6ef06738 100644 --- a/platform/terraform/modules/cloud_aws/outputs.tf +++ b/platform/terraform/modules/cloud_aws/outputs.tf @@ -54,7 +54,7 @@ output "secret_manager_irsa_role" { value = module.secret_manager_irsa_role.iam_role_arn } output "cluster_autoscaler_irsa_role" { - description = "AWS Secretsmanager IAM Role ARN" + description = "Cluster Autoscaler IAM Role ARN" value = module.cluster_autoscaler_irsa_role.iam_role_arn } diff --git a/platform/terraform/modules/cloud_azure/aks.tf b/platform/terraform/modules/cloud_azure/aks.tf index 35e5f695..dbc1109b 100644 --- a/platform/terraform/modules/cloud_azure/aks.tf +++ b/platform/terraform/modules/cloud_azure/aks.tf @@ -33,12 +33,12 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { # pod_subnet_id = [] ?? do we need it zones = local.azs node_labels = var.cluster_node_labels - enable_auto_scaling = true + enable_auto_scaling = local.enable_native_auto_scaling enable_host_encryption = false enable_node_public_ip = false node_count = local.default_node_group.desired_size - min_count = local.default_node_group.min_size - max_count = local.default_node_group.max_size + min_count = local.enable_native_auto_scaling? local.default_node_group.min_size : null + max_count = local.enable_native_auto_scaling? local.default_node_group.max_size : null max_pods = local.max_pods tags = local.tags } @@ -107,7 +107,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { node_labels = var.cluster_node_labels orchestrator_version = var.cluster_version tags = local.tags - # check with serg enable_node_public_ip = false max_pods = local.max_pods priority = each.value.capacity_type diff --git a/platform/terraform/modules/cloud_azure/main.tf b/platform/terraform/modules/cloud_azure/main.tf index 54bce070..6f9c68ae 100644 --- a/platform/terraform/modules/cloud_azure/main.tf +++ b/platform/terraform/modules/cloud_azure/main.tf @@ -25,6 +25,7 @@ locals { additional_node_pools = try(slice(var.node_groups, 1, length(var.node_groups)), []) max_pods = 100 node_admin_username = "azadmin" + enable_native_auto_scaling = false } resource "azurerm_resource_group" "rg" { diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index f4a9f16a..f583a973 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -32,6 +32,11 @@ output "secret_manager_irsa_role" { description = "Secrets Manager IAM role for a K8s service account" } +output "cluster_autoscaler_irsa_role" { + description = "Cluster Autoscaler IAM Role ARN" + value = module.cluster_autoscaler_sa.app_client_id +} + # cluster output "cluster_endpoint" { value = azurerm_kubernetes_cluster.aks_cluster.kube_admin_config.0.host diff --git a/platform/terraform/modules/cloud_azure/service_accounts.tf b/platform/terraform/modules/cloud_azure/service_accounts.tf index 2038e51d..bf0d4aa2 100644 --- a/platform/terraform/modules/cloud_azure/service_accounts.tf +++ b/platform/terraform/modules/cloud_azure/service_accounts.tf @@ -63,5 +63,19 @@ module "secret_manager_sa" { role_definitions = [{ "name" = "Key Vault Administrator", "scope" = "" }] namespace = "vault" + depends_on = [azurerm_kubernetes_cluster.aks_cluster] +} + +# Cluster Autoscaler +module "cluster_autoscaler_sa" { + source = "./modules/aks_rbac" + + oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + resource_group_name = azurerm_resource_group.rg.name + name = "cluster-autoscaler" + service_account_name = "cluster-autoscaler" + role_definitions = [{ "name" = "Contributor", "scope" = "" }] + namespace = "cluster-autoscaler" + depends_on = [azurerm_kubernetes_cluster.aks_cluster] } \ No newline at end of file From 6878aa8d6ca6f7447211b83918e0678203756acb Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Mon, 8 Jan 2024 14:34:56 +0100 Subject: [PATCH 10/16] feat: add cluster_oidc_issuer_url to tf output --- platform/terraform/hosting_provider/output.tf | 5 +++++ platform/terraform/modules/cloud_aws/outputs.tf | 1 + platform/terraform/modules/cloud_azure/outputs.tf | 6 ++++++ tools/cli/commands/setup.py | 1 + 4 files changed, 13 insertions(+) diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index 67e21f49..1d51669e 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -41,6 +41,11 @@ output "cluster_certificate_authority_data" { description = "K8s cluster Certificate Authority certificate data" sensitive = true } +output "cluster_oidc_issuer_url" { + value = module.hosting-provider.cluster_oidc_issuer_url + description = "The URL on the K8s cluster for the OpenID Connect identity provider" + sensitive = true +} # secret manager output "secret_manager_seal_key" { diff --git a/platform/terraform/modules/cloud_aws/outputs.tf b/platform/terraform/modules/cloud_aws/outputs.tf index 6ef06738..0edc8490 100644 --- a/platform/terraform/modules/cloud_aws/outputs.tf +++ b/platform/terraform/modules/cloud_aws/outputs.tf @@ -90,6 +90,7 @@ output "cluster_name" { output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster for the OpenID Connect identity provider" value = module.eks.cluster_oidc_issuer_url + sensitive = true } output "cluster_platform_version" { diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index f583a973..3a4b3fc5 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -49,6 +49,12 @@ output "cluster_certificate_authority_data" { sensitive = true } +output "cluster_oidc_issuer_url" { + value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + description = "Cluster OIDC provider" + sensitive = true +} + # secret manager output "secret_manager_unseal_key" { value = azurerm_key_vault_key.secret_manager_unseal_kms_key.name diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index bd37331e..70c4bcf6 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -340,6 +340,7 @@ def setup( p.internals["CC_CLUSTER_ENDPOINT"] = hp_out["cluster_endpoint"] p.internals["CC_CLUSTER_CA_CERT_DATA"] = hp_out["cluster_certificate_authority_data"] p.internals["CC_CLUSTER_CA_CERT_PATH"] = write_ca_cert(hp_out["cluster_certificate_authority_data"]) + p.internals["CC_CLUSTER_OIDC_ISSUER_URL"] = hp_out["cluster_oidc_issuer_url"] # artifact storage p.parameters[""] = hp_out["artifact_storage"] # kms keys From ad9a07cf96ee32d3a77f31bed32cc2591a77688e Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Tue, 30 Jan 2024 21:40:20 +0100 Subject: [PATCH 11/16] feat: k8s autoscaler configuration templating --- QUICKSTART.md | 2 +- .../cluster-autoscaler/application.yaml | 7 +++-- platform/terraform/hosting_provider/output.tf | 28 ++++++++++++++----- .../terraform/modules/cloud_aws/outputs.tf | 5 ++++ platform/terraform/modules/cloud_azure/aks.tf | 1 + .../terraform/modules/cloud_azure/outputs.tf | 16 +++++++++++ tools/cli/commands/setup.py | 7 +++++ tools/cli/services/cloud/aws/aws_manager.py | 4 +++ .../cli/services/cloud/azure/azure_manager.py | 24 ++++++++++++++++ tools/cli/services/cloud/azure/azure_sdk.py | 7 +++++ .../services/cloud/cloud_provider_manager.py | 12 ++++++++ tools/poetry.lock | 18 +++++++++++- tools/pyproject.toml | 1 + 13 files changed, 120 insertions(+), 12 deletions(-) diff --git a/QUICKSTART.md b/QUICKSTART.md index 7a3f4f43..670b30ae 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -19,7 +19,7 @@ - AWS - Supported - Azure - Experimental* -- GCP - Will be added to a future release +- GCP - Will be added to a future release \* Experimental functions are provided for you to try, but are not documented or supported, and are likely to be buggy, or to change after release. diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml index 09da8560..41d12226 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml @@ -41,15 +41,16 @@ spec: roles: - worker - # awsRegion -- AWS region (required if `cloudProvider=aws`) - awsRegion: - # cloudProvider -- The cloud provider where the autoscaler runs. # Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. # `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. # `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. cloudProvider: + # cloud provider specific configuration + # + # --- + # containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) containerSecurityContext: capabilities: diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index 1d51669e..f948da4d 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -1,10 +1,14 @@ +################################################################################ # network +################################################################################ output "network_id" { value = module.hosting-provider.network_id description = "Platform primary K8s cluster network ID" } +################################################################################ # IAM roles +################################################################################ output "iam_ci_role" { value = module.hosting-provider.iam_ci_irsa_role description = "Continuous Integration IAM role for K8s service account" @@ -30,7 +34,9 @@ output "cluster_autoscaler_role" { description = "Secrets Manager IAM role for a K8s service account" } +################################################################################ # cluster +################################################################################ output "cluster_endpoint" { value = module.hosting-provider.cluster_endpoint description = "K8s cluster admin API endpoint" @@ -46,23 +52,31 @@ output "cluster_oidc_issuer_url" { description = "The URL on the K8s cluster for the OpenID Connect identity provider" sensitive = true } +output "cluster_node_groups" { + value = module.hosting-provider.cluster_node_groups + description = "K8s cluster node groups" +} + +# Output part for Azure module only: +output "kube_config_raw" { + value = module.hosting-provider.kube_config_raw + sensitive = true + description = "Contains the Kubernetes config to be used by kubectl and other compatible tools." +} +################################################################################ # secret manager +################################################################################ output "secret_manager_seal_key" { value = module.hosting-provider.secret_manager_unseal_key description = "Secret Manager seal key" sensitive = true } +################################################################################ # artifact storage +################################################################################ output "artifact_storage" { value = module.hosting-provider.artifacts_storage description = "Continuous Integration Artifact Repository storage backend" } - -# Output part for Azure module only: -output "kube_config_raw" { - value = module.hosting-provider.kube_config_raw - sensitive = true - description = "Contains the Kubernetes config to be used by kubectl and other compatible tools." -} diff --git a/platform/terraform/modules/cloud_aws/outputs.tf b/platform/terraform/modules/cloud_aws/outputs.tf index 0edc8490..2101cfbb 100644 --- a/platform/terraform/modules/cloud_aws/outputs.tf +++ b/platform/terraform/modules/cloud_aws/outputs.tf @@ -108,6 +108,11 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +output "cluster_node_groups" { + value = var.node_groups + description = "Cluster node groups" +} + ################################################################################ # KMS Key ################################################################################ diff --git a/platform/terraform/modules/cloud_azure/aks.tf b/platform/terraform/modules/cloud_azure/aks.tf index dbc1109b..5b9d3cc8 100644 --- a/platform/terraform/modules/cloud_azure/aks.tf +++ b/platform/terraform/modules/cloud_azure/aks.tf @@ -25,6 +25,7 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { image_cleaner_enabled = false azure_policy_enabled = true http_application_routing_enabled = false + node_resource_group = "${local.name}-vmss-rg" default_node_pool { name = local.default_node_group.name diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index 3a4b3fc5..3406bc75 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -4,13 +4,17 @@ output "kube_config_raw" { description = "Contains the Kubernetes config to be used by kubectl and other compatible tools." } +################################################################################ # network +################################################################################ output "network_id" { value = azurerm_virtual_network.vnet.id description = "Platform primary K8s cluster network ID" } +################################################################################ # IAM roles +################################################################################ output "iam_ci_irsa_role" { value = module.ci_sa.app_client_id description = "Continuous Integration IAM role for K8s service account" @@ -37,12 +41,15 @@ output "cluster_autoscaler_irsa_role" { value = module.cluster_autoscaler_sa.app_client_id } +################################################################################ # cluster +################################################################################ output "cluster_endpoint" { value = azurerm_kubernetes_cluster.aks_cluster.kube_admin_config.0.host description = "K8s cluster admin API endpoint" sensitive = true } + output "cluster_certificate_authority_data" { value = azurerm_kubernetes_cluster.aks_cluster.kube_admin_config.0.cluster_ca_certificate description = "K8s cluster Certificate Authority certificate data" @@ -55,14 +62,23 @@ output "cluster_oidc_issuer_url" { sensitive = true } +output "cluster_node_groups" { + value = var.node_groups + description = "Cluster node groups" +} + +################################################################################ # secret manager +################################################################################ output "secret_manager_unseal_key" { value = azurerm_key_vault_key.secret_manager_unseal_kms_key.name description = "Secret Manager seal key" sensitive = true } +################################################################################ # artifact storage +################################################################################ output "artifacts_storage" { value = azurerm_storage_container.artifacts_repository.name description = "Continuous Integration Artifact Repository storage backend" diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 70c4bcf6..b465a1be 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -341,6 +341,13 @@ def setup( p.internals["CC_CLUSTER_CA_CERT_DATA"] = hp_out["cluster_certificate_authority_data"] p.internals["CC_CLUSTER_CA_CERT_PATH"] = write_ca_cert(hp_out["cluster_certificate_authority_data"]) p.internals["CC_CLUSTER_OIDC_ISSUER_URL"] = hp_out["cluster_oidc_issuer_url"] + + # generate cluster autoscaler config here as it could depend on node groups configuration + p.fragments["# "] = cloud_man.create_autoscaler_snippet( + p.parameters[""], + hp_out["cluster_node_groups"] + ) + # artifact storage p.parameters[""] = hp_out["artifact_storage"] # kms keys diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index 56bd4f36..571d8c6f 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -160,3 +160,7 @@ def create_sidecar_annotation(self) -> str: @trace() def create_external_secrets_config(self, **kwargs) -> str: return "" + + @trace() + def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): + return '''awsRegion: ''' diff --git a/tools/cli/services/cloud/azure/azure_manager.py b/tools/cli/services/cloud/azure/azure_manager.py index 89eeb073..9ca6dbfd 100644 --- a/tools/cli/services/cloud/azure/azure_manager.py +++ b/tools/cli/services/cloud/azure/azure_manager.py @@ -210,3 +210,27 @@ def create_external_secrets_config(self, **kwargs) -> str: "resourceGroup": "{resource_group}", "useWorkloadIdentityExtension": true }}'''.format(subscription_id=self.__azure_sdk.subscription_id, resource_group=location) + + @trace() + def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): + autoscaling_groups = "" + vmss_list = self.__azure_sdk.get_vmss(f"{cluster_name}-vmss-rg") + + if not len(vmss_list): + raise Exception("Could not find vmss") + + for node in node_groups: + vmss_name = [vmss for vmss in vmss_list if vmss.startswith(f'aks-{node["name"]}-')][0] + autoscaling_groups += f''' + - name: {vmss_name} + minSize: {node["min_size"]} + maxSize: {node["max_size"]}''' + + tenant_id = self.__azure_sdk.get_tenant_id() + return f'''autoscalingGroups: {autoscaling_groups} + azureClientID: "" + azureResourceGroup: {cluster_name}-vmss-rg + azureSubscriptionID: {self.__azure_sdk.subscription_id} + azureTenantID: {tenant_id} + azureUseWorkloadIdentityExtension: true + azureVMType: "vmss"''' diff --git a/tools/cli/services/cloud/azure/azure_sdk.py b/tools/cli/services/cloud/azure/azure_sdk.py index 7825ffdf..a031d391 100644 --- a/tools/cli/services/cloud/azure/azure_sdk.py +++ b/tools/cli/services/cloud/azure/azure_sdk.py @@ -5,6 +5,7 @@ from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, AzureError, ResourceExistsError from azure.identity import AzureCliCredential from azure.mgmt.authorization import AuthorizationManagementClient +from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.dns import DnsManagementClient from azure.mgmt.privatedns import PrivateDnsManagementClient from azure.mgmt.resource import ResourceManagementClient @@ -12,6 +13,7 @@ from azure.mgmt.storage.v2021_04_01.models import SkuName, Kind from azure.mgmt.subscription import SubscriptionClient from azure.storage.blob import BlobServiceClient + from common.logging_config import logger from services.dns.dns_provider_manager import get_domain_txt_records_dot @@ -29,6 +31,7 @@ def __init__(self, subscription_id: str, location: Optional[str] = None): self.dns_client: DnsManagementClient = DnsManagementClient(self.credential, self.subscription_id) self.storage_mgmt_client = StorageManagementClient(self.credential, self.subscription_id) self.private_dns_client = PrivateDnsManagementClient(self.credential, self.subscription_id) + self.compute_client = ComputeManagementClient(self.credential, self.subscription_id) self.subscription_client = SubscriptionClient(self.credential) self.location = self._validate_location(location) @@ -513,3 +516,7 @@ def get_tenant_id(self) -> str: """ for tenant in self.subscription_client.tenants.list(): return tenant.tenant_id + + def get_vmss(self, rg_name): + vmss_list = self.compute_client.virtual_machine_scale_sets.list(rg_name) + return [v.name for v in vmss_list] diff --git a/tools/cli/services/cloud/cloud_provider_manager.py b/tools/cli/services/cloud/cloud_provider_manager.py index 5402ec71..24c7ebc2 100644 --- a/tools/cli/services/cloud/cloud_provider_manager.py +++ b/tools/cli/services/cloud/cloud_provider_manager.py @@ -63,6 +63,18 @@ def create_hosting_provider_snippet(self): """ pass + @abstractmethod + def create_autoscaler_snippet(self, cluster_name: str, node_groups: [] = []): + """ + Creates K8s Autoscaler configuration snippet + :return: Configuration snippet + + Args: + cluster_name: K8s cluster name + node_groups: Node groups definition + """ + pass + @abstractmethod def create_seal_snippet(self, key_id: str, **kwargs): """ diff --git a/tools/poetry.lock b/tools/poetry.lock index f0902f52..63a9e54b 100644 --- a/tools/poetry.lock +++ b/tools/poetry.lock @@ -114,6 +114,22 @@ azure-common = ">=1.1,<2.0" azure-mgmt-core = ">=1.3.2,<2.0.0" isodate = ">=0.6.1,<1.0.0" +[[package]] +name = "azure-mgmt-compute" +version = "30.5.0" +description = "Microsoft Azure Compute Management Client Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "azure-mgmt-compute-30.5.0.tar.gz", hash = "sha256:ed3ea34b799db0d52ee55e2f1ab4b0f09fa4a08f35e061ecb9aad9fb5a218844"}, + {file = "azure_mgmt_compute-30.5.0-py3-none-any.whl", hash = "sha256:b65a6c1e22be7334604257d8d9f96a9c6dc4c6d4869f95d0d551c7c8170a2e71"}, +] + +[package.dependencies] +azure-common = ">=1.1,<2.0" +azure-mgmt-core = ">=1.3.2,<2.0.0" +isodate = ">=0.6.1,<1.0.0" + [[package]] name = "azure-mgmt-containerservice" version = "26.0.0" @@ -1728,4 +1744,4 @@ test = ["websockets"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "6216898e620f01261302f5372da85cc395eb37dec1aab5ba6997b9fd5c5692c9" +content-hash = "ea4b350073272f85b4e0a7a302286f137dbab5594a7da1ec6d416d5989b2e93e" diff --git a/tools/pyproject.toml b/tools/pyproject.toml index 4de39882..6de5ffcf 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -29,6 +29,7 @@ azure-mgmt-privatedns = "^1.1.0" azure-mgmt-containerservice = "^26.0.0" azure-mgmt-storage = "^21.1.0" azure-mgmt-subscription = "^3.1.1" +azure-mgmt-compute = "^30.5.0" gcloud = "0.18.3" gitpython = "3.1.32" ghrepo = "0.7.0" From 1119fc8a3462645da945cbbf42bfd82eca944eab Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Fri, 2 Feb 2024 19:21:28 +0100 Subject: [PATCH 12/16] compact cluster autoscaling manifest and sa fix --- .../cluster-autoscaler/application.yaml | 150 +----------------- .../modules/cloud_azure/service_accounts.tf | 1 + 2 files changed, 7 insertions(+), 144 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml index 41d12226..c82c0669 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-autoscaler/application.yaml @@ -10,159 +10,65 @@ spec: source: repoURL: https://kubernetes.github.io/autoscaler chart: cluster-autoscaler - targetRevision: 9.34.0 + targetRevision: 9.34.1 helm: values: |- - ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - # affinity -- Affinity for pod assignment affinity: {} - - # additionalLabels -- Labels to add to each object of the chart. additionalLabels: {} - autoDiscovery: - # cloudProviders `aws`, `gce`, `azure`, `magnum`, `clusterapi` and `oci` are supported by auto-discovery at this time - # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup - - # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. - # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=azure`, using tags defined in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md#auto-discovery-setup. - # Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. - # Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. - # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. clusterName: - - # autoDiscovery.tags -- ASG tags to match, run through `tpl`. tags: - k8s.io/cluster-autoscaler/enabled - k8s.io/cluster-autoscaler/ - # - kubernetes.io/cluster/{{ .Values.autoDiscovery.clusterName }} - - # autoDiscovery.roles -- Magnum node group roles to match. roles: - worker - - # cloudProvider -- The cloud provider where the autoscaler runs. - # Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. - # `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. - # `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. cloudProvider: - # cloud provider specific configuration # # --- - - # containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) containerSecurityContext: capabilities: drop: - ALL - deployment: - # deployment.annotations -- Annotations to add to the Deployment object. annotations: {} - - # dnsPolicy -- Defaults to `ClusterFirst`. Valid values are: - # `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. - # If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. dnsPolicy: ClusterFirst - - # envFromConfigMap -- ConfigMap name to use as envFrom. envFromConfigMap: "" - - # envFromSecret -- Secret name to use as envFrom. envFromSecret: "" - - # extraArgs -- Additional container arguments. - # Refer to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca for the full list of cluster autoscaler - # parameters and their default values. - # Everything after the first _ will be ignored allowing the use of multi-string arguments. extraArgs: logtostderr: true stderrthreshold: info v: 4 - # write-status-configmap: true - # status-config-map-name: cluster-autoscaler-status - # leader-elect: true - # leader-elect-resource-lock: endpoints - # skip-nodes-with-local-storage: true - # expander: random - # scale-down-enabled: true - # balance-similar-node-groups: true - # min-replica-count: 0 - # scale-down-utilization-threshold: 0.5 - # scale-down-non-empty-candidates-count: 30 - # max-node-provision-time: 15m0s - # scan-interval: 10s - # scale-down-delay-after-add: 10m - # scale-down-delay-after-delete: 0s - # scale-down-delay-after-failure: 3m - # scale-down-unneeded-time: 10m - # skip-nodes-with-system-pods: true - # balancing-ignore-label_1: first-label-to-ignore - # balancing-ignore-label_2: second-label-to-ignore - - # hostNetwork -- Whether to expose network interfaces of the host machine to pods. hostNetwork: false - - # kwokConfigMapName -- configmap for configuring kwok provider kwokConfigMapName: "kwok-provider-config" - - # nodeSelector -- Node labels for pod assignment. Ref: https://kubernetes.io/docs/user-guide/node-selection/. nodeSelector: {} - - # podAnnotations -- Annotations to add to each pod. podAnnotations: {} - - # podDisruptionBudget -- Pod disruption budget. podDisruptionBudget: maxUnavailable: 1 # minAvailable: 2 - - # podLabels -- Labels to add to each pod. - podLabels: {} - - # priorityClassName -- priorityClassName + podLabels: { + # + } priorityClassName: "system-cluster-critical" - - ## Custom PrometheusRule to be defined - ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart - ## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions prometheusRule: - # prometheusRule.enabled -- If true, creates a Prometheus Operator PrometheusRule. enabled: true - # prometheusRule.additionalLabels -- Additional labels to be set in metadata. - additionalLabels: ClusterAutoscalerRules - # prometheusRule.namespace -- Namespace which Prometheus is running in. + additionalLabels: + ruleType: ClusterAutoscalerRules namespace: monitoring - # prometheusRule.interval -- How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). interval: null - # prometheusRule.rules -- Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). rules: [] - rbac: - # rbac.create -- If `true`, create and use RBAC resources. create: true - # rbac.pspEnabled -- If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. - # Must be used with `rbac.create` set to `true`. pspEnabled: false - # rbac.clusterScoped -- if set to false will only provision RBAC to alter resources in the current namespace. Most useful for Cluster-API clusterScoped: true serviceAccount: - # rbac.serviceAccount.annotations -- Additional Service Account annotations. annotations: : "" - # rbac.serviceAccount.create -- If `true` and `rbac.create` is also true, a Service Account will be created. create: true - # rbac.serviceAccount.name -- The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. name: cluster-autoscaler - # rbac.serviceAccount.automountServiceAccountToken -- Automount API credentials for a Service Account. automountServiceAccountToken: true - - # replicaCount -- Desired number of pods replicaCount: 1 - - # resources -- Pod resource requests and limits. resources: limits: cpu: 100m @@ -170,77 +76,33 @@ spec: requests: cpu: 100m memory: 300Mi - - # revisionHistoryLimit -- The number of revisions to keep. revisionHistoryLimit: 10 - - # securityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) securityContext: runAsNonRoot: true runAsUser: 1001 runAsGroup: 1001 - service: - # service.create -- If `true`, a Service will be created. create: true - # service.annotations -- Annotations to add to service annotations: {} - # service.labels -- Labels to add to service labels: {} - # service.externalIPs -- List of IP addresses at which the service is available. Ref: https://kubernetes.io/docs/user-guide/services/#external-ips. externalIPs: [] - - # service.loadBalancerIP -- IP address to assign to load balancer (if supported). loadBalancerIP: "" - # service.loadBalancerSourceRanges -- List of IP CIDRs allowed access to load balancer (if supported). loadBalancerSourceRanges: [] - # service.servicePort -- Service port to expose. servicePort: 8085 - # service.portName -- Name for service port. portName: http - # service.type -- Type of service to create. type: ClusterIP - - ## Are you using Prometheus Operator? serviceMonitor: - # serviceMonitor.enabled -- If true, creates a Prometheus Operator ServiceMonitor. enabled: true - # serviceMonitor.interval -- Interval that Prometheus scrapes Cluster Autoscaler metrics. interval: 10s - # serviceMonitor.namespace -- Namespace which Prometheus is running in. namespace: monitoring - ## [Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#prometheus-operator-1) - ## [Kube Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#exporters) - # serviceMonitor.selector -- Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install. selector: release: prometheus-operator - # serviceMonitor.path -- The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard) path: /metrics - # serviceMonitor.annotations -- Annotations to add to service monitor annotations: {} - ## [RelabelConfig](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig) - # serviceMonitor.metricRelabelings -- MetricRelabelConfigs to apply to samples before ingestion. metricRelabelings: {} - - # tolerations -- List of node taints to tolerate (requires Kubernetes >= 1.6). tolerations: [] - - # topologySpreadConstraints -- You can use topology spread constraints to control how Pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains. (requires Kubernetes >= 1.19). topologySpreadConstraints: [] - # - maxSkew: 1 - # topologyKey: topology.kubernetes.io/zone - # whenUnsatisfiable: DoNotSchedule - # labelSelector: - # matchLabels: - # app.kubernetes.io/instance: cluster-autoscaler - - # updateStrategy -- [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) updateStrategy: {} - # rollingUpdate: - # maxSurge: 1 - # maxUnavailable: 0 - # type: RollingUpdate - destination: server: https://kubernetes.default.svc namespace: cluster-autoscaler diff --git a/platform/terraform/modules/cloud_azure/service_accounts.tf b/platform/terraform/modules/cloud_azure/service_accounts.tf index a90ae900..03fe354f 100644 --- a/platform/terraform/modules/cloud_azure/service_accounts.tf +++ b/platform/terraform/modules/cloud_azure/service_accounts.tf @@ -78,6 +78,7 @@ module "cluster_autoscaler_sa" { oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url resource_group_name = azurerm_resource_group.rg.name + resource_group_location = azurerm_resource_group.rg.location name = "cluster-autoscaler" service_account_name = "cluster-autoscaler" role_definitions = [{ "name" = "Contributor", "scope" = "" }] From 25bffd24a184736f37eb0340b105f76ce85ca367 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Fri, 2 Feb 2024 19:24:10 +0100 Subject: [PATCH 13/16] fix --- tools/cli/services/cloud/aws/aws_manager.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index f1b17ed7..c2b226a1 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -170,6 +170,11 @@ def create_sidecar_annotation(self) -> str: def create_external_secrets_config(self, **kwargs) -> str: return "" + @trace() + def create_iac_pr_automation_config_snippet(self): + return '''# aws specific section + # ----''' + @trace() def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): return '''awsRegion: ''' From 315e07018ed95cf0ff9e5631c0c1dea042871273 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Fri, 2 Feb 2024 20:08:36 +0100 Subject: [PATCH 14/16] fix --- .../cli/services/cloud/azure/azure_manager.py | 6 +- tools/poetry.lock | 201 +++++++++--------- 2 files changed, 104 insertions(+), 103 deletions(-) diff --git a/tools/cli/services/cloud/azure/azure_manager.py b/tools/cli/services/cloud/azure/azure_manager.py index e5d1c26d..9c1598b6 100644 --- a/tools/cli/services/cloud/azure/azure_manager.py +++ b/tools/cli/services/cloud/azure/azure_manager.py @@ -241,7 +241,7 @@ def create_external_secrets_config(self, **kwargs) -> str: @trace() def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): autoscaling_groups = "" - vmss_list = self.__azure_sdk.get_vmss(f"{cluster_name}-vmss-rg") + vmss_list = self._azure_sdk.get_vmss(f"{cluster_name}-vmss-rg") if not len(vmss_list): raise Exception("Could not find vmss") @@ -253,11 +253,11 @@ def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): minSize: {node["min_size"]} maxSize: {node["max_size"]}''' - tenant_id = self.__azure_sdk.get_tenant_id() + tenant_id = self._azure_sdk.get_tenant_id() return f'''autoscalingGroups: {autoscaling_groups} azureClientID: "" azureResourceGroup: {cluster_name}-vmss-rg - azureSubscriptionID: {self.__azure_sdk.subscription_id} + azureSubscriptionID: {self._azure_sdk.subscription_id} azureTenantID: {tenant_id} azureUseWorkloadIdentityExtension: true azureVMType: "vmss"''' diff --git a/tools/poetry.lock b/tools/poetry.lock index 5aa9e49c..12ee5a3b 100644 --- a/tools/poetry.lock +++ b/tools/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. [[package]] name = "about-time" @@ -91,13 +91,13 @@ files = [ [[package]] name = "azure-core" -version = "1.29.7" +version = "1.30.0" description = "Microsoft Azure Core Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "azure-core-1.29.7.tar.gz", hash = "sha256:2944faf1a7ff1558b1f457cabf60f279869cabaeef86b353bed8eb032c7d8c5e"}, - {file = "azure_core-1.29.7-py3-none-any.whl", hash = "sha256:95a7b41b4af102e5fcdfac9500fcc82ff86e936c7145a099b7848b9ac0501250"}, + {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"}, + {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"}, ] [package.dependencies] @@ -338,13 +338,13 @@ files = [ [[package]] name = "certifi" -version = "2023.11.17" +version = "2024.2.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, - {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] [[package]] @@ -566,43 +566,43 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", [[package]] name = "cryptography" -version = "42.0.0" +version = "42.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:c640b0ef54138fde761ec99a6c7dc4ce05e80420262c20fa239e694ca371d434"}, - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:678cfa0d1e72ef41d48993a7be75a76b0725d29b820ff3cfd606a5b2b33fda01"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:146e971e92a6dd042214b537a726c9750496128453146ab0ee8971a0299dc9bd"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87086eae86a700307b544625e3ba11cc600c3c0ef8ab97b0fda0705d6db3d4e3"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0a68bfcf57a6887818307600c3c0ebc3f62fbb6ccad2240aa21887cda1f8df1b"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5a217bca51f3b91971400890905a9323ad805838ca3fa1e202a01844f485ee87"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ca20550bb590db16223eb9ccc5852335b48b8f597e2f6f0878bbfd9e7314eb17"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:33588310b5c886dfb87dba5f013b8d27df7ffd31dc753775342a1e5ab139e59d"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9515ea7f596c8092fdc9902627e51b23a75daa2c7815ed5aa8cf4f07469212ec"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:35cf6ed4c38f054478a9df14f03c1169bb14bd98f0b1705751079b25e1cb58bc"}, - {file = "cryptography-42.0.0-cp37-abi3-win32.whl", hash = "sha256:8814722cffcfd1fbd91edd9f3451b88a8f26a5fd41b28c1c9193949d1c689dc4"}, - {file = "cryptography-42.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:a2a8d873667e4fd2f34aedab02ba500b824692c6542e017075a2efc38f60a4c0"}, - {file = "cryptography-42.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:8fedec73d590fd30c4e3f0d0f4bc961aeca8390c72f3eaa1a0874d180e868ddf"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be41b0c7366e5549265adf2145135dca107718fa44b6e418dc7499cfff6b4689"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca482ea80626048975360c8e62be3ceb0f11803180b73163acd24bf014133a0"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c58115384bdcfe9c7f644c72f10f6f42bed7cf59f7b52fe1bf7ae0a622b3a139"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:56ce0c106d5c3fec1038c3cca3d55ac320a5be1b44bf15116732d0bc716979a2"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:324721d93b998cb7367f1e6897370644751e5580ff9b370c0a50dc60a2003513"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:d97aae66b7de41cdf5b12087b5509e4e9805ed6f562406dfcf60e8481a9a28f8"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:85f759ed59ffd1d0baad296e72780aa62ff8a71f94dc1ab340386a1207d0ea81"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:206aaf42e031b93f86ad60f9f5d9da1b09164f25488238ac1dc488334eb5e221"}, - {file = "cryptography-42.0.0-cp39-abi3-win32.whl", hash = "sha256:74f18a4c8ca04134d2052a140322002fef535c99cdbc2a6afc18a8024d5c9d5b"}, - {file = "cryptography-42.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:14e4b909373bc5bf1095311fa0f7fcabf2d1a160ca13f1e9e467be1ac4cbdf94"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3005166a39b70c8b94455fdbe78d87a444da31ff70de3331cdec2c568cf25b7e"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:be14b31eb3a293fc6e6aa2807c8a3224c71426f7c4e3639ccf1a2f3ffd6df8c3"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:bd7cf7a8d9f34cc67220f1195884151426ce616fdc8285df9054bfa10135925f"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c310767268d88803b653fffe6d6f2f17bb9d49ffceb8d70aed50ad45ea49ab08"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bdce70e562c69bb089523e75ef1d9625b7417c6297a76ac27b1b8b1eb51b7d0f"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e9326ca78111e4c645f7e49cbce4ed2f3f85e17b61a563328c85a5208cf34440"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:69fd009a325cad6fbfd5b04c711a4da563c6c4854fc4c9544bff3088387c77c0"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:988b738f56c665366b1e4bfd9045c3efae89ee366ca3839cd5af53eaa1401bce"}, - {file = "cryptography-42.0.0.tar.gz", hash = "sha256:6cf9b76d6e93c62114bd19485e5cb003115c134cf9ce91f8ac924c44f8c8c3f4"}, + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be"}, + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2"}, + {file = "cryptography-42.0.2-cp37-abi3-win32.whl", hash = "sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee"}, + {file = "cryptography-42.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee"}, + {file = "cryptography-42.0.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33"}, + {file = "cryptography-42.0.2-cp39-abi3-win32.whl", hash = "sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635"}, + {file = "cryptography-42.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65"}, + {file = "cryptography-42.0.2.tar.gz", hash = "sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888"}, ] [package.dependencies] @@ -817,13 +817,13 @@ gitdb = ">=4.0.1,<5" [[package]] name = "google-auth" -version = "2.26.2" +version = "2.27.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, - {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, ] [package.dependencies] @@ -917,69 +917,69 @@ oauth2client = ">=1.4.11" [[package]] name = "grpcio" -version = "1.60.0" +version = "1.60.1" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.60.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:d020cfa595d1f8f5c6b343530cd3ca16ae5aefdd1e832b777f9f0eb105f5b139"}, - {file = "grpcio-1.60.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b98f43fcdb16172dec5f4b49f2fece4b16a99fd284d81c6bbac1b3b69fcbe0ff"}, - {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:20e7a4f7ded59097c84059d28230907cd97130fa74f4a8bfd1d8e5ba18c81491"}, - {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452ca5b4afed30e7274445dd9b441a35ece656ec1600b77fff8c216fdf07df43"}, - {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43e636dc2ce9ece583b3e2ca41df5c983f4302eabc6d5f9cd04f0562ee8ec1ae"}, - {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e306b97966369b889985a562ede9d99180def39ad42c8014628dd3cc343f508"}, - {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f897c3b127532e6befdcf961c415c97f320d45614daf84deba0a54e64ea2457b"}, - {file = "grpcio-1.60.0-cp310-cp310-win32.whl", hash = "sha256:b87efe4a380887425bb15f220079aa8336276398dc33fce38c64d278164f963d"}, - {file = "grpcio-1.60.0-cp310-cp310-win_amd64.whl", hash = "sha256:a9c7b71211f066908e518a2ef7a5e211670761651039f0d6a80d8d40054047df"}, - {file = "grpcio-1.60.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:fb464479934778d7cc5baf463d959d361954d6533ad34c3a4f1d267e86ee25fd"}, - {file = "grpcio-1.60.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4b44d7e39964e808b071714666a812049765b26b3ea48c4434a3b317bac82f14"}, - {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:90bdd76b3f04bdb21de5398b8a7c629676c81dfac290f5f19883857e9371d28c"}, - {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91229d7203f1ef0ab420c9b53fe2ca5c1fbeb34f69b3bc1b5089466237a4a134"}, - {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b36a2c6d4920ba88fa98075fdd58ff94ebeb8acc1215ae07d01a418af4c0253"}, - {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:297eef542156d6b15174a1231c2493ea9ea54af8d016b8ca7d5d9cc65cfcc444"}, - {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:87c9224acba0ad8bacddf427a1c2772e17ce50b3042a789547af27099c5f751d"}, - {file = "grpcio-1.60.0-cp311-cp311-win32.whl", hash = "sha256:95ae3e8e2c1b9bf671817f86f155c5da7d49a2289c5cf27a319458c3e025c320"}, - {file = "grpcio-1.60.0-cp311-cp311-win_amd64.whl", hash = "sha256:467a7d31554892eed2aa6c2d47ded1079fc40ea0b9601d9f79204afa8902274b"}, - {file = "grpcio-1.60.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:a7152fa6e597c20cb97923407cf0934e14224af42c2b8d915f48bc3ad2d9ac18"}, - {file = "grpcio-1.60.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:7db16dd4ea1b05ada504f08d0dca1cd9b926bed3770f50e715d087c6f00ad748"}, - {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:b0571a5aef36ba9177e262dc88a9240c866d903a62799e44fd4aae3f9a2ec17e"}, - {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fd9584bf1bccdfff1512719316efa77be235469e1e3295dce64538c4773840b"}, - {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6a478581b1a1a8fdf3318ecb5f4d0cda41cacdffe2b527c23707c9c1b8fdb55"}, - {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:77c8a317f0fd5a0a2be8ed5cbe5341537d5c00bb79b3bb27ba7c5378ba77dbca"}, - {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1c30bb23a41df95109db130a6cc1b974844300ae2e5d68dd4947aacba5985aa5"}, - {file = "grpcio-1.60.0-cp312-cp312-win32.whl", hash = "sha256:2aef56e85901c2397bd557c5ba514f84de1f0ae5dd132f5d5fed042858115951"}, - {file = "grpcio-1.60.0-cp312-cp312-win_amd64.whl", hash = "sha256:e381fe0c2aa6c03b056ad8f52f8efca7be29fb4d9ae2f8873520843b6039612a"}, - {file = "grpcio-1.60.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:92f88ca1b956eb8427a11bb8b4a0c0b2b03377235fc5102cb05e533b8693a415"}, - {file = "grpcio-1.60.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:e278eafb406f7e1b1b637c2cf51d3ad45883bb5bd1ca56bc05e4fc135dfdaa65"}, - {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:a48edde788b99214613e440fce495bbe2b1e142a7f214cce9e0832146c41e324"}, - {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de2ad69c9a094bf37c1102b5744c9aec6cf74d2b635558b779085d0263166454"}, - {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:073f959c6f570797272f4ee9464a9997eaf1e98c27cb680225b82b53390d61e6"}, - {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c826f93050c73e7769806f92e601e0efdb83ec8d7c76ddf45d514fee54e8e619"}, - {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9e30be89a75ee66aec7f9e60086fadb37ff8c0ba49a022887c28c134341f7179"}, - {file = "grpcio-1.60.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b0fb2d4801546598ac5cd18e3ec79c1a9af8b8f2a86283c55a5337c5aeca4b1b"}, - {file = "grpcio-1.60.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:9073513ec380434eb8d21970e1ab3161041de121f4018bbed3146839451a6d8e"}, - {file = "grpcio-1.60.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:74d7d9fa97809c5b892449b28a65ec2bfa458a4735ddad46074f9f7d9550ad13"}, - {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:1434ca77d6fed4ea312901122dc8da6c4389738bf5788f43efb19a838ac03ead"}, - {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e61e76020e0c332a98290323ecfec721c9544f5b739fab925b6e8cbe1944cf19"}, - {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675997222f2e2f22928fbba640824aebd43791116034f62006e19730715166c0"}, - {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5208a57eae445ae84a219dfd8b56e04313445d146873117b5fa75f3245bc1390"}, - {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:428d699c8553c27e98f4d29fdc0f0edc50e9a8a7590bfd294d2edb0da7be3629"}, - {file = "grpcio-1.60.0-cp38-cp38-win32.whl", hash = "sha256:83f2292ae292ed5a47cdcb9821039ca8e88902923198f2193f13959360c01860"}, - {file = "grpcio-1.60.0-cp38-cp38-win_amd64.whl", hash = "sha256:705a68a973c4c76db5d369ed573fec3367d7d196673fa86614b33d8c8e9ebb08"}, - {file = "grpcio-1.60.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c193109ca4070cdcaa6eff00fdb5a56233dc7610216d58fb81638f89f02e4968"}, - {file = "grpcio-1.60.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:676e4a44e740deaba0f4d95ba1d8c5c89a2fcc43d02c39f69450b1fa19d39590"}, - {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5ff21e000ff2f658430bde5288cb1ac440ff15c0d7d18b5fb222f941b46cb0d2"}, - {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c86343cf9ff7b2514dd229bdd88ebba760bd8973dac192ae687ff75e39ebfab"}, - {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fd3b3968ffe7643144580f260f04d39d869fcc2cddb745deef078b09fd2b328"}, - {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:30943b9530fe3620e3b195c03130396cd0ee3a0d10a66c1bee715d1819001eaf"}, - {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b10241250cb77657ab315270b064a6c7f1add58af94befa20687e7c8d8603ae6"}, - {file = "grpcio-1.60.0-cp39-cp39-win32.whl", hash = "sha256:79a050889eb8d57a93ed21d9585bb63fca881666fc709f5d9f7f9372f5e7fd03"}, - {file = "grpcio-1.60.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a97a681e82bc11a42d4372fe57898d270a2707f36c45c6676e49ce0d5c41353"}, - {file = "grpcio-1.60.0.tar.gz", hash = "sha256:2199165a1affb666aa24adf0c97436686d0a61bc5fc113c037701fb7c7fceb96"}, + {file = "grpcio-1.60.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:14e8f2c84c0832773fb3958240c69def72357bc11392571f87b2d7b91e0bb092"}, + {file = "grpcio-1.60.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:33aed0a431f5befeffd9d346b0fa44b2c01aa4aeae5ea5b2c03d3e25e0071216"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:fead980fbc68512dfd4e0c7b1f5754c2a8e5015a04dea454b9cada54a8423525"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:082081e6a36b6eb5cf0fd9a897fe777dbb3802176ffd08e3ec6567edd85bc104"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55ccb7db5a665079d68b5c7c86359ebd5ebf31a19bc1a91c982fd622f1e31ff2"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b54577032d4f235452f77a83169b6527bf4b77d73aeada97d45b2aaf1bf5ce0"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7d142bcd604166417929b071cd396aa13c565749a4c840d6c702727a59d835eb"}, + {file = "grpcio-1.60.1-cp310-cp310-win32.whl", hash = "sha256:2a6087f234cb570008a6041c8ffd1b7d657b397fdd6d26e83d72283dae3527b1"}, + {file = "grpcio-1.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:f2212796593ad1d0235068c79836861f2201fc7137a99aa2fea7beeb3b101177"}, + {file = "grpcio-1.60.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:79ae0dc785504cb1e1788758c588c711f4e4a0195d70dff53db203c95a0bd303"}, + {file = "grpcio-1.60.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4eec8b8c1c2c9b7125508ff7c89d5701bf933c99d3910e446ed531cd16ad5d87"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8c9554ca8e26241dabe7951aa1fa03a1ba0856688ecd7e7bdbdd286ebc272e4c"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91422ba785a8e7a18725b1dc40fbd88f08a5bb4c7f1b3e8739cab24b04fa8a03"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cba6209c96828711cb7c8fcb45ecef8c8859238baf15119daa1bef0f6c84bfe7"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c71be3f86d67d8d1311c6076a4ba3b75ba5703c0b856b4e691c9097f9b1e8bd2"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5ef6cfaf0d023c00002ba25d0751e5995fa0e4c9eec6cd263c30352662cbce"}, + {file = "grpcio-1.60.1-cp311-cp311-win32.whl", hash = "sha256:a09506eb48fa5493c58f946c46754ef22f3ec0df64f2b5149373ff31fb67f3dd"}, + {file = "grpcio-1.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:49c9b6a510e3ed8df5f6f4f3c34d7fbf2d2cae048ee90a45cd7415abab72912c"}, + {file = "grpcio-1.60.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:b58b855d0071575ea9c7bc0d84a06d2edfbfccec52e9657864386381a7ce1ae9"}, + {file = "grpcio-1.60.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:a731ac5cffc34dac62053e0da90f0c0b8560396a19f69d9703e88240c8f05858"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:cf77f8cf2a651fbd869fbdcb4a1931464189cd210abc4cfad357f1cacc8642a6"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c557e94e91a983e5b1e9c60076a8fd79fea1e7e06848eb2e48d0ccfb30f6e073"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:069fe2aeee02dfd2135d562d0663fe70fbb69d5eed6eb3389042a7e963b54de8"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb0af13433dbbd1c806e671d81ec75bd324af6ef75171fd7815ca3074fe32bfe"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2f44c32aef186bbba254129cea1df08a20be414144ac3bdf0e84b24e3f3b2e05"}, + {file = "grpcio-1.60.1-cp312-cp312-win32.whl", hash = "sha256:a212e5dea1a4182e40cd3e4067ee46be9d10418092ce3627475e995cca95de21"}, + {file = "grpcio-1.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:6e490fa5f7f5326222cb9f0b78f207a2b218a14edf39602e083d5f617354306f"}, + {file = "grpcio-1.60.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:4216e67ad9a4769117433814956031cb300f85edc855252a645a9a724b3b6594"}, + {file = "grpcio-1.60.1-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:73e14acd3d4247169955fae8fb103a2b900cfad21d0c35f0dcd0fdd54cd60367"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6ecf21d20d02d1733e9c820fb5c114c749d888704a7ec824b545c12e78734d1c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33bdea30dcfd4f87b045d404388469eb48a48c33a6195a043d116ed1b9a0196c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b69e79d00f78c81eecfb38f4516080dc7f36a198b6b37b928f1c13b3c063e9"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:39aa848794b887120b1d35b1b994e445cc028ff602ef267f87c38122c1add50d"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:72153a0d2e425f45b884540a61c6639436ddafa1829a42056aa5764b84108b8e"}, + {file = "grpcio-1.60.1-cp37-cp37m-win_amd64.whl", hash = "sha256:50d56280b482875d1f9128ce596e59031a226a8b84bec88cb2bf76c289f5d0de"}, + {file = "grpcio-1.60.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6d140bdeb26cad8b93c1455fa00573c05592793c32053d6e0016ce05ba267549"}, + {file = "grpcio-1.60.1-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:bc808924470643b82b14fe121923c30ec211d8c693e747eba8a7414bc4351a23"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:70c83bb530572917be20c21f3b6be92cd86b9aecb44b0c18b1d3b2cc3ae47df0"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b106bc52e7f28170e624ba61cc7dc6829566e535a6ec68528f8e1afbed1c41f"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e980cd6db1088c144b92fe376747328d5554bc7960ce583ec7b7d81cd47287"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0c5807e9152eff15f1d48f6b9ad3749196f79a4a050469d99eecb679be592acc"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1c3dc536b3ee124e8b24feb7533e5c70b9f2ef833e3b2e5513b2897fd46763a"}, + {file = "grpcio-1.60.1-cp38-cp38-win32.whl", hash = "sha256:d7404cebcdb11bb5bd40bf94131faf7e9a7c10a6c60358580fe83913f360f929"}, + {file = "grpcio-1.60.1-cp38-cp38-win_amd64.whl", hash = "sha256:c8754c75f55781515a3005063d9a05878b2cfb3cb7e41d5401ad0cf19de14872"}, + {file = "grpcio-1.60.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:0250a7a70b14000fa311de04b169cc7480be6c1a769b190769d347939d3232a8"}, + {file = "grpcio-1.60.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:660fc6b9c2a9ea3bb2a7e64ba878c98339abaf1811edca904ac85e9e662f1d73"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76eaaba891083fcbe167aa0f03363311a9f12da975b025d30e94b93ac7a765fc"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d97c65ea7e097056f3d1ead77040ebc236feaf7f71489383d20f3b4c28412a"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb2a2911b028f01c8c64d126f6b632fcd8a9ac975aa1b3855766c94e4107180"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5a1ebbae7e2214f51b1f23b57bf98eeed2cf1ba84e4d523c48c36d5b2f8829ff"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a66f4d2a005bc78e61d805ed95dedfcb35efa84b7bba0403c6d60d13a3de2d6"}, + {file = "grpcio-1.60.1-cp39-cp39-win32.whl", hash = "sha256:8d488fbdbf04283f0d20742b64968d44825617aa6717b07c006168ed16488804"}, + {file = "grpcio-1.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b7199cd2a55e62e45bfb629a35b71fc2c0cb88f686a047f25b1112d3810904"}, + {file = "grpcio-1.60.1.tar.gz", hash = "sha256:dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.60.0)"] +protobuf = ["grpcio-tools (>=1.60.1)"] [[package]] name = "h11" @@ -1587,6 +1587,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -1797,4 +1798,4 @@ test = ["websockets"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "44623f76e2e6f9835eb3b617629847f8f1cf152bd9caab8c35e03e5a768f4d53" +content-hash = "55fab4364d2c92dc9780bb69dec873dec23293fcdfd06e83a9b6d11833c20585" From 3855021b164e51a50082358cd3b7c3bb2789b074 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Fri, 2 Feb 2024 23:59:11 +0100 Subject: [PATCH 15/16] add _ fix --- .../cli/services/cloud/azure/azure_manager.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/cli/services/cloud/azure/azure_manager.py b/tools/cli/services/cloud/azure/azure_manager.py index 9c1598b6..6ffc2509 100644 --- a/tools/cli/services/cloud/azure/azure_manager.py +++ b/tools/cli/services/cloud/azure/azure_manager.py @@ -20,11 +20,11 @@ def __init__( self, subscription_id: str, location: Optional[str] = None, storage_container_name: Optional[str] = None ): self.iac_backend_storage_container_name: Optional[str] = storage_container_name - self._azure_sdk = AzureSdk(subscription_id, location) + self.__azure_sdk = AzureSdk(subscription_id, location) @property def region(self): - return self._azure_sdk.location + return self.__azure_sdk.location @trace() def protect_iac_state_storage(self, name: str, identity: str): @@ -34,7 +34,7 @@ def protect_iac_state_storage(self, name: str, identity: str): self.iac_backend_storage_container_name = name resource_group_name = self._generate_resource_group_name() storage_account_name = self._generate_storage_account_name() - self._azure_sdk.set_storage_access(identity, storage_account_name, resource_group_name) + self.__azure_sdk.set_storage_access(identity, storage_account_name, resource_group_name) @trace() def destroy_iac_state_storage(self, bucket: str) -> bool: @@ -53,7 +53,7 @@ def destroy_iac_state_storage(self, bucket: str) -> bool: bool: True if the resource group was successfully destroyed, False otherwise. """ self.iac_backend_storage_container_name = bucket - return self._azure_sdk.destroy_resource_group(self._generate_resource_group_name()) + return self.__azure_sdk.destroy_resource_group(self._generate_resource_group_name()) @trace() def create_iac_backend_snippet(self, location: str, service: str, **kwargs) -> str: @@ -138,13 +138,13 @@ def create_iac_state_storage(self, name: str, **kwargs: dict) -> Tuple[str, str] resource_group_name = self._generate_resource_group_name() storage_account_name = self._generate_storage_account_name() - self._azure_sdk.create_storage( + self.__azure_sdk.create_storage( container_name=self.iac_backend_storage_container_name, storage_account_name=storage_account_name, resource_group_name=resource_group_name) - keys = self._azure_sdk.get_storage_account_keys(resource_group_name, storage_account_name) - self._azure_sdk.set_storage_account_versioning(storage_account_name, resource_group_name) + keys = self.__azure_sdk.get_storage_account_keys(resource_group_name, storage_account_name) + self.__azure_sdk.set_storage_account_versioning(storage_account_name, resource_group_name) return self.iac_backend_storage_container_name, keys[0].value @@ -155,10 +155,10 @@ def evaluate_permissions(self) -> bool: :return: True or False """ missing_permissions = [] - missing_permissions.extend(self._azure_sdk.blocked(aks_permissions)) - missing_permissions.extend(self._azure_sdk.blocked(blob_permissions)) - missing_permissions.extend(self._azure_sdk.blocked(vnet_permissions)) - missing_permissions.extend(self._azure_sdk.blocked(rbac_permissions)) + missing_permissions.extend(self.__azure_sdk.blocked(aks_permissions)) + missing_permissions.extend(self.__azure_sdk.blocked(blob_permissions)) + missing_permissions.extend(self.__azure_sdk.blocked(vnet_permissions)) + missing_permissions.extend(self.__azure_sdk.blocked(rbac_permissions)) return len(missing_permissions) == 0 @staticmethod @@ -236,12 +236,12 @@ def create_external_secrets_config(self, **kwargs) -> str: "subscriptionId": "{subscription_id}", "resourceGroup": "{resource_group}", "useWorkloadIdentityExtension": true - }}'''.format(subscription_id=self._azure_sdk.subscription_id, resource_group=location) + }}'''.format(subscription_id=self.__azure_sdk.subscription_id, resource_group=location) @trace() def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): autoscaling_groups = "" - vmss_list = self._azure_sdk.get_vmss(f"{cluster_name}-vmss-rg") + vmss_list = self.__azure_sdk.get_vmss(f"{cluster_name}-vmss-rg") if not len(vmss_list): raise Exception("Could not find vmss") @@ -253,11 +253,11 @@ def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): minSize: {node["min_size"]} maxSize: {node["max_size"]}''' - tenant_id = self._azure_sdk.get_tenant_id() + tenant_id = self.__azure_sdk.get_tenant_id() return f'''autoscalingGroups: {autoscaling_groups} azureClientID: "" azureResourceGroup: {cluster_name}-vmss-rg - azureSubscriptionID: {self._azure_sdk.subscription_id} + azureSubscriptionID: {self.__azure_sdk.subscription_id} azureTenantID: {tenant_id} azureUseWorkloadIdentityExtension: true azureVMType: "vmss"''' From 3024fd0cc82cd1049e93783f1d4c19e9567da271 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Mon, 12 Feb 2024 15:23:55 +0100 Subject: [PATCH 16/16] refactor: minor changes requested during PR review --- .../terraform/modules/cloud_azure/main.tf | 2 +- .../modules/cloud_azure/service_accounts.tf | 12 ++++---- .../modules/cloud_azure/variables.tf | 6 ++++ tools/cli/services/cloud/aws/aws_manager.py | 24 +++++++-------- .../cli/services/cloud/azure/azure_manager.py | 30 +++++++++---------- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/platform/terraform/modules/cloud_azure/main.tf b/platform/terraform/modules/cloud_azure/main.tf index 7e72bac3..aa79fcd6 100644 --- a/platform/terraform/modules/cloud_azure/main.tf +++ b/platform/terraform/modules/cloud_azure/main.tf @@ -23,7 +23,7 @@ locals { additional_node_pools = try(slice(var.node_groups, 1, length(var.node_groups)), []) max_pods = 100 node_admin_username = "azadmin" - enable_native_auto_scaling = false + enable_native_auto_scaling = var.enable_native_auto_scaling } resource "azurerm_resource_group" "rg" { diff --git a/platform/terraform/modules/cloud_azure/service_accounts.tf b/platform/terraform/modules/cloud_azure/service_accounts.tf index 03fe354f..8babc6e8 100644 --- a/platform/terraform/modules/cloud_azure/service_accounts.tf +++ b/platform/terraform/modules/cloud_azure/service_accounts.tf @@ -76,13 +76,13 @@ module "secret_manager_sa" { module "cluster_autoscaler_sa" { source = "./modules/aks_rbac" - oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - resource_group_name = azurerm_resource_group.rg.name + oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + resource_group_name = azurerm_resource_group.rg.name resource_group_location = azurerm_resource_group.rg.location - name = "cluster-autoscaler" - service_account_name = "cluster-autoscaler" - role_definitions = [{ "name" = "Contributor", "scope" = "" }] - namespace = "cluster-autoscaler" + name = "cluster-autoscaler" + service_account_name = "cluster-autoscaler" + role_definitions = [{ "name" = "Contributor", "scope" = "" }] + namespace = "cluster-autoscaler" depends_on = [azurerm_kubernetes_cluster.aks_cluster] } \ No newline at end of file diff --git a/platform/terraform/modules/cloud_azure/variables.tf b/platform/terraform/modules/cloud_azure/variables.tf index f9a6fe36..66e2edc1 100644 --- a/platform/terraform/modules/cloud_azure/variables.tf +++ b/platform/terraform/modules/cloud_azure/variables.tf @@ -85,3 +85,9 @@ variable "cluster_ssh_public_key" { type = string default = "" } + +variable "enable_native_auto_scaling" { + description = "Enables AKS native autoscaling feature." + type = bool + default = false +} diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index c2b226a1..3730e36e 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -16,11 +16,11 @@ class AWSManager(CloudProviderManager): """AWS wrapper.""" def __init__(self, region, profile, key, secret): - self.__aws_sdk = AwsSdk(region, profile, key, secret) + self._aws_sdk = AwsSdk(region, profile, key, secret) @property def region(self): - return self.__aws_sdk.region + return self._aws_sdk.region @classmethod def detect_cli_presence(cls) -> bool: @@ -49,8 +49,8 @@ def create_iac_state_storage(self, name: str, **kwargs: dict) -> Tuple[str, str] region = kwargs["region"] tf_backend_storage_name = f'{name}-{random_string_generator()}'.lower() - self.__aws_sdk.create_bucket(tf_backend_storage_name, region) - self.__aws_sdk.enable_bucket_versioning(tf_backend_storage_name, region) + self._aws_sdk.create_bucket(tf_backend_storage_name, region) + self._aws_sdk.enable_bucket_versioning(tf_backend_storage_name, region) return tf_backend_storage_name, "" @@ -60,14 +60,14 @@ def protect_iac_state_storage(self, name: str, identity: str, **kwargs: dict): if kwargs and "region" in kwargs: region = kwargs["region"] - self.__aws_sdk.set_bucket_policy(name, identity, region) + self._aws_sdk.set_bucket_policy(name, identity, region) @trace() def destroy_iac_state_storage(self, name: str) -> bool: """ Destroy cloud native terraform remote state storage """ - return self.__aws_sdk.delete_bucket(name) + return self._aws_sdk.delete_bucket(name) @trace() def create_iac_backend_snippet(self, location: str, service: str, **kwargs: dict) -> str: @@ -136,7 +136,7 @@ def get_k8s_auth_command(self) -> tuple[str, [str]]: @trace() def get_k8s_token(self, cluster_name: str) -> str: - token = self.__aws_sdk.get_token(cluster_name=cluster_name) + token = self._aws_sdk.get_token(cluster_name=cluster_name) return token['status']['token'] @trace() @@ -146,11 +146,11 @@ def evaluate_permissions(self) -> bool: :return: True or False """ missing_permissions = [] - missing_permissions.extend(self.__aws_sdk.blocked(vpc_permissions)) - missing_permissions.extend(self.__aws_sdk.blocked(eks_permissions)) - missing_permissions.extend(self.__aws_sdk.blocked(iam_permissions)) - missing_permissions.extend(self.__aws_sdk.blocked(s3_permissions)) - missing_permissions.extend(self.__aws_sdk.blocked(own_iam_permissions, [self.__aws_sdk.current_user_arn()])) + missing_permissions.extend(self._aws_sdk.blocked(vpc_permissions)) + missing_permissions.extend(self._aws_sdk.blocked(eks_permissions)) + missing_permissions.extend(self._aws_sdk.blocked(iam_permissions)) + missing_permissions.extend(self._aws_sdk.blocked(s3_permissions)) + missing_permissions.extend(self._aws_sdk.blocked(own_iam_permissions, [self._aws_sdk.current_user_arn()])) return len(missing_permissions) == 0 @trace() diff --git a/tools/cli/services/cloud/azure/azure_manager.py b/tools/cli/services/cloud/azure/azure_manager.py index 6ffc2509..9c1598b6 100644 --- a/tools/cli/services/cloud/azure/azure_manager.py +++ b/tools/cli/services/cloud/azure/azure_manager.py @@ -20,11 +20,11 @@ def __init__( self, subscription_id: str, location: Optional[str] = None, storage_container_name: Optional[str] = None ): self.iac_backend_storage_container_name: Optional[str] = storage_container_name - self.__azure_sdk = AzureSdk(subscription_id, location) + self._azure_sdk = AzureSdk(subscription_id, location) @property def region(self): - return self.__azure_sdk.location + return self._azure_sdk.location @trace() def protect_iac_state_storage(self, name: str, identity: str): @@ -34,7 +34,7 @@ def protect_iac_state_storage(self, name: str, identity: str): self.iac_backend_storage_container_name = name resource_group_name = self._generate_resource_group_name() storage_account_name = self._generate_storage_account_name() - self.__azure_sdk.set_storage_access(identity, storage_account_name, resource_group_name) + self._azure_sdk.set_storage_access(identity, storage_account_name, resource_group_name) @trace() def destroy_iac_state_storage(self, bucket: str) -> bool: @@ -53,7 +53,7 @@ def destroy_iac_state_storage(self, bucket: str) -> bool: bool: True if the resource group was successfully destroyed, False otherwise. """ self.iac_backend_storage_container_name = bucket - return self.__azure_sdk.destroy_resource_group(self._generate_resource_group_name()) + return self._azure_sdk.destroy_resource_group(self._generate_resource_group_name()) @trace() def create_iac_backend_snippet(self, location: str, service: str, **kwargs) -> str: @@ -138,13 +138,13 @@ def create_iac_state_storage(self, name: str, **kwargs: dict) -> Tuple[str, str] resource_group_name = self._generate_resource_group_name() storage_account_name = self._generate_storage_account_name() - self.__azure_sdk.create_storage( + self._azure_sdk.create_storage( container_name=self.iac_backend_storage_container_name, storage_account_name=storage_account_name, resource_group_name=resource_group_name) - keys = self.__azure_sdk.get_storage_account_keys(resource_group_name, storage_account_name) - self.__azure_sdk.set_storage_account_versioning(storage_account_name, resource_group_name) + keys = self._azure_sdk.get_storage_account_keys(resource_group_name, storage_account_name) + self._azure_sdk.set_storage_account_versioning(storage_account_name, resource_group_name) return self.iac_backend_storage_container_name, keys[0].value @@ -155,10 +155,10 @@ def evaluate_permissions(self) -> bool: :return: True or False """ missing_permissions = [] - missing_permissions.extend(self.__azure_sdk.blocked(aks_permissions)) - missing_permissions.extend(self.__azure_sdk.blocked(blob_permissions)) - missing_permissions.extend(self.__azure_sdk.blocked(vnet_permissions)) - missing_permissions.extend(self.__azure_sdk.blocked(rbac_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(aks_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(blob_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(vnet_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(rbac_permissions)) return len(missing_permissions) == 0 @staticmethod @@ -236,12 +236,12 @@ def create_external_secrets_config(self, **kwargs) -> str: "subscriptionId": "{subscription_id}", "resourceGroup": "{resource_group}", "useWorkloadIdentityExtension": true - }}'''.format(subscription_id=self.__azure_sdk.subscription_id, resource_group=location) + }}'''.format(subscription_id=self._azure_sdk.subscription_id, resource_group=location) @trace() def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): autoscaling_groups = "" - vmss_list = self.__azure_sdk.get_vmss(f"{cluster_name}-vmss-rg") + vmss_list = self._azure_sdk.get_vmss(f"{cluster_name}-vmss-rg") if not len(vmss_list): raise Exception("Could not find vmss") @@ -253,11 +253,11 @@ def create_autoscaler_snippet(self, cluster_name: str, node_groups=[]): minSize: {node["min_size"]} maxSize: {node["max_size"]}''' - tenant_id = self.__azure_sdk.get_tenant_id() + tenant_id = self._azure_sdk.get_tenant_id() return f'''autoscalingGroups: {autoscaling_groups} azureClientID: "" azureResourceGroup: {cluster_name}-vmss-rg - azureSubscriptionID: {self.__azure_sdk.subscription_id} + azureSubscriptionID: {self._azure_sdk.subscription_id} azureTenantID: {tenant_id} azureUseWorkloadIdentityExtension: true azureVMType: "vmss"'''