From 5f8ba6cc4ca92f1e21cc6130a41f02b0ebcc1f9f Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Thu, 30 Nov 2023 12:48:18 +0100 Subject: [PATCH 01/49] feat: workload management "create workload" first draft --- platform/terraform/core_services/main.tf | 8 +- .../core_services/terraform.tfvars.json | 3 + platform/terraform/core_services/variables.tf | 8 ++ .../modules/vcs_github/repository/main.tf | 1 - .../modules/vcs_github/repository/variable.tf | 5 - .../terraform/modules/vcs_github/variable.tf | 26 ++-- .../modules/vcs_github/workload_repos.tf | 13 +- platform/terraform/secrets/main.tf | 7 +- .../terraform/secrets/terraform.tfvars.json | 3 + platform/terraform/secrets/variable.tf | 8 ++ platform/terraform/vcs/main.tf | 17 +-- platform/terraform/vcs/terraform.tfvars.json | 23 ++++ platform/terraform/vcs/variable.tf | 17 +++ tools/cli/commands/workload/create.py | 119 +++++++++++++++++- tools/cli/services/template_manager.py | 2 +- 15 files changed, 204 insertions(+), 56 deletions(-) create mode 100644 platform/terraform/core_services/terraform.tfvars.json create mode 100644 platform/terraform/secrets/terraform.tfvars.json create mode 100644 platform/terraform/vcs/terraform.tfvars.json diff --git a/platform/terraform/core_services/main.tf b/platform/terraform/core_services/main.tf index e00af787..3bab879f 100644 --- a/platform/terraform/core_services/main.tf +++ b/platform/terraform/core_services/main.tf @@ -42,15 +42,11 @@ provider "restapi" { locals { oidc_endpoint = "https://" code_quality_url = "https://" - workloads = { - # "workload-demo" = { - # }, - } } module "registry" { source = "../modules/registry_harbor" - workloads = local.workloads + workloads = var.workloads oidc_endpoint = local.oidc_endpoint oidc_client_id = var.registry_oidc_client_id oidc_client_secret = var.registry_oidc_client_secret @@ -59,7 +55,7 @@ module "registry" { module "code_quality" { source = "../modules/code_quality_sonarqube" - workloads = local.workloads + workloads = var.workloads oidc_endpoint = local.oidc_endpoint code_quality_url = local.code_quality_url oidc_client_id = var.code_quality_oidc_client_id diff --git a/platform/terraform/core_services/terraform.tfvars.json b/platform/terraform/core_services/terraform.tfvars.json new file mode 100644 index 00000000..56c2a5d3 --- /dev/null +++ b/platform/terraform/core_services/terraform.tfvars.json @@ -0,0 +1,3 @@ +{ + "workloads": {} +} \ No newline at end of file diff --git a/platform/terraform/core_services/variables.tf b/platform/terraform/core_services/variables.tf index 45cf5d83..cf1caea2 100644 --- a/platform/terraform/core_services/variables.tf +++ b/platform/terraform/core_services/variables.tf @@ -25,3 +25,11 @@ variable "registry_main_robot_password" { type = string sensitive = true } + +variable "workloads" { + description = "Workloads configuration" + type = map(object({ + description = optional(string, "") + })) + default = {} +} diff --git a/platform/terraform/modules/vcs_github/repository/main.tf b/platform/terraform/modules/vcs_github/repository/main.tf index 7729bbf6..c4d3a43a 100644 --- a/platform/terraform/modules/vcs_github/repository/main.tf +++ b/platform/terraform/modules/vcs_github/repository/main.tf @@ -6,7 +6,6 @@ resource "github_repository" "repo" { auto_init = var.auto_init archive_on_destroy = var.archive_on_destroy has_issues = var.has_issues - is_template = var.is_template delete_branch_on_merge = var.delete_branch_on_merge dynamic "template" { diff --git a/platform/terraform/modules/vcs_github/repository/variable.tf b/platform/terraform/modules/vcs_github/repository/variable.tf index 60dabc6d..7c489265 100644 --- a/platform/terraform/modules/vcs_github/repository/variable.tf +++ b/platform/terraform/modules/vcs_github/repository/variable.tf @@ -27,11 +27,6 @@ variable "has_issues" { default = false } -variable "is_template" { - type = bool - default = false -} - variable "default_branch_name" { type = string default = "main" diff --git a/platform/terraform/modules/vcs_github/variable.tf b/platform/terraform/modules/vcs_github/variable.tf index 9409b318..cb25fafc 100644 --- a/platform/terraform/modules/vcs_github/variable.tf +++ b/platform/terraform/modules/vcs_github/variable.tf @@ -19,21 +19,19 @@ variable "vcs_bot_ssh_public_key" { default = "" } -variable "workload_repos" { - description = "workloads repos configuration" +variable "workloads" { + description = "workloads configuration" type = map(object({ - description = optional(string, "") - visibility = optional(string, "private") - auto_init = optional(bool, false) - archive_on_destroy = optional(bool, false) - has_issues = optional(bool, false) - is_template = optional(bool, false) - default_branch_name = optional(string, "main") - delete_branch_on_merge = optional(bool, true) - template = optional(map(string), {}) - atlantis_enabled = optional(bool, false) - atlantis_url = optional(string, "") - atlantis_repo_webhook_secret = optional(string, "") + description = optional(string, "") + repos = map(object({ + visibility = optional(string, "private") + auto_init = optional(bool, false) + archive_on_destroy = optional(bool, false) + has_issues = optional(bool, false) + default_branch_name = optional(string, "main") + delete_branch_on_merge = optional(bool, true) + atlantis_enabled = optional(bool, false) + })) })) default = {} } diff --git a/platform/terraform/modules/vcs_github/workload_repos.tf b/platform/terraform/modules/vcs_github/workload_repos.tf index 118acfb7..987bd1ca 100644 --- a/platform/terraform/modules/vcs_github/workload_repos.tf +++ b/platform/terraform/modules/vcs_github/workload_repos.tf @@ -1,6 +1,11 @@ module "workload_repos" { source = "./repository" - for_each = var.workload_repos + for_each = { + for r in flatten([ + for wl in var.workloads : + [for k, v in wl.repos : { k = k, v = v }] + ]) : r.k => r.v + } repo_name = each.key description = each.value.description @@ -8,11 +13,9 @@ module "workload_repos" { auto_init = each.value.auto_init archive_on_destroy = each.value.archive_on_destroy has_issues = each.value.has_issues - is_template = each.value.is_template default_branch_name = each.value.default_branch_name delete_branch_on_merge = each.value.delete_branch_on_merge - template = each.value.template atlantis_enabled = each.value.atlantis_enabled - atlantis_url = each.value.atlantis_url - atlantis_repo_webhook_secret = each.value.atlantis_repo_webhook_secret + atlantis_url = var.atlantis_url + atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret } diff --git a/platform/terraform/secrets/main.tf b/platform/terraform/secrets/main.tf index b1cc0a00..ccc46763 100644 --- a/platform/terraform/secrets/main.tf +++ b/platform/terraform/secrets/main.tf @@ -20,18 +20,13 @@ provider "vault" { locals { cluster_name = "" provisioned_by = "cgdevx" - ### Workload groups definition bellow - workloads = { - # "workload-demo" = { - # }, - } } module "secrets" { source = "../modules/secrets_vault" cluster_name = local.cluster_name - workloads = local.workloads + workloads = var.workloads vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key vcs_bot_ssh_private_key = var.vcs_bot_ssh_private_key vcs_token = var.vcs_token diff --git a/platform/terraform/secrets/terraform.tfvars.json b/platform/terraform/secrets/terraform.tfvars.json new file mode 100644 index 00000000..56c2a5d3 --- /dev/null +++ b/platform/terraform/secrets/terraform.tfvars.json @@ -0,0 +1,3 @@ +{ + "workloads": {} +} \ No newline at end of file diff --git a/platform/terraform/secrets/variable.tf b/platform/terraform/secrets/variable.tf index b8dd18c5..d798ff4e 100644 --- a/platform/terraform/secrets/variable.tf +++ b/platform/terraform/secrets/variable.tf @@ -42,3 +42,11 @@ variable "cluster_endpoint" { description = "(Required) K8s cluster endpoint" type = string } + +variable "workloads" { + description = "Workloads configuration" + type = map(object({ + description = optional(string, "") + })) + default = {} +} diff --git a/platform/terraform/vcs/main.tf b/platform/terraform/vcs/main.tf index c201d5e3..2e82ff2a 100644 --- a/platform/terraform/vcs/main.tf +++ b/platform/terraform/vcs/main.tf @@ -6,24 +6,13 @@ terraform { # Configure Git Provider # + locals { gitops_repo_name = "" atlantis_url = "https:///events" - ### Workload repos definition bellow - workload_repos = { - # "workload-demo-iac" = { - # atlantis_enabled = true - # atlantis_url = local.atlantis_url - # atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret - # }, - # "workload-demo-template" = { - # is_template = true - # }, - # "workload-demo" = { - # }, - } } + module "vcs" { source = "../modules/vcs_" @@ -31,5 +20,5 @@ module "vcs" { atlantis_url = local.atlantis_url atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key - workload_repos = local.workload_repos + workloads = var.workloads } diff --git a/platform/terraform/vcs/terraform.tfvars.json b/platform/terraform/vcs/terraform.tfvars.json new file mode 100644 index 00000000..7a3e2589 --- /dev/null +++ b/platform/terraform/vcs/terraform.tfvars.json @@ -0,0 +1,23 @@ +{ + "workloads": { + "cnask": { + "description": "wl1", + "repos": { + "cnask-demo-iac": { + "atlantis_enabled": true + }, + "cnask-demo": {} + } + }, + "qgif": { + "description": "wl2", + "repos": { + "qgif-demo-iac": { + "atlantis_enabled": true + }, + "qgif-demo": {} + } + } + }, + "atlantis_repo_webhook_secret": "test" +} \ No newline at end of file diff --git a/platform/terraform/vcs/variable.tf b/platform/terraform/vcs/variable.tf index 435c7cde..e80c4740 100644 --- a/platform/terraform/vcs/variable.tf +++ b/platform/terraform/vcs/variable.tf @@ -7,3 +7,20 @@ variable "vcs_bot_ssh_public_key" { type = string default = "" } + +variable "workloads" { + description = "Workloads configuration" + type = map(object({ + description = optional(string, "") + repos = map(object({ + visibility = optional(string, "private") + auto_init = optional(bool, false) + archive_on_destroy = optional(bool, false) + has_issues = optional(bool, false) + default_branch_name = optional(string, "main") + delete_branch_on_merge = optional(bool, true) + atlantis_enabled = optional(bool, false) + })) + })) + default = {} +} diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 26c18a65..6302640e 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -1,12 +1,22 @@ +import json +import os + import click +import requests +from git import Actor, Repo + +from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ + LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES +from common.state_store import StateStore +from common.logging_config import configure_logging, logger @click.command() @click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) -@click.option('--workload-repository-name', '-wrn', 'wl_repo_name', help='Workload repository name', type=click.STRING, - prompt=True) +@click.option('--workload-repository-name', '-wrn', 'wl_repo_name', help='Workload repository name', + type=click.STRING) @click.option('--workload-gitops-repository-name', '-wgrn', 'wl_gitops_repo_name', - help='Workload GitOps repository name', type=click.STRING, prompt=True) + help='Workload GitOps repository name', type=click.STRING) @click.option('--workload-template-url', '-wtu', 'wl_template_url', help='Workload repository template', type=click.STRING) @click.option('--workload-template-branch', '-wtb', 'wl_template_branch', help='Workload repository template', @@ -16,7 +26,108 @@ @click.option('--workload-gitops-template-branch', '-wgb', 'wl_gitops_template_branch', help='Workload GitOps repository template', type=click.STRING) +@click.option('--verbosity', type=click.Choice( + ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + case_sensitive=False +), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_template_url: str, wl_template_branch: str, - wl_gitops_template_url: str, wl_gitops_template_branch: str): + wl_gitops_template_url: str, wl_gitops_template_branch: str, verbosity: str): """Create workload boilerplate.""" click.echo("Create workload.") + + # Set up global logger + configure_logging(verbosity) + + p: StateStore = StateStore() + + if not os.path.exists(LOCAL_GITOPS_FOLDER): + raise Exception("GitOps repo does not exist") + + # reset & update repo just in case + repo = Repo(LOCAL_GITOPS_FOLDER) + main_branch = repo.active_branch.name + repo.git.reset("--hard") + origin = repo.remotes.origin + origin.pull(repo.active_branch) + + # create new branch + branch_name = f"feature/{wl_name}-init" + current = repo.create_head(branch_name) + current.checkout() + + if not wl_repo_name: + wl_repo_name = wl_name + + if not wl_gitops_repo_name: + if wl_repo_name: + wl_gitops_repo_name = f"{wl_repo_name}-gitops" + else: + wl_gitops_repo_name = f"{wl_name}-gitops" + + # repos + with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "r") as file: + vcs_tf_vars = json.load(file) + + vcs_tf_vars["workloads"][wl_name] = { + "description": f"CG DevX {wl_name} workload definition", + "repos": {} + } + vcs_tf_vars["workloads"][wl_name]["repos"][wl_repo_name] = {} + vcs_tf_vars["workloads"][wl_name]["repos"][wl_gitops_repo_name] = { + "atlantis_enabled": True, + } + with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(vcs_tf_vars, indent=2)) + + # secrets + with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "r") as file: + secrets_tf_vars = json.load(file) + + secrets_tf_vars["workloads"][wl_name] = {} + + with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(secrets_tf_vars, indent=2)) + + # core services + with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "r") as file: + secrets_tf_vars = json.load(file) + + secrets_tf_vars["workloads"][wl_name] = {} + + with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(secrets_tf_vars, indent=2)) + + ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]}' + with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + + repo.git.add(all=True) + author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) + repo.index.commit("initial", author=author, committer=author) + + repo.remotes.origin.push(repo.active_branch.name) + + git_pulls_api = "https://api.github.com/repos/{0}/{1}/pulls".format( + p.parameters[""], + p.parameters[""] + ) + headers = { + "Authorization": "token {0}".format(p.internals["GIT_ACCESS_TOKEN"]), + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + + payload = { + "title": f"introduce {wl_name}", + "body": "Add default secrets, user and default repository structure. ", + "head": branch_name, + "base": main_branch + } + + r = requests.post( + git_pulls_api, + headers=headers, + data=json.dumps(payload)) + + if not r.ok: + raise click.ClickException("Could not create PR") + logger.error("GitHub API Request Failed: {0}".format(r.text)) diff --git a/tools/cli/services/template_manager.py b/tools/cli/services/template_manager.py index fd45477f..3d25674c 100644 --- a/tools/cli/services/template_manager.py +++ b/tools/cli/services/template_manager.py @@ -17,7 +17,7 @@ class GitOpsTemplateManager: """CG DevX Git repo templates manager.""" - def __init__(self, gitops_template_url: str, gitops_template_branch: str, token=None): + def __init__(self, gitops_template_url: str = None, gitops_template_branch: str = None, token=None): if gitops_template_url is None: self.__url = GITOPS_REPOSITORY_URL else: From 1dacf5f6de62b05a9b9499a8a3eb7be6dbb4ee9e Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Fri, 1 Dec 2023 18:10:46 +0100 Subject: [PATCH 02/49] create harbor robot user for workload and secret for it --- platform/terraform/core_services/main.tf | 3 +++ .../modules/registry_harbor/project/main.tf | 3 +++ .../modules/registry_harbor/project/robot.tf | 10 +------ .../modules/registry_harbor/project/vault.tf | 27 +++++++++++++++++++ 4 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 platform/terraform/modules/registry_harbor/project/vault.tf diff --git a/platform/terraform/core_services/main.tf b/platform/terraform/core_services/main.tf index 3bab879f..94c6a2bb 100644 --- a/platform/terraform/core_services/main.tf +++ b/platform/terraform/core_services/main.tf @@ -12,6 +12,9 @@ terraform { restapi = { source = "Mastercard/restapi" } + vault = { + source = "hashicorp/vault" + } } } diff --git a/platform/terraform/modules/registry_harbor/project/main.tf b/platform/terraform/modules/registry_harbor/project/main.tf index 8c2d6786..a6915195 100644 --- a/platform/terraform/modules/registry_harbor/project/main.tf +++ b/platform/terraform/modules/registry_harbor/project/main.tf @@ -3,6 +3,9 @@ terraform { harbor = { source = "goharbor/harbor" } + vault = { + source = "hashicorp/vault" + } } } diff --git a/platform/terraform/modules/registry_harbor/project/robot.tf b/platform/terraform/modules/registry_harbor/project/robot.tf index 981891a2..b1894c4e 100644 --- a/platform/terraform/modules/registry_harbor/project/robot.tf +++ b/platform/terraform/modules/registry_harbor/project/robot.tf @@ -7,7 +7,7 @@ resource "random_password" "robot_password" { } resource "harbor_robot_account" "workload_robot" { - name = "${var.project_name}-robot" + name = "robot" description = "${var.project_name} workload project level robot account" level = "project" secret = resource.random_password.robot_password.result @@ -17,14 +17,6 @@ resource "harbor_robot_account" "workload_robot" { action = "pull" resource = "repository" } - access { - action = "push" - resource = "repository" - } - access { - action = "create" - resource = "labels" - } kind = "project" namespace = harbor_project.this.name } diff --git a/platform/terraform/modules/registry_harbor/project/vault.tf b/platform/terraform/modules/registry_harbor/project/vault.tf new file mode 100644 index 00000000..e62098ec --- /dev/null +++ b/platform/terraform/modules/registry_harbor/project/vault.tf @@ -0,0 +1,27 @@ +locals { + b64_docker_auth = base64encode("robot@${var.project_name}+robot:${random_password.robot_password.result}") +} + +resource "vault_generic_secret" "docker_config" { + path = "workloads/${var.project_name}/dockerconfigjson" + + data_json = jsonencode( + { + dockerconfig = jsonencode({ "auths" : { "" : { "auth" : "${local.b64_docker_auth}" } } }), + } + ) + +} + +resource "vault_generic_secret" "harbor_main_robot_secret" { + path = "workloads/${var.project_name}/workload-robot-auth" + + data_json = jsonencode( + { + HARBOR_ROBOT_NAME = "robot@${var.project_name}+robot", + HARBOR_ROBOT_PASSWORD = random_password.robot_password.result, + HARBOR_ROBOT_B64_AUTH = local.b64_docker_auth, + } + ) + +} \ No newline at end of file From e8eeb19ba94a5907efc1723852ded65a9e07581c Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Fri, 1 Dec 2023 21:27:53 +0100 Subject: [PATCH 03/49] add workload ClusterSecretStore --- .../clustersecretstore.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-secret-store/clustersecretstore.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-secret-store/clustersecretstore.yaml index 1fcc461f..a2b57014 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-secret-store/clustersecretstore.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/cluster-secret-store/clustersecretstore.yaml @@ -20,3 +20,26 @@ spec: serviceAccountRef: name: 'external-secrets' namespace: 'external-secrets-operator' +--- +apiVersion: external-secrets.io/v1beta1 +kind: ClusterSecretStore +metadata: + name: vault-kv-workloads-secret + annotations: + argocd.argoproj.io/sync-wave: '10' +spec: + provider: + vault: + server: 'http://vault.vault.svc:8200' + # Path is the mount path of the Vault KV backend endpoint + path: 'workloads' + version: 'v2' + auth: + kubernetes: + # Path where the Kubernetes authentication backend is mounted in Vault + mountPath: 'kubernetes/cgdevx' + # A required field containing the Vault Role to assume. + role: 'external-secrets' + serviceAccountRef: + name: 'external-secrets' + namespace: 'external-secrets-operator' \ No newline at end of file From 811cd51e4bb6e339db3992b517c7d8959c96462b Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Fri, 1 Dec 2023 17:35:46 +0100 Subject: [PATCH 04/49] feat: add "workload destroy" command --- .../core-services/200-wl-argocd.yaml | 28 +++++ .../clusters/cc-cluster/workload/.placeholder | 0 .../workload/workload-template.yaml | 81 +++++++++++++ platform/terraform/vcs/terraform.tfvars.json | 12 +- tools/cli/commands/workload/create.py | 36 ++++-- tools/cli/commands/workload/delete.py | 110 +++++++++++++++++- 6 files changed, 245 insertions(+), 22 deletions(-) create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/.placeholder create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml new file mode 100644 index 00000000..61c0aaac --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: argocd + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: '200' +spec: + project: core + source: + repoURL: + path: gitops-pipelines/delivery/clusters/cc-cluster/workloads + targetRevision: HEAD + directory: + exclude: '{workload-template.yaml}' + destination: + server: 'https://kubernetes.default.svc' + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + retry: + limit: 5 + backoff: + duration: 5s + maxDuration: 5m0s + factor: 2 diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/.placeholder b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/.placeholder deleted file mode 100644 index e69de29b..00000000 diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml new file mode 100644 index 00000000..66f5aac8 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: minimalistic-app-sergs + namespace: argocd +spec: + sourceRepos: + - + destinations: + - namespace: -* + server: '*' + clusterResourceWhitelist: + - group: '*' + kind: '*' + roles: + # A role which provides read-only access to all applications in the project + - name: read-only + description: Read-only privileges to project + policies: + - p, proj::read-only, applications, get, /*, allow + groups: + - developers + - name: full-access + description: Full privileges to project for workload admins and developers groups + policies: + - p, proj::full-access, applications, *, /*, allow + groups: + - -admins + - -developers + +--- +apiVersion: v1 +kind: Secret +metadata: + name: -apps + labels: + argocd.argoproj.io/secret-type: repository +type: Opaque +stringData: + # Project scoped + project: + name: + url: +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: -appset + namespace: argocd +spec: + generators: + - git: + repoURL: + revision: HEAD + directories: + - path: gitops/environments/envs/* + template: + metadata: + name: '-{{path.basename}}' + spec: + project: + source: + repoURL: + targetRevision: HEAD + path: '{{path}}' + destination: + server: https://kubernetes.default.svc + namespace: 'wl--{{path.basename}}' + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + retry: + limit: 5 + backoff: + duration: 5s + maxDuration: 5m0s + factor: 2 \ No newline at end of file diff --git a/platform/terraform/vcs/terraform.tfvars.json b/platform/terraform/vcs/terraform.tfvars.json index 7a3e2589..0e40e978 100644 --- a/platform/terraform/vcs/terraform.tfvars.json +++ b/platform/terraform/vcs/terraform.tfvars.json @@ -8,16 +8,6 @@ }, "cnask-demo": {} } - }, - "qgif": { - "description": "wl2", - "repos": { - "qgif-demo-iac": { - "atlantis_enabled": true - }, - "qgif-demo": {} - } } - }, - "atlantis_repo_webhook_secret": "test" + } } \ No newline at end of file diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 6302640e..30383677 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -6,10 +6,10 @@ from git import Actor, Repo from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ - LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES -from common.state_store import StateStore + LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER from common.logging_config import configure_logging, logger - +from common.state_store import StateStore +from ghrepo import GHRepo @click.command() @click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) @@ -33,7 +33,7 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_template_url: str, wl_template_branch: str, wl_gitops_template_url: str, wl_gitops_template_branch: str, verbosity: str): """Create workload boilerplate.""" - click.echo("Create workload.") + click.echo("Create workload GitOps code.") # Set up global logger configure_logging(verbosity) @@ -45,7 +45,9 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_templat # reset & update repo just in case repo = Repo(LOCAL_GITOPS_FOLDER) - main_branch = repo.active_branch.name + main_branch = "main" + current = repo.create_head(main_branch) + current.checkout() repo.git.reset("--hard") origin = repo.remotes.origin origin.pull(repo.active_branch) @@ -90,13 +92,29 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_templat # core services with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "r") as file: - secrets_tf_vars = json.load(file) + services_tf_vars = json.load(file) - secrets_tf_vars["workloads"][wl_name] = {} + services_tf_vars["workloads"][wl_name] = {} with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(secrets_tf_vars, indent=2)) + file.write(json.dumps(services_tf_vars, indent=2)) + + # prepare ArgoCD manifest + wl_gitops_repo = GHRepo(p.parameters[""], wl_gitops_repo_name) + params = { + "": wl_name, + "": wl_gitops_repo.git_url, + } + + with open(LOCAL_FOLDER / "/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml", + "r") as file: + data = file.read() + for k, v in params.items(): + data = data.replace(k, v) + with open(LOCAL_FOLDER / f"/gitops-pipelines/delivery/clusters/cc-cluster/workload/{wl_name}.yaml", "w") as file: + file.write(data) + # commit and prepare a PR ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]}' with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): @@ -118,7 +136,7 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_templat payload = { "title": f"introduce {wl_name}", - "body": "Add default secrets, user and default repository structure. ", + "body": "Add default secrets, user and default repository structure.", "head": branch_name, "base": main_branch } diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index 837de3df..36fe34a1 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -1,8 +1,114 @@ +import json +import os + import click +import requests +from git import Actor, Repo + +from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ + LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER +from common.logging_config import configure_logging, logger +from common.state_store import StateStore @click.command() @click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) -def delete(wl_name: str): +@click.option('--verbosity', type=click.Choice( + ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + case_sensitive=False +), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') +def delete(wl_name: str, verbosity: str): """Deletes all the workload boilerplate.""" - click.echo("Delete workload.") + + click.echo("Delete workload GitOps code.") + + # Set up global logger + configure_logging(verbosity) + + p: StateStore = StateStore() + + if not os.path.exists(LOCAL_GITOPS_FOLDER): + raise Exception("GitOps repo does not exist") + + # reset & update repo just in case + repo = Repo(LOCAL_GITOPS_FOLDER) + main_branch = "main" + current = repo.create_head(main_branch) + current.checkout() + repo.git.reset("--hard") + origin = repo.remotes.origin + origin.pull(repo.active_branch) + + # create new branch + branch_name = f"feature/{wl_name}-destroy" + current = repo.create_head(branch_name) + current.checkout() + + # repos + with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "r") as file: + vcs_tf_vars = json.load(file) + + if wl_name in vcs_tf_vars["workloads"]: + del vcs_tf_vars["workloads"][wl_name] + + with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(vcs_tf_vars, indent=2)) + + # secrets + with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "r") as file: + secrets_tf_vars = json.load(file) + + if wl_name in secrets_tf_vars["workloads"]: + del secrets_tf_vars["workloads"][wl_name] + + with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(secrets_tf_vars, indent=2)) + + # core services + with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "r") as file: + services_tf_vars = json.load(file) + + if wl_name in services_tf_vars["workloads"]: + del services_tf_vars["workloads"][wl_name] + + with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(services_tf_vars, indent=2)) + + # delete ArgoCD manifest + os.remove(LOCAL_FOLDER / f"/gitops-pipelines/delivery/clusters/cc-cluster/workload/{wl_name}.yaml") + + # commit and prepare a PR + ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]}' + with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + + repo.git.add(all=True) + author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) + repo.index.commit("initial", author=author, committer=author) + + repo.remotes.origin.push(repo.active_branch.name) + + git_pulls_api = "https://api.github.com/repos/{0}/{1}/pulls".format( + p.parameters[""], + p.parameters[""] + ) + headers = { + "Authorization": "token {0}".format(p.internals["GIT_ACCESS_TOKEN"]), + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + + payload = { + "title": f"remove {wl_name}", + "body": "Remove default secrets, user and repository structure.", + "head": branch_name, + "base": main_branch + } + + r = requests.post( + git_pulls_api, + headers=headers, + data=json.dumps(payload)) + + if not r.ok: + raise click.ClickException("Could not create PR") + logger.error("GitHub API Request Failed: {0}".format(r.text)) From d47480906740ec6a96342676c1c16c95a278bf97 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Thu, 7 Dec 2023 20:14:16 +0100 Subject: [PATCH 05/49] feat: add "workload bootstrap" command --- .../core-services/200-wl-argocd.yaml | 2 +- .../cluster-workflow-templates}/.placeholder | 0 .../cluster-workflow-templates/cwft-git.yaml | 97 --------- .../cluster-workflow-templates/cwft-helm.yaml | 129 ------------ .../cwft-kaniko.yaml | 44 ---- .../workload-template.yaml | 0 .../argo-workflow/.argo/build_publish.yaml | 5 - .../argo-workflow/.argo/deploy.yaml | 5 - .../argo-workflow/.argo/promote.yaml | 3 - .../.github/workflows/workflow_trigger.yaml | 1 - .../integration/github/.placeholder | 0 .../integration/gitlab/.placeholder | 0 .../terraform/modules/cloud_aws/variables.tf | 4 + .../code_quality_sonarqube/project/main.tf | 4 +- .../modules/secrets_vault/secrets.tf | 4 +- .../terraform/modules/vcs_github/variable.tf | 1 + platform/terraform/vcs/terraform.tfvars.json | 12 +- platform/terraform/vcs/variable.tf | 1 + tools/cli/commands/destroy.py | 1 - tools/cli/commands/setup.py | 10 +- tools/cli/commands/workload/bootstrap.py | 194 +++++++++++++++++- tools/cli/commands/workload/create.py | 100 ++++----- tools/cli/commands/workload/delete.py | 61 +++--- tools/cli/common/command_utils.py | 72 +++++++ tools/cli/common/const/common_path.py | 1 + tools/cli/common/const/const.py | 2 +- tools/cli/services/cloud/aws/aws_sdk.py | 2 +- 27 files changed, 348 insertions(+), 407 deletions(-) rename platform/gitops-pipelines/{integration/argo-workflow => delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates}/.placeholder (100%) delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-git.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-helm.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-kaniko.yaml rename platform/gitops-pipelines/delivery/clusters/cc-cluster/{workload => workloads}/workload-template.yaml (100%) delete mode 100644 platform/gitops-pipelines/integration/argo-workflow/.argo/build_publish.yaml delete mode 100644 platform/gitops-pipelines/integration/argo-workflow/.argo/deploy.yaml delete mode 100644 platform/gitops-pipelines/integration/argo-workflow/.argo/promote.yaml delete mode 100644 platform/gitops-pipelines/integration/github/.github/workflows/workflow_trigger.yaml delete mode 100644 platform/gitops-pipelines/integration/github/.placeholder delete mode 100644 platform/gitops-pipelines/integration/gitlab/.placeholder diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml index 61c0aaac..97edad82 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: argocd + name: wl-argocd namespace: argocd annotations: argocd.argoproj.io/sync-wave: '200' diff --git a/platform/gitops-pipelines/integration/argo-workflow/.placeholder b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/.placeholder similarity index 100% rename from platform/gitops-pipelines/integration/argo-workflow/.placeholder rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/.placeholder diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-git.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-git.yaml deleted file mode 100644 index aa746676..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-git.yaml +++ /dev/null @@ -1,97 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: cwft-git - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - templates: - - name: checkout-with-gitops - inputs: - parameters: - - name: appName - - name: branch - default: main - - name: gitUrlNoProtocol - artifacts: - - name: repo-source - path: "/src/{{inputs.parameters.appName}}" - git: - repo: "{{inputs.parameters.gitUrlNoProtocol}}/{{inputs.parameters.appName}}.git" - branch: "{{inputs.parameters.branch}}" - singleBranch: true - insecureIgnoreHostKey: true - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - - name: gitops-source - path: /src/gitops - git: - repo: "{{inputs.parameters.gitUrlNoProtocol}}/gitops.git" - branch: "main" - singleBranch: true - insecureIgnoreHostKey: true - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - container: - image: golang:latest - command: ["/bin/sh", "-c"] - args: - - ls -la /src && - ls -la /src/{{inputs.parameters.appName}} - outputs: - artifacts: - - name: repo-source - path: /src - - name: pull-commit-push - retryStrategy: - limit: "5" - # todo get ssh item not all secrets - volumes: - - name: ssh-key - secret: - defaultMode: 256 - secretName: ci-secrets - inputs: - artifacts: - - name: repo-source - path: /src - parameters: - - name: commitMessage - - name: gitUrlNoProtocol - - name: repoName - container: - workingDir: "/src/{{inputs.parameters.repoName}}" - image: golang:latest - command: ["/bin/sh", "-c"] - volumeMounts: - - mountPath: "/mnt/secrets" - name: ssh-key - readOnly: true - args: - - set -e; - - eval `ssh-agent -s`; - mkdir $HOME/.ssh; - cat /mnt/secrets/SSH_PRIVATE_KEY > $HOME/.ssh/id_ed25519; - echo -n "\\n" >> $HOME/.ssh/id_ed25519; - chmod 0600 $HOME/.ssh/id_ed25519; - ssh-add $HOME/.ssh/id_ed25519; - - echo "Host *" >> $HOME/.ssh/config; - echo " StrictHostKeyChecking no" >> $HOME/.ssh/config; - echo " User git" >> $HOME/.ssh/config; - echo " IdentitiesOnly yes" >> $HOME/.ssh/config; - echo " UserKnownHostsFile /dev/null" >> $HOME/.ssh/config; - chmod 0700 $HOME/.ssh/config; - - git config --global user.email 'bot@cgdevx.io'; - git config --global user.name 'bot'; - git remote set-url origin '{{inputs.parameters.gitUrlNoProtocol}}/{{inputs.parameters.repoName}}.git'; - git remote -v; - git status; - git pull; - git add .; - git commit -m "{{inputs.parameters.commitMessage}}" || echo "Assuming this was committed on previous run, not erroring out" ; - git push; diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-helm.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-helm.yaml deleted file mode 100644 index f882fccc..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-helm.yaml +++ /dev/null @@ -1,129 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: cwft-helm - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - templates: - - name: get-chart-version - inputs: - artifacts: - - name: repo-source - path: /src - parameters: - - name: appName - - name: chartDir - script: - image: gcr.io/kaniko-project/executor - command: [python3] - workingDir: "/src/{{inputs.parameters.appName}}" - source: | - import yaml, semver - with open('./{{inputs.parameters.chartDir}}/Chart.yaml') as f: - chart_yaml = yaml.load(f, Loader=yaml.FullLoader) - print(chart_yaml['version']) - - name: set-chart-versions - inputs: - artifacts: - - name: repo-source - path: /src - parameters: - - name: appName - - name: chartDir - - name: chartVersion - - name: shortSha - script: - image: gcr.io/kaniko-project/executor - command: [bash] - workingDir: "/src/{{inputs.parameters.appName}}" - source: | - set -e - NEW_CHART_VERSION={{inputs.parameters.chartVersion}} - echo "setting ./{{inputs.parameters.chartDir}}/Chart.yaml to version: ${NEW_CHART_VERSION}" - sed -i "s/version:.*/version: ${NEW_CHART_VERSION}/g" /src/{{inputs.parameters.appName}}/{{inputs.parameters.chartDir}}/Chart.yaml - echo "setting ./{{inputs.parameters.chartDir}}/Chart.yaml to appVersion: {{inputs.parameters.shortSha}}" - sed -i "s/appVersion:.*/appVersion: {{inputs.parameters.shortSha}}/g" /src/{{inputs.parameters.appName}}/{{inputs.parameters.chartDir}}/Chart.yaml - echo "adjusted chart:" - cat /src/{{inputs.parameters.appName}}/{{inputs.parameters.chartDir}}/Chart.yaml - outputs: - artifacts: - - name: repo-source - path: /src - - name: publish-chart - retryStrategy: - limit: '5' - inputs: - artifacts: - - name: repo-source - path: /src - parameters: - - name: appName - - name: chartDir - container: - image: gcr.io/kaniko-project/executor - command: ['bash', '-c'] - workingDir: '/src/{{inputs.parameters.appName}}' - args: - - helm repo add cg-devx-platform https:// || bash -c "sleep 10 && echo 'waiting before trying again' && exit 1"; - helm push {{inputs.parameters.chartDir}} cg-devx-platform || bash -c "sleep 10 && echo 'waiting before trying again' && exit 1"; - env: - - name: BASIC_AUTH_PASS - valueFrom: - secretKeyRef: - name: ci-secrets - key: BASIC_AUTH_PASS - - name: BASIC_AUTH_USER - valueFrom: - secretKeyRef: - name: ci-secrets - key: BASIC_AUTH_USER - - name: set-environment-version - inputs: - artifacts: - - name: repo-source - path: /src - parameters: - - name: chartVersion - - name: environment - - name: fullChartPath - script: - image: gcr.io/kaniko-project/executor - command: [bash] - workingDir: "/src/gitops" - source: | - set -e - echo "setting wrapper Chart.yaml to version: {{inputs.parameters.chartVersion}}" - sed -i "s/ version:.*/ version: {{inputs.parameters.chartVersion}}/g" "{{inputs.parameters.fullChartPath}}" - echo "updated {{inputs.parameters.environment}} wrapper chart version to {{inputs.parameters.chartVersion}}" - outputs: - artifacts: - - name: repo-source - path: /src - - name: increment-chart-minor - inputs: - artifacts: - - name: repo-source - path: /src - parameters: - - name: appName - - name: chartDir - - name: chartVersion - script: - image: gcr.io/kaniko-project/executor - command: [python3] - workingDir: "/src/{{inputs.parameters.appName}}" - source: | - import yaml, semver - with open('./{{inputs.parameters.chartDir}}/Chart.yaml') as f: - chart_yaml = yaml.load(f, Loader=yaml.FullLoader) - chart_version = semver.parse('{{inputs.parameters.chartVersion}}') - next_chart_version = '{}.{}.0'.format(chart_version['major'],chart_version['minor']+1) - chart_yaml['version'] = next_chart_version - with open('./{{inputs.parameters.chartDir}}/Chart.yaml', 'w') as f: - yaml.dump(chart_yaml, f) - print('prepared next release in {{inputs.parameters.chartDir}} with bumped chart version after releasing {}'.format(next_chart_version)) - outputs: - artifacts: - - name: repo-source - path: /src \ No newline at end of file diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-kaniko.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-kaniko.yaml deleted file mode 100644 index 9cb4b7c4..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/cwft-kaniko.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: cwft-kaniko -spec: - entrypoint: build-push - templates: - - name: build-push - inputs: - parameters: - - name: appName - - name: branch - - name: containerRegistryURL - - name: gitUrlNoProtocol - artifacts: - - name: app-source - path: '/src/{{inputs.parameters.appName}}' - git: - repo: '{{inputs.parameters.gitUrlNoProtocol}}/{{inputs.parameters.appName}}.git' - branch: '{{inputs.parameters.branch}}' - singleBranch: true - insecureIgnoreHostKey: true - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - volumes: - - name: docker-config - secret: - secretName: 'container-registry-auth' - container: - image: gcr.io/kaniko-project/executor:latest - volumeMounts: - - name: docker-config - mountPath: /.docker - env: - - name: DOCKER_CONFIG - value: /.docker - args: - - '--dockerfile' - - 'Dockerfile' - - '--context' - - 'dir:///src/{{inputs.parameters.appName}}/' - - '--destination' - - '{{inputs.parameters.containerRegistryURL}}' diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml similarity index 100% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml diff --git a/platform/gitops-pipelines/integration/argo-workflow/.argo/build_publish.yaml b/platform/gitops-pipelines/integration/argo-workflow/.argo/build_publish.yaml deleted file mode 100644 index 6b3bdd42..00000000 --- a/platform/gitops-pipelines/integration/argo-workflow/.argo/build_publish.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - namespace: argo -spec: diff --git a/platform/gitops-pipelines/integration/argo-workflow/.argo/deploy.yaml b/platform/gitops-pipelines/integration/argo-workflow/.argo/deploy.yaml deleted file mode 100644 index 6b3bdd42..00000000 --- a/platform/gitops-pipelines/integration/argo-workflow/.argo/deploy.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - namespace: argo -spec: diff --git a/platform/gitops-pipelines/integration/argo-workflow/.argo/promote.yaml b/platform/gitops-pipelines/integration/argo-workflow/.argo/promote.yaml deleted file mode 100644 index 560a4561..00000000 --- a/platform/gitops-pipelines/integration/argo-workflow/.argo/promote.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -spec: diff --git a/platform/gitops-pipelines/integration/github/.github/workflows/workflow_trigger.yaml b/platform/gitops-pipelines/integration/github/.github/workflows/workflow_trigger.yaml deleted file mode 100644 index c7523285..00000000 --- a/platform/gitops-pipelines/integration/github/.github/workflows/workflow_trigger.yaml +++ /dev/null @@ -1 +0,0 @@ -#run argo workflow \ No newline at end of file diff --git a/platform/gitops-pipelines/integration/github/.placeholder b/platform/gitops-pipelines/integration/github/.placeholder deleted file mode 100644 index e69de29b..00000000 diff --git a/platform/gitops-pipelines/integration/gitlab/.placeholder b/platform/gitops-pipelines/integration/gitlab/.placeholder deleted file mode 100644 index e69de29b..00000000 diff --git a/platform/terraform/modules/cloud_aws/variables.tf b/platform/terraform/modules/cloud_aws/variables.tf index 31f5cfbe..68579cf1 100644 --- a/platform/terraform/modules/cloud_aws/variables.tf +++ b/platform/terraform/modules/cloud_aws/variables.tf @@ -30,6 +30,10 @@ variable "cluster_name" { condition = (length(var.cluster_name) <= 16) && (length(var.cluster_name) >= 2) error_message = "Must be between 2 and 16 symbols long" } + validation { + condition = can(regex("[a-z0-9]+(?:-[a-z0-9]+)*", var.cluster_name)) + error_message = "Invalid input, string should be in kebab-case." + } } variable "cluster_version" { diff --git a/platform/terraform/modules/code_quality_sonarqube/project/main.tf b/platform/terraform/modules/code_quality_sonarqube/project/main.tf index fe380e45..79e352c9 100644 --- a/platform/terraform/modules/code_quality_sonarqube/project/main.tf +++ b/platform/terraform/modules/code_quality_sonarqube/project/main.tf @@ -26,10 +26,12 @@ resource "sonarqube_permissions" "workload-admins" { group_name = "${var.project_name}-admins" project_key = sonarqube_project.this.project permissions = ["admin"] + depends_on = [sonarqube_group.workload_admins] } resource "sonarqube_permissions" "workload-developers" { group_name = "${var.project_name}-developers" project_key = sonarqube_project.this.project - permissions = ["codeviewer", "issueadmin", "securityhotspotadmin", "scan"] + permissions = ["codeviewer", "issueadmin", "scan", "securityhotspotadmin"] + depends_on = [sonarqube_group.workload_developers] } diff --git a/platform/terraform/modules/secrets_vault/secrets.tf b/platform/terraform/modules/secrets_vault/secrets.tf index 11f1b486..d42f9925 100644 --- a/platform/terraform/modules/secrets_vault/secrets.tf +++ b/platform/terraform/modules/secrets_vault/secrets.tf @@ -83,8 +83,8 @@ resource "vault_generic_secret" "atlantis_secrets" { VAULT_ADDR = "http://vault.vault.svc.cluster.local:8200", VAULT_TOKEN = var.vault_token, # code quality specific section - TF_VAR_code_quality_oidc_client_id = module.harbor.vault_oidc_client_id, - TF_VAR_code_quality_oidc_client_secret = module.harbor.vault_oidc_client_secret, + TF_VAR_code_quality_oidc_client_id = module.sonarqube.vault_oidc_client_id, + TF_VAR_code_quality_oidc_client_secret = module.sonarqube.vault_oidc_client_secret, TF_VAR_code_quality_admin_password = random_password.sonarqube_password.result, } ) diff --git a/platform/terraform/modules/vcs_github/variable.tf b/platform/terraform/modules/vcs_github/variable.tf index cb25fafc..af7018bd 100644 --- a/platform/terraform/modules/vcs_github/variable.tf +++ b/platform/terraform/modules/vcs_github/variable.tf @@ -24,6 +24,7 @@ variable "workloads" { type = map(object({ description = optional(string, "") repos = map(object({ + description = optional(string, "") visibility = optional(string, "private") auto_init = optional(bool, false) archive_on_destroy = optional(bool, false) diff --git a/platform/terraform/vcs/terraform.tfvars.json b/platform/terraform/vcs/terraform.tfvars.json index 0e40e978..56c2a5d3 100644 --- a/platform/terraform/vcs/terraform.tfvars.json +++ b/platform/terraform/vcs/terraform.tfvars.json @@ -1,13 +1,3 @@ { - "workloads": { - "cnask": { - "description": "wl1", - "repos": { - "cnask-demo-iac": { - "atlantis_enabled": true - }, - "cnask-demo": {} - } - } - } + "workloads": {} } \ No newline at end of file diff --git a/platform/terraform/vcs/variable.tf b/platform/terraform/vcs/variable.tf index e80c4740..c097238b 100644 --- a/platform/terraform/vcs/variable.tf +++ b/platform/terraform/vcs/variable.tf @@ -13,6 +13,7 @@ variable "workloads" { type = map(object({ description = optional(string, "") repos = map(object({ + description = optional(string, "") visibility = optional(string, "private") auto_init = optional(bool, false) archive_on_destroy = optional(bool, false) diff --git a/tools/cli/commands/destroy.py b/tools/cli/commands/destroy.py index 7d5be254..892e3411 100644 --- a/tools/cli/commands/destroy.py +++ b/tools/cli/commands/destroy.py @@ -89,7 +89,6 @@ def destroy(verbosity: str): argocd_token = get_argocd_token(p.internals["ARGOCD_USER"], p.internals["ARGOCD_PASSWORD"]) delete_application(registry_app_name, argocd_token) - # need to wait here # need to wait here wait(300) except Exception as e: diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 05b3ca52..30c91245 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -7,7 +7,7 @@ import yaml from common.command_utils import init_cloud_provider, init_git_provider, prepare_cloud_provider_auth_env_vars, \ - set_envs, unset_envs, wait + set_envs, unset_envs, wait, wait_http_endpoint_readiness from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_HOSTING_PROVIDER, \ LOCAL_TF_FOLDER_SECRETS_MANAGER, LOCAL_TF_FOLDER_USERS, LOCAL_TF_FOLDER_CORE_SERVICES from common.const.const import GITOPS_REPOSITORY_URL, GITOPS_REPOSITORY_BRANCH, KUBECTL_VERSION, PLATFORM_USER_NAME @@ -736,14 +736,18 @@ def setup( sonar_tls_cert = kube_client.get_certificate(SONARQUBE_NAMESPACE, "sonarqube-tls") kube_client.wait_for_certificate(sonar_tls_cert) - wait(30) + # wait for registry API endpoint readiness + wait_http_endpoint_readiness(f'https://{p.parameters[""]}') + p.internals["REGISTRY_USERNAME"] = "admin" # run security manager tf to create secrets and roles core_services_tf_env_vars = { **{ "HARBOR_URL": f'https://{p.parameters[""]}', "HARBOR_USERNAME": p.internals["REGISTRY_USERNAME"], - "HARBOR_PASSWORD": p.internals["REGISTRY_PASSWORD"] + "HARBOR_PASSWORD": p.internals["REGISTRY_PASSWORD"], + "VAULT_TOKEN": p.internals["VAULT_ROOT_TOKEN"], + "VAULT_ADDR": f'https://{p.parameters[""]}', }, **cloud_provider_auth_env_vars} # set envs as required by tf diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 00da6672..867e5482 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -1,11 +1,23 @@ +import os +import pathlib +import shutil + import click +from ghrepo import GHRepo +from git import Repo, GitError, Actor + +from common.command_utils import str_to_kebab, init_cloud_provider +from common.const.common_path import LOCAL_GITOPS_FOLDER, LOCAL_FOLDER +from common.logging_config import configure_logging, logger +from common.state_store import StateStore +from services.template_manager import GitOpsTemplateManager @click.command() -@click.option('--workload-repository-name', '-wrn', 'wl_repo_name', help='Workload repository name', type=click.STRING, - prompt=True) +@click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) +@click.option('--workload-repository-name', '-wrn', 'wl_repo_name', help='Workload repository name', type=click.STRING) @click.option('--workload-gitops-repository-name', '-wgrn', 'wl_gitops_repo_name', - help='Workload GitOps repository name', type=click.STRING, prompt=True) + help='Workload GitOps repository name', type=click.STRING) @click.option('--workload-template-url', '-wtu', 'wl_template_url', help='Workload repository template', type=click.STRING) @click.option('--workload-template-branch', '-wtb', 'wl_template_branch', help='Workload repository template', @@ -15,7 +27,175 @@ @click.option('--workload-gitops-template-branch', '-wgb', 'wl_gitops_template_branch', help='Workload GitOps repository template', type=click.STRING) -def bootstrap(wl_repo_name: str, wl_gitops_repo_name: str, wl_template_url: str, wl_template_branch: str, - wl_gitops_template_url: str, wl_gitops_template_branch: str): - """Bootstraps workload repository with template.""" - click.echo("Bootstrap workload.") +@click.option('--workload-service-name', '-ws', 'wl_svc_name', help='Workload service name', type=click.STRING) +@click.option('--workload-service-port', '-wsp', 'wl_svc_port', help='Workload service port', type=click.INT) +@click.option('--verbosity', type=click.Choice( + ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + case_sensitive=False +), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') +def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_template_url: str, wl_template_branch: str, + wl_gitops_template_url: str, wl_gitops_template_branch: str, wl_svc_name: str, wl_svc_port: int, + verbosity: str): + """Bootstrap workload repository with template.""" + click.echo("Bootstrapping workload...") + # Set up global logger + configure_logging(verbosity) + + if not os.path.exists(LOCAL_FOLDER): + raise click.ClickException("CG DevX metadata does not exist on this machine") + + p: StateStore = StateStore() + + wl_name = str_to_kebab(wl_name) + + if not wl_repo_name: + wl_repo_name = wl_name + + if not wl_gitops_repo_name: + if wl_repo_name: + wl_gitops_repo_name = f"{wl_repo_name}-gitops" + else: + wl_gitops_repo_name = f"{wl_name}-gitops" + + if not wl_svc_name: + wl_svc_name = "wl-service" + + if not wl_svc_port: + wl_svc_port = 3000 + + wl_svc_name = str_to_kebab(wl_svc_name) + wl_repo_name = str_to_kebab(wl_repo_name) + wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) + + if not wl_template_branch: + wl_template_branch = "main" + if not wl_template_url: + wl_template_url = "git@github.com:CloudGeometry/cg-devx-wl-template.git" + + if not wl_gitops_template_branch: + wl_gitops_template_branch = "main" + if not wl_gitops_template_url: + wl_gitops_template_url = "git@github.com:CloudGeometry/cg-devx-wl-gitops-template.git" + + if not os.path.exists(LOCAL_GITOPS_FOLDER): + raise click.ClickException("GitOps repo does not exist") + + temp_folder = LOCAL_FOLDER / ".wl_tmp" + + if os.path.exists(temp_folder): + shutil.rmtree(temp_folder) + + os.makedirs(temp_folder) + + key_path = p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] + + # workload repo + wl_repo_folder = temp_folder / wl_repo_name + os.makedirs(wl_repo_folder) + + try: + wl_repo = Repo.clone_from(GHRepo(p.parameters[""], wl_repo_name).ssh_url, + wl_repo_folder, + env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) + except GitError as e: + raise click.ClickException("Failed cloning repo") + + wl_template_repo_folder = temp_folder / GHRepo.parse(wl_template_url).name + os.makedirs(wl_template_repo_folder) + + try: + wl_template_repo = Repo.clone_from(wl_template_url, + wl_template_repo_folder, + branch=wl_template_branch, + env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) + except GitError as e: + raise click.ClickException("Failed cloning template repo") + + shutil.rmtree(wl_template_repo_folder / ".git") + shutil.copytree(wl_template_repo_folder, wl_repo_folder, dirs_exist_ok=True) + shutil.rmtree(wl_template_repo_folder) + shutil.move(wl_repo_folder / "wl-service-name", wl_repo_folder / wl_svc_name) + + wl_repo_params = { + "": wl_name, + "": wl_svc_name, + } + for root, dirs, files in os.walk(wl_repo_folder): + for name in files: + if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): + file_path = os.path.join(root, name) + with open(file_path, "r") as file: + data = file.read() + for k, v in wl_repo_params.items(): + data = data.replace(k, v) + with open(file_path, "w") as file: + file.write(data) + + wl_repo.git.add(all=True) + author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) + wl_repo.index.commit("initial", author=author, committer=author) + + wl_repo.remotes.origin.push(wl_repo.active_branch.name) + + # wl gitops repo + cloud_man, dns_man = init_cloud_provider(p) + + wl_gitops_repo_params = { + "": wl_name, + "": wl_svc_name, + "": f'{wl_svc_name}.{wl_name}.{p.parameters[""]}', + "": f'{p.parameters[""]}/{wl_name}/{wl_svc_name}', + "": str(wl_svc_port), + "# ": cloud_man.create_k8s_cluster_role_mapping_snippet(), + "": "[Put your workload service role mapping]", + "# ": cloud_man.create_additional_labels(), + } + + wl_gitops_repo_folder = temp_folder / wl_gitops_repo_name + os.makedirs(wl_gitops_repo_folder) + + try: + wl_gitops_repo = Repo.clone_from(GHRepo(p.parameters[""], wl_gitops_repo_name).ssh_url, + wl_gitops_repo_folder, + env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) + except GitError as e: + raise click.ClickException("Failed cloning repo") + + wl_gitops_template_repo_folder = temp_folder / GHRepo.parse(wl_gitops_template_url).name + os.makedirs(wl_gitops_template_repo_folder) + + try: + wl_gitops_template_repo = Repo.clone_from(wl_gitops_template_url, + wl_gitops_template_repo_folder, + branch=wl_gitops_template_branch, + env={ + "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) + except GitError as e: + raise click.ClickException("Failed cloning template repo") + + shutil.rmtree(wl_gitops_template_repo_folder / ".git") + shutil.copytree(wl_gitops_template_repo_folder, wl_gitops_repo_folder, dirs_exist_ok=True) + shutil.rmtree(wl_gitops_template_repo_folder) + + for root, dirs, files in os.walk(wl_gitops_repo_folder): + for name in files: + if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): + file_path = os.path.join(root, name) + with open(file_path, "r") as file: + data = file.read() + for k, v in wl_gitops_repo_params.items(): + data = data.replace(k, v) + with open(file_path, "w") as file: + file.write(data) + + wl_gitops_repo.git.add(all=True) + author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) + wl_gitops_repo.index.commit("initial", author=author, committer=author) + + wl_gitops_repo.remotes.origin.push(wl_gitops_repo.active_branch.name) + + # remove temp folder + shutil.rmtree(temp_folder) + + click.echo("Bootstrapping workload. Done!") + return True diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 30383677..260f386b 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -2,14 +2,15 @@ import os import click -import requests -from git import Actor, Repo +from ghrepo import GHRepo +from git import Actor +from common.command_utils import str_to_kebab, update_gitops_repo, create_pr from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ - LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER -from common.logging_config import configure_logging, logger + LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER, LOCAL_CC_CLUSTER_WORKLOAD_FOLDER +from common.logging_config import configure_logging from common.state_store import StateStore -from ghrepo import GHRepo + @click.command() @click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) @@ -17,45 +18,23 @@ type=click.STRING) @click.option('--workload-gitops-repository-name', '-wgrn', 'wl_gitops_repo_name', help='Workload GitOps repository name', type=click.STRING) -@click.option('--workload-template-url', '-wtu', 'wl_template_url', help='Workload repository template', - type=click.STRING) -@click.option('--workload-template-branch', '-wtb', 'wl_template_branch', help='Workload repository template', - type=click.STRING) -@click.option('--workload-gitops-template-url', '-wgu', 'wl_gitops_template_url', - help='Workload GitOps repository template', type=click.STRING) -@click.option('--workload-gitops-template-branch', '-wgb', 'wl_gitops_template_branch', - help='Workload GitOps repository template', - type=click.STRING) @click.option('--verbosity', type=click.Choice( ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False ), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') -def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_template_url: str, wl_template_branch: str, - wl_gitops_template_url: str, wl_gitops_template_branch: str, verbosity: str): +def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: str): """Create workload boilerplate.""" - click.echo("Create workload GitOps code.") + click.echo("Creating workload GitOps code...") # Set up global logger configure_logging(verbosity) - p: StateStore = StateStore() - - if not os.path.exists(LOCAL_GITOPS_FOLDER): - raise Exception("GitOps repo does not exist") + if not os.path.exists(LOCAL_FOLDER): + raise click.ClickException("CG DevX metadata does not exist on this machine") - # reset & update repo just in case - repo = Repo(LOCAL_GITOPS_FOLDER) - main_branch = "main" - current = repo.create_head(main_branch) - current.checkout() - repo.git.reset("--hard") - origin = repo.remotes.origin - origin.pull(repo.active_branch) + p: StateStore = StateStore() - # create new branch - branch_name = f"feature/{wl_name}-init" - current = repo.create_head(branch_name) - current.checkout() + wl_name = str_to_kebab(wl_name) if not wl_repo_name: wl_repo_name = wl_name @@ -66,6 +45,21 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_templat else: wl_gitops_repo_name = f"{wl_name}-gitops" + wl_repo_name = str_to_kebab(wl_repo_name) + wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) + + if not os.path.exists(LOCAL_GITOPS_FOLDER): + raise click.ClickException("GitOps repo does not exist") + + main_branch = "main" + + repo = update_gitops_repo() + + # create new branch + branch_name = f"feature/{wl_name}-init" + current = repo.create_head(branch_name) + current.checkout() + # repos with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "r") as file: vcs_tf_vars = json.load(file) @@ -106,12 +100,14 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_templat "": wl_gitops_repo.git_url, } - with open(LOCAL_FOLDER / "/gitops-pipelines/delivery/clusters/cc-cluster/workload/workload-template.yaml", - "r") as file: + workload_template_file = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / "workload-template.yaml" + with open(workload_template_file, "r") as file: data = file.read() for k, v in params.items(): data = data.replace(k, v) - with open(LOCAL_FOLDER / f"/gitops-pipelines/delivery/clusters/cc-cluster/workload/{wl_name}.yaml", "w") as file: + + workload_file = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml" + with open(workload_file, "w") as file: file.write(data) # commit and prepare a PR @@ -124,28 +120,14 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_templat repo.remotes.origin.push(repo.active_branch.name) - git_pulls_api = "https://api.github.com/repos/{0}/{1}/pulls".format( - p.parameters[""], - p.parameters[""] - ) - headers = { - "Authorization": "token {0}".format(p.internals["GIT_ACCESS_TOKEN"]), - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28" - } - - payload = { - "title": f"introduce {wl_name}", - "body": "Add default secrets, user and default repository structure.", - "head": branch_name, - "base": main_branch - } + if not create_pr(p.parameters[""], p.parameters[""], + p.internals["GIT_ACCESS_TOKEN"], + branch_name, main_branch, + f"introduce {wl_name}", + f"Add default secrets, user and default repository structure."): + raise click.ClickException("Could not create PR") - r = requests.post( - git_pulls_api, - headers=headers, - data=json.dumps(payload)) + repo.heads.main.checkout() - if not r.ok: - raise click.ClickException("Could not create PR") - logger.error("GitHub API Request Failed: {0}".format(r.text)) + click.echo("Creating workload GitOps code. Done!") + return True diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index 36fe34a1..c5ad02d7 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -3,10 +3,11 @@ import click import requests -from git import Actor, Repo +from git import Actor, Repo, RemoteReference +from common.command_utils import str_to_kebab, update_gitops_repo, create_pr from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ - LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER + LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER, LOCAL_CC_CLUSTER_WORKLOAD_FOLDER from common.logging_config import configure_logging, logger from common.state_store import StateStore @@ -20,24 +21,24 @@ def delete(wl_name: str, verbosity: str): """Deletes all the workload boilerplate.""" - click.echo("Delete workload GitOps code.") + click.echo("Deleting workload GitOps code...") # Set up global logger configure_logging(verbosity) + if not os.path.exists(LOCAL_FOLDER): + raise click.ClickException("CG DevX metadata does not exist on this machine") + p: StateStore = StateStore() + wl_name = str_to_kebab(wl_name) + if not os.path.exists(LOCAL_GITOPS_FOLDER): - raise Exception("GitOps repo does not exist") + raise click.ClickException("GitOps repo does not exist") - # reset & update repo just in case - repo = Repo(LOCAL_GITOPS_FOLDER) main_branch = "main" - current = repo.create_head(main_branch) - current.checkout() - repo.git.reset("--hard") - origin = repo.remotes.origin - origin.pull(repo.active_branch) + + repo = update_gitops_repo() # create new branch branch_name = f"feature/{wl_name}-destroy" @@ -75,7 +76,7 @@ def delete(wl_name: str, verbosity: str): file.write(json.dumps(services_tf_vars, indent=2)) # delete ArgoCD manifest - os.remove(LOCAL_FOLDER / f"/gitops-pipelines/delivery/clusters/cc-cluster/workload/{wl_name}.yaml") + os.remove(LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml") # commit and prepare a PR ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]}' @@ -87,28 +88,16 @@ def delete(wl_name: str, verbosity: str): repo.remotes.origin.push(repo.active_branch.name) - git_pulls_api = "https://api.github.com/repos/{0}/{1}/pulls".format( - p.parameters[""], - p.parameters[""] - ) - headers = { - "Authorization": "token {0}".format(p.internals["GIT_ACCESS_TOKEN"]), - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28" - } - - payload = { - "title": f"remove {wl_name}", - "body": "Remove default secrets, user and repository structure.", - "head": branch_name, - "base": main_branch - } - - r = requests.post( - git_pulls_api, - headers=headers, - data=json.dumps(payload)) - - if not r.ok: + if not create_pr(p.parameters[""], p.parameters[""], + p.internals["GIT_ACCESS_TOKEN"], + branch_name, main_branch, + f"remove {wl_name}", + f"Remove default secrets, user and repository structure."): raise click.ClickException("Could not create PR") - logger.error("GitHub API Request Failed: {0}".format(r.text)) + + repo.heads.main.checkout() + + click.echo("Deleting workload GitOps code. Done!") + return True + + diff --git a/tools/cli/common/command_utils.py b/tools/cli/common/command_utils.py index 3ce0fa51..1b25c74d 100644 --- a/tools/cli/common/command_utils.py +++ b/tools/cli/common/command_utils.py @@ -1,12 +1,21 @@ +import json import os import time +from re import sub +import requests +from git import Repo +from requests import HTTPError + +from common.const.common_path import LOCAL_GITOPS_FOLDER from common.const.parameter_names import CLOUD_REGION, CLOUD_PROFILE, CLOUD_ACCOUNT_ACCESS_KEY, \ CLOUD_ACCOUNT_ACCESS_SECRET, DNS_REGISTRAR_ACCESS_KEY, DNS_REGISTRAR_ACCESS_SECRET, GIT_ACCESS_TOKEN, \ GIT_ORGANIZATION_NAME from common.enums.cloud_providers import CloudProviders from common.enums.dns_registrars import DnsRegistrars from common.enums.git_providers import GitProviders +from common.logging_config import logger +from common.retry_decorator import exponential_backoff from common.state_store import StateStore from services.cloud.aws.aws_manager import AWSManager from services.cloud.azure.azure_manager import AzureManager @@ -101,3 +110,66 @@ def unset_envs(env_vars): def wait(seconds: float = 15): time.sleep(seconds) + + +@exponential_backoff() +def wait_http_endpoint_readiness(endpoint: str): + try: + response = requests.get(endpoint, + verify=False, + headers={"Content-Type": "application/json"}, + ) + if response.ok: + return + else: + raise Exception(f"Endpoint {endpoint} not ready.") + except HTTPError as e: + return + + +def str_to_kebab(string: str): + """ + Convert string to kebab case + :param string: input string + :return: kebab string + """ + return '-'.join( + sub(r"(\s|_|-)+", " ", + sub(r"[A-Z]{2,}(?=[A-Z][a-z]+[0-9]*|\b)|[A-Z]?[a-z]+[0-9]*|[A-Z]|[0-9]+", + lambda match: ' ' + match.group(0).lower(), string)).split()) + + +def update_gitops_repo(): + repo = Repo(LOCAL_GITOPS_FOLDER) + # clean stale branches + repo.remotes.origin.fetch(prune=True) + # update repo just in case + repo.heads.main.checkout() + repo.remotes.origin.pull(repo.active_branch) + return repo + + +def create_pr(org_name: str, repo_name: str, token: str, head_branch: str, base_branch: str, title: str, + body: str) -> bool: + git_pulls_api = f"https://api.github.com/repos/{org_name}/{repo_name}/pulls" + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + payload = { + "title": title, + "body": body, + "head": head_branch, + "base": base_branch + } + res = requests.post( + git_pulls_api, + headers=headers, + data=json.dumps(payload)) + + if not res.ok: + logger.error("GitHub API Request Failed: {0}".format(res.text)) + return False + + return True diff --git a/tools/cli/common/const/common_path.py b/tools/cli/common/const/common_path.py index 16e08f80..33b1f93c 100644 --- a/tools/cli/common/const/common_path.py +++ b/tools/cli/common/const/common_path.py @@ -14,3 +14,4 @@ LOCAL_TF_TOOL = LOCAL_TOOLS_FOLDER / "terraform" LOCAL_KCTL_TOOL = LOCAL_TOOLS_FOLDER / "kubectl" LOCAL_STATE_FILE = LOCAL_FOLDER / "state.yaml" +LOCAL_CC_CLUSTER_WORKLOAD_FOLDER = LOCAL_GITOPS_FOLDER / "gitops-pipelines/delivery/clusters/cc-cluster/workloads" diff --git a/tools/cli/common/const/const.py b/tools/cli/common/const/const.py index 2ff8d83e..625440ef 100644 --- a/tools/cli/common/const/const.py +++ b/tools/cli/common/const/const.py @@ -1,5 +1,5 @@ DEFAULT_ENUM_VALUE = "unknown" -GITOPS_REPOSITORY_URL = "https://github.com/CloudGeometry/cgdevx-core.git" +GITOPS_REPOSITORY_URL = "https://github.com/CloudGeometry/cg-devx-core.git" GITOPS_REPOSITORY_BRANCH = "main" STATE_INPUT_PARAMS = "input" STATE_INTERNAL_PARAMS = "internal" diff --git a/tools/cli/services/cloud/aws/aws_sdk.py b/tools/cli/services/cloud/aws/aws_sdk.py index 776f81b7..c95b6ab6 100644 --- a/tools/cli/services/cloud/aws/aws_sdk.py +++ b/tools/cli/services/cloud/aws/aws_sdk.py @@ -117,7 +117,7 @@ def enable_bucket_versioning(self, bucket_name, region=None): if region is None: region = self.region - resource = boto3.resource("s3", region_name=region) + resource = self._session_manager.session.resource("s3", region_name=region) versioning = resource.BucketVersioning(bucket_name) versioning.enable() From 809acaa68954124523c62f5c615f679a47d08747 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Fri, 8 Dec 2023 17:31:09 +0100 Subject: [PATCH 06/49] docs update minor appset fixes --- .../core-services/200-wl-argocd.yaml | 2 +- .../workloads/workload-template.yaml | 6 +- tools/README.md | 8 +- tools/cli/commands/README.md | 10 +- tools/cli/commands/workload/README.md | 135 ++++++++++++++++++ tools/cli/commands/workload/bootstrap.py | 20 +-- tools/cli/commands/workload/create.py | 8 +- tools/cli/commands/workload/delete.py | 2 +- 8 files changed, 170 insertions(+), 21 deletions(-) create mode 100644 tools/cli/commands/workload/README.md diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml index 97edad82..b8f37bdf 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: wl-argocd + name: workloads-appset namespace: argocd annotations: argocd.argoproj.io/sync-wave: '200' diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml index 66f5aac8..dfb85fd7 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml @@ -2,13 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: - name: minimalistic-app-sergs + name: namespace: argocd spec: sourceRepos: - - + - destinations: - - namespace: -* + - namespace: wl--* server: '*' clusterResourceWhitelist: - group: '*' diff --git a/tools/README.md b/tools/README.md index 9bcc56c5..22b9937e 100644 --- a/tools/README.md +++ b/tools/README.md @@ -90,8 +90,14 @@ Commands: - `setup` Creates new CG DevX installation - `destroy` Destroys existing CG DevX installation +- `workload` Commands related to Workload Management + - `create` Generates configuration of key Workload resources + - `bootstrap` Bootstraps Workload with configuration templates + - `delete` Removes configuration of key Workload resources Arguments: Are command specific and could be supplied via command lime, environment variables, or file -For more details, please check [commands](cli/commands/README.md) +For more details, +please check CG DevX quickstart [commands](cli/commands/README.md) +and [workload commands](cli/commands/workload/README.md) diff --git a/tools/cli/commands/README.md b/tools/cli/commands/README.md index b2162aac..201e6fcc 100644 --- a/tools/cli/commands/README.md +++ b/tools/cli/commands/README.md @@ -24,7 +24,7 @@ checkpoints) and could be re-run `setup` command could be executed using arguments, environment variables, or input file. -Arguments: +**Arguments**: | Name (short, full) | Type | Description | |--------------------------------|-----------------------------------------|---------------------------------------------------| @@ -50,6 +50,8 @@ Arguments: | -f, --config-file | FILENAME | Load parameters from file | | --verbosity | [DEBUG, INFO, WARNING, ERROR, CRITICAL] | Set the logging verbosity level, default CRITICAL | +> **Note!**: For all names use kebab-case. + `parameters.yaml` file example ```yaml @@ -112,6 +114,12 @@ process. **NOTE!**: this process is irreversible +**Arguments**: + +| Name (short, full) | Type | Description | +|--------------------|-----------------------------------------|---------------------------------------------------| +| --verbosity | [DEBUG, INFO, WARNING, ERROR, CRITICAL] | Set the logging verbosity level, default CRITICAL | + **Command snippet** ```bash diff --git a/tools/cli/commands/workload/README.md b/tools/cli/commands/workload/README.md new file mode 100644 index 00000000..ea0def98 --- /dev/null +++ b/tools/cli/commands/workload/README.md @@ -0,0 +1,135 @@ +# CG DevX CLI Workload Management commands + +Below is a list of Workload Management commands supported by CG DevX CLI tool. +Workload Management commands depend on local cluster metadata and: +a) could only be executed when CG DevX cluster is already provisioned; +b) should be executed from the same machine as cluster provisioning. + +For more details on cluster provisioning, please check [setup command documentation](../README.md#setup) + +## Create + +Generates declarative configuration of resources required for workload to function. Configuration is placed in the +platform +GitOps repository. CLI automatically pushes changes to feature branch and creates a Pull Request (PR). Changes +introduced with PR should be reviewed and adjusted when necessary. +All the changes to platform GitOps repository should be applied via Atlantis by typing `atlantis apply` in the PR +comments section. + +`workload create` command creates: + +- Configuration for **VCS** module to create Workload source code monorepo and Workload GitOps repo; +- Configuration for **Secrets** module to create Workload secrets namespace and RBAC; +- Configuration for **Core Services** module to create Image Registry, Code Quality, etc. projects for Workload; +- ArgoCD dedicated Workload Project AppSet to automatically deliver Workload services. + +`workload create` command could be executed using arguments, or environment variables. + +**Arguments**: + +| Name (short, full) | Type | Description | +|-------------------------------------------|-----------------------------------------|---------------------------------------------------| +| -wl, --workload-name | TEXT | Workload name | +| -wlrn, --workload-repository-name | TEXT | Name for Workload repository | +| -wlgrn, --workload-gitops-repository-name | TEXT | Name for Workload GitOps repository | +| --verbosity | [DEBUG, INFO, WARNING, ERROR, CRITICAL] | Set the logging verbosity level, default CRITICAL | + +> **Note!**: For all names use kebab-case. + +**Command snippet** + +Using command arguments + +```bash +cgdevxcli workload create --workload-name your-workload-name \ + --workload-repository-name your-workload-repository-name + --workload-gitops-repository-name your-workload-gitops-repository-name +``` + +## Bootstrap + +Creates folder structure and injects all the necessary configurations for your Workload into repositories created +by [create command](#create). + +By default, bootstrap command uses: + +- CG DevX Workload template [repository](https://github.com/CloudGeometry/cg-devx-wl-template) +- CG DevX Workload GitOps template [repository](https://github.com/CloudGeometry/cg-devx-wl-gitops-template) + +Those templates provide you: + +- Workload repository structure +- Pre-defined Docker file +- CI process +- CD process +- Release promotion process +- GitOps style environment definition +- IaC for out of the cluster cloud resources + +You could fork and customize existing template repositories and use them by providing custom template repository URLs +and branches. + +> **Note!**: Boostrap command must be executed using the same workload and workload repository names. + +`workload bootstrap` command could be executed using arguments, or environment variables. + +**Arguments**: + +| Name (short, full) | Type | Description | +|-------------------------------------------|-----------------------------------------|---------------------------------------------------| +| -wl, --workload-name | TEXT | Workload name | +| -wlrn, --workload-repository-name | TEXT | Name for Workload repository | +| -wlgrn, --workload-gitops-repository-name | TEXT | Name for Workload GitOps repository | +| -wltu, --workload-template-url | TEXT | Workload repository template | +| -wltb, --workload-template-branch | TEXT | Workload repository template branch | +| -wlgu, --workload-gitops-template-url | TEXT | Workload GitOps repository template | +| -wlgb, --workload-gitops-template-branch | TEXT | Workload GitOps repository template branch | +| -wls, --workload-service-name | TEXT | Workload service name | +| -wlsp, --workload-service-port | NUMBER | Workload service port, default 3000 | +| --verbosity | [DEBUG, INFO, WARNING, ERROR, CRITICAL] | Set the logging verbosity level, default CRITICAL | + +> **Note!**: For all names use kebab-case. + +**Command snippet** + +Using command arguments + +```bash +cgdevxcli workload create --workload-name your-workload-name \ + --workload-repository-name your-workload-repository-name + --workload-gitops-repository-name your-workload-gitops-repository-name + --workload-service-name your-first-service-name + --workload-service-port your-first-service-name-port +``` + +## Delete + +Removes declarative configuration of resources required for workload to function. CLI automatically pushes changes to +feature branch and creates a Pull Request (PR). Changes introduced with PR should be reviewed and adjusted when +necessary. +All the changes to platform GitOps repository should be applied via Atlantis by typing `atlantis apply` in the PR +comments section. + +`workload delete` command deletes all the configuration generated by `workload create` [command](#create): + +**NOTE!**: this process is irreversible + +`workload delete` command could be executed using arguments, or environment variables. + +**Arguments**: + +| Name (short, full) | Type | Description | +|----------------------|-----------------------------------------|---------------------------------------------------| +| -wl, --workload-name | TEXT | Workload name | +| --verbosity | [DEBUG, INFO, WARNING, ERROR, CRITICAL] | Set the logging verbosity level, default CRITICAL | + +> **Note!**: For all names use kebab-case. + +**Command snippet** + +Using command arguments + +```bash +cgdevxcli workload delete --workload-name your-workload-name +``` + diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 867e5482..b595ac47 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -14,21 +14,21 @@ @click.command() -@click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) -@click.option('--workload-repository-name', '-wrn', 'wl_repo_name', help='Workload repository name', type=click.STRING) -@click.option('--workload-gitops-repository-name', '-wgrn', 'wl_gitops_repo_name', +@click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) +@click.option('--workload-repository-name', '-wlrn', 'wl_repo_name', help='Workload repository name', type=click.STRING) +@click.option('--workload-gitops-repository-name', '-wlgrn', 'wl_gitops_repo_name', help='Workload GitOps repository name', type=click.STRING) -@click.option('--workload-template-url', '-wtu', 'wl_template_url', help='Workload repository template', +@click.option('--workload-template-url', '-wltu', 'wl_template_url', help='Workload repository template', type=click.STRING) -@click.option('--workload-template-branch', '-wtb', 'wl_template_branch', help='Workload repository template', +@click.option('--workload-template-branch', '-wltb', 'wl_template_branch', help='Workload repository template branch', type=click.STRING) -@click.option('--workload-gitops-template-url', '-wgu', 'wl_gitops_template_url', +@click.option('--workload-gitops-template-url', '-wlgu', 'wl_gitops_template_url', help='Workload GitOps repository template', type=click.STRING) -@click.option('--workload-gitops-template-branch', '-wgb', 'wl_gitops_template_branch', - help='Workload GitOps repository template', +@click.option('--workload-gitops-template-branch', '-wlgb', 'wl_gitops_template_branch', + help='Workload GitOps repository template branch', type=click.STRING) -@click.option('--workload-service-name', '-ws', 'wl_svc_name', help='Workload service name', type=click.STRING) -@click.option('--workload-service-port', '-wsp', 'wl_svc_port', help='Workload service port', type=click.INT) +@click.option('--workload-service-name', '-wls', 'wl_svc_name', help='Workload service name', type=click.STRING) +@click.option('--workload-service-port', '-wlsp', 'wl_svc_port', help='Workload service port', type=click.INT) @click.option('--verbosity', type=click.Choice( ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 260f386b..5ef1185b 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -13,10 +13,10 @@ @click.command() -@click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) -@click.option('--workload-repository-name', '-wrn', 'wl_repo_name', help='Workload repository name', +@click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) +@click.option('--workload-repository-name', '-wlrn', 'wl_repo_name', help='Workload repository name', type=click.STRING) -@click.option('--workload-gitops-repository-name', '-wgrn', 'wl_gitops_repo_name', +@click.option('--workload-gitops-repository-name', '-wlgrn', 'wl_gitops_repo_name', help='Workload GitOps repository name', type=click.STRING) @click.option('--verbosity', type=click.Choice( ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], @@ -97,7 +97,7 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: wl_gitops_repo = GHRepo(p.parameters[""], wl_gitops_repo_name) params = { "": wl_name, - "": wl_gitops_repo.git_url, + "": wl_gitops_repo.ssh_url, } workload_template_file = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / "workload-template.yaml" diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index c5ad02d7..d3a89811 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -13,7 +13,7 @@ @click.command() -@click.option('--workload-name', '-w', 'wl_name', help='Workload name', type=click.STRING, prompt=True) +@click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) @click.option('--verbosity', type=click.Choice( ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False From bb909d494c55491012c430bf72473b39c0ee0ce6 Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Mon, 11 Dec 2023 17:53:32 +0300 Subject: [PATCH 07/49] feat: Added wft to core-services/components --- .../workflowtemplates/black-wft.yaml | 15 ++++ .../workflowtemplates/crane-wft.yaml | 17 +++++ .../workflowtemplates/git-clone-wft.yaml | 38 ++++++++++ .../workflowtemplates/kaniko-wft.yaml | 17 +++++ .../minimalistic-app-dag-wft.yaml | 74 +++++++++++++++++++ .../workflowtemplates/sillypy-dag-wft.yaml | 72 ++++++++++++++++++ .../workflowtemplates/trivy-fs-scan-wft.yaml | 20 +++++ .../workflowtemplates/tslint-wft.yaml | 15 ++++ 8 files changed, 268 insertions(+) create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/trivy-fs-scan-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml new file mode 100644 index 00000000..a9dd0fa2 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: black-wft +spec: + templates: + - name: black + container: + image: pyfound/black + imagePullPolicy: IfNotPresent + command: ["black", "--check", "-v", "/{{workflow.parameters.workload}}"] + volumeMounts: + - name: build + mountPath: "{{workflow.parameters.workload}}" + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml new file mode 100644 index 00000000..9d7da839 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: crane-wft +spec: + templates: + - name: crane + container: + image: gcr.io/go-containerregistry/crane + imagePullPolicy: IfNotPresent + args: ["push", "/workspace/{{workflow.parameters.workload}}.tar", "{{workflow.parameters.image}}:{{workflow.parameters.tag}}"] + volumeMounts: + - name: kaniko-secret + mountPath: /home/nonroot/.docker + - name: build + mountPath: /workspace + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml new file mode 100644 index 00000000..f080a248 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml @@ -0,0 +1,38 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: git-clone-wft +spec: + templates: + - name: git-clone + inputs: + artifacts: + - name: git-src + path: /src + git: + repo: "{{workflow.parameters.repo}}" + # revision: "{{workflow.parameters.revision}}" + sshPrivateKeySecret: + name: ci-secrets + key: SSH_PRIVATE_KEY + depth: 1 + script: + image: python:latest #subj to change + env: + - name: WORKLOAD + value: "{{workflow.parameters.workload}}" + command: [sh] + source: | + pwd + git status && ls -lra && cat dockerfile + # mkdir /build/${WORKLOAD} + #cp -r * /build/${WORKLOAD} + cp -rv src/$WORKLOAD /build/ + cd /build/ + pwd + ls -l + echo "I've been here" | tee test.txt + volumeMounts: + - mountPath: "/build" + name: build + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml new file mode 100644 index 00000000..2a39ec54 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: kaniko-wft +spec: + templates: + - name: kaniko + container: + image: gcr.io/kaniko-project/executor + imagePullPolicy: IfNotPresent + args: ["--dockerfile=/workspace/dockerfile", "--no-push", "--tar-path=/workspace/{{workflow.parameters.workload}}.tar"] + volumeMounts: + - name: kaniko-secret + mountPath: /kaniko/.docker + - name: build + mountPath: /workspace + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml new file mode 100644 index 00000000..a086d716 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml @@ -0,0 +1,74 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: minimalistic-app-dag-wft +spec: + arguments: + parameters: + - name: image + value: harbor.delta.cgdevx.io/minimalistic-app-test/minimalistic-app + - name: tag + value: "0.3" + - name: workload + value: minimalistic-app-test + - name: wl-service-name + value: minimalistic-app + - name: revision + value: "v1.0" + - name: repo + value: git@github.com:CloudGeometry/cg-devx-minimalistic-demo-app.git + volumeClaimTemplates: + - metadata: + name: build + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 4Gi + entrypoint: ci-sequence + onExit: exit-handler + volumes: + # - name: build + # persistentVolumeClaim: +#claimName: build-volume-claim + - name: kaniko-secret + secret: + secretName: harbcred + items: + - key: .dockerconfigjson + path: config.json + + templates: + - name: ci-sequence + dag: + tasks: + - name: git-clone + templateRef: + name: git-clone-wft + template: git-clone + - name: linter + dependencies: [git-clone] + templateRef: + name: tslint-wft + template: tslint + - name: kaniko + dependencies: [git-clone] + templateRef: + name: kaniko-wft + template: kaniko + - name: trivy-fs-scan + dependencies: [git-clone] + templateRef: + name: trivy-fs-scan-wft + template: trivy-fs-scan + - name: crane + dependencies: [git-clone,linter,kaniko,trivy-fs-scan] + templateRef: + name: crane-wft + template: crane + - name: exit-handler + container: + image: alpine:latest + command: [sh, -c] + args: ["echo {{workflow.name}} {{workflow.status}} {{workflow.duration}}"] + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml new file mode 100644 index 00000000..3d5c3c81 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml @@ -0,0 +1,72 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: sillypy-dag-template +spec: + arguments: + parameters: + - name: image + value: harbor.aws-test.cgdevx.io/sillypy/sillypy + - name: tag + value: "0.3" + - name: workload + value: sillypy + - name: revision + value: "v1.0" + - name: repo + value: git@github.com:CGDevX-Demo/workload-sillypy.git + volumeClaimTemplates: + - metadata: + name: build + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 4Gi + entrypoint: ci-sequence + onExit: exit-handler + volumes: + # - name: build + # persistentVolumeClaim: +#claimName: build-volume-claim + - name: kaniko-secret + secret: + secretName: harbcred + items: + - key: .dockerconfigjson + path: config.json + + templates: + - name: ci-sequence + dag: + tasks: + - name: git-clone + templateRef: + name: git-clone-wft + template: git-clone + - name: linter + dependencies: [git-clone] + templateRef: + name: black-wft + template: black + - name: kaniko + dependencies: [linter] + templateRef: + name: kaniko-wft + template: kaniko + - name: trivy-fs-scan + dependencies: [kaniko] + templateRef: + name: trivy-fs-scan-wft + template: trivy-fs-scan + - name: crane + dependencies: [git-clone,linter,kaniko,trivy-fs-scan] + templateRef: + name: crane-wft + template: crane + - name: exit-handler + container: + image: alpine:latest + command: [sh, -c] + args: ["echo {{workflow.name}} {{workflow.status}} {{workflow.duration}}"] + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/trivy-fs-scan-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/trivy-fs-scan-wft.yaml new file mode 100644 index 00000000..d55d228e --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/trivy-fs-scan-wft.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: trivy-fs-scan-wft +#TODO: +# in the end: clean up unpacked-fs +# script for untar and run trivy fs +spec: + templates: + - name: trivy-fs-scan + container: + image: aquasec/trivy + args: + - fs + - /workdir + # - /workdir/{{workflow.parameters.workload}} + volumeMounts: + - name: build + mountPath: /workdir + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml new file mode 100644 index 00000000..3812899d --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: tslint-wft +spec: + templates: + - name: tslint + container: + image: registry.gitlab.com/pipeline-components/tslint:latest + imagePullPolicy: IfNotPresent + command: ["tslint", "/{{workflow.parameters.workload}}/src/**/*.ts"] + volumeMounts: + - name: build + mountPath: "{{workflow.parameters.workload}}" + From 80f5254400cffab68adbd9e556b770c27742cbf6 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Tue, 12 Dec 2023 17:16:01 +0100 Subject: [PATCH 08/49] - git worker groups - tf destroy of wl resources --- .../50-actions-runner-controller.yaml | 2 +- .../core-services/60-github-runner.yaml | 2 +- .../application.yaml | 5 + .../externalsecret.yaml | 0 .../actions-runner-controller/wait.yaml | 0 .../github-runner/runnerdeployment.yaml | 0 .../github}/github-runner/serviceaccount.yaml | 0 tools/cli/commands/destroy.py | 2 + tools/cli/commands/setup.py | 6 +- tools/cli/commands/workload/README.md | 13 ++- tools/cli/commands/workload/bootstrap.py | 3 + tools/cli/commands/workload/delete.py | 102 +++++++++++++++--- tools/cli/common/enums/git_plans.py | 12 +++ .../cli/services/vcs/git_provider_manager.py | 18 ++++ .../cli/services/vcs/github/github_manager.py | 36 +++++++ .../cli/services/vcs/gitlab/gitlab_manager.py | 13 +++ 16 files changed, 192 insertions(+), 22 deletions(-) rename platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/{kube-system => runners/github}/actions-runner-controller/application.yaml (97%) rename platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/{kube-system => runners/github}/actions-runner-controller/externalsecret.yaml (100%) rename platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/{kube-system => runners/github}/actions-runner-controller/wait.yaml (100%) rename platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/{ => runners/github}/github-runner/runnerdeployment.yaml (100%) rename platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/{ => runners/github}/github-runner/serviceaccount.yaml (100%) create mode 100644 tools/cli/common/enums/git_plans.py diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/50-actions-runner-controller.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/50-actions-runner-controller.yaml index eec18344..b0493e14 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/50-actions-runner-controller.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/50-actions-runner-controller.yaml @@ -9,7 +9,7 @@ spec: project: core source: repoURL: - path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller + path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller targetRevision: HEAD destination: server: https://kubernetes.default.svc diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/60-github-runner.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/60-github-runner.yaml index 5c8eff12..618b1183 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/60-github-runner.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/60-github-runner.yaml @@ -9,7 +9,7 @@ spec: project: core source: repoURL: - path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/github-runner + path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner targetRevision: HEAD destination: server: https://kubernetes.default.svc diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/application.yaml similarity index 97% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller/application.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/application.yaml index b60a4ad6..afd045e1 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/application.yaml @@ -15,6 +15,11 @@ spec: labels: {} replicaCount: 1 + + template: + spec:{ + # + } webhookPort: 9443 syncPeriod: 1m diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller/externalsecret.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/externalsecret.yaml similarity index 100% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller/externalsecret.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/externalsecret.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller/wait.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/wait.yaml similarity index 100% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/actions-runner-controller/wait.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/wait.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/github-runner/runnerdeployment.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/runnerdeployment.yaml similarity index 100% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/github-runner/runnerdeployment.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/runnerdeployment.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/github-runner/serviceaccount.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/serviceaccount.yaml similarity index 100% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/github-runner/serviceaccount.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/serviceaccount.yaml diff --git a/tools/cli/commands/destroy.py b/tools/cli/commands/destroy.py index 892e3411..91df9e40 100644 --- a/tools/cli/commands/destroy.py +++ b/tools/cli/commands/destroy.py @@ -26,6 +26,8 @@ ), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') def destroy(verbosity: str): """Destroy existing CG DevX installation.""" + click.confirm("This will destroy CG DevX installation. Please confirm to continue", abort=True) + # Set up global logger configure_logging(verbosity) diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 30c91245..6be9cdeb 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -65,7 +65,7 @@ @click.option('--gitops-template-branch', '-gtb', 'gitops_template_branch', help='GitOps repository template branch', default=GITOPS_REPOSITORY_BRANCH, type=click.STRING) @click.option('--setup-demo-workload', '-dw', 'install_demo', help='Setup demo workload', default=False, - flag_value='setup-demo') + is_flag=True) @click.option('--config-file', '-f', 'config', help='Load parameters from file', type=click.File(mode='r')) @click.option('--verbosity', type=click.Choice( ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], @@ -148,6 +148,10 @@ def setup( p.internals["GIT_USER_EMAIL"] = git_user_email p.fragments["# "] = git_man.create_tf_module_snippet() + if git_man.get_organization_plan(p.get_input_param(GIT_ORGANIZATION_NAME)) > 1: + p.fragments["# "] = git_man.create_runner_group_snippet() + p.parameters[""] = p.get_input_param(PRIMARY_CLUSTER_NAME) + dns_provider_check(dns_man, p) click.echo("DNS provider pre-flight check. Done!") diff --git a/tools/cli/commands/workload/README.md b/tools/cli/commands/workload/README.md index ea0def98..fe6b5601 100644 --- a/tools/cli/commands/workload/README.md +++ b/tools/cli/commands/workload/README.md @@ -112,16 +112,21 @@ comments section. `workload delete` command deletes all the configuration generated by `workload create` [command](#create): +When executed with --destroy-resources flag it will also destroy all the resources created for a specific workload. +Please note that workload GitOps repository name should match one for workload. + **NOTE!**: this process is irreversible `workload delete` command could be executed using arguments, or environment variables. **Arguments**: -| Name (short, full) | Type | Description | -|----------------------|-----------------------------------------|---------------------------------------------------| -| -wl, --workload-name | TEXT | Workload name | -| --verbosity | [DEBUG, INFO, WARNING, ERROR, CRITICAL] | Set the logging verbosity level, default CRITICAL | +| Name (short, full) | Type | Description | +|-------------------------------------------|-----------------------------------------|---------------------------------------------------| +| -wl, --workload-name | TEXT | Workload name | +| -wldr, --destroy-resources | Flag | Destroy workload resources | +| -wlgrn, --workload-gitops-repository-name | TEXT | Name for Workload GitOps repository | +| --verbosity | [DEBUG, INFO, WARNING, ERROR, CRITICAL] | Set the logging verbosity level, default CRITICAL | > **Note!**: For all names use kebab-case. diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index b595ac47..95548e64 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -149,6 +149,9 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp "# ": cloud_man.create_k8s_cluster_role_mapping_snippet(), "": "[Put your workload service role mapping]", "# ": cloud_man.create_additional_labels(), + "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], f"workloads/{wl_name}/secrets"), + "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], f"workloads/{wl_name}/hosting_provider"), + "# ": cloud_man.create_hosting_provider_snippet(), } wl_gitops_repo_folder = temp_folder / wl_gitops_repo_name diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index d3a89811..fc6b0ec7 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -1,25 +1,33 @@ import json import os +import shutil import click -import requests -from git import Actor, Repo, RemoteReference +from ghrepo import GHRepo +from git import Repo, GitError, Actor -from common.command_utils import str_to_kebab, update_gitops_repo, create_pr +from common.command_utils import str_to_kebab, update_gitops_repo, prepare_cloud_provider_auth_env_vars, set_envs, \ + create_pr from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER, LOCAL_CC_CLUSTER_WORKLOAD_FOLDER -from common.logging_config import configure_logging, logger +from common.const.parameter_names import GIT_ACCESS_TOKEN, GIT_ORGANIZATION_NAME +from common.logging_config import configure_logging from common.state_store import StateStore +from services.tf_wrapper import TfWrapper @click.command() @click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) +@click.option('--workload-gitops-repository-name', '-wlgrn', 'wl_gitops_repo_name', + help='Workload GitOps repository name', type=click.STRING) +@click.option('--destroy-resources', '-wldr', 'destroy_resources', help='Destroy workload resources', is_flag=True, default=False) @click.option('--verbosity', type=click.Choice( ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False ), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') -def delete(wl_name: str, verbosity: str): +def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verbosity: str): """Deletes all the workload boilerplate.""" + click.confirm("This will delete workload. Please confirm to continue", abort=True) click.echo("Deleting workload GitOps code...") @@ -33,6 +41,11 @@ def delete(wl_name: str, verbosity: str): wl_name = str_to_kebab(wl_name) + if not wl_gitops_repo_name: + wl_gitops_repo_name = f"{wl_name}-gitops" + + wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) + if not os.path.exists(LOCAL_GITOPS_FOLDER): raise click.ClickException("GitOps repo does not exist") @@ -52,8 +65,8 @@ def delete(wl_name: str, verbosity: str): if wl_name in vcs_tf_vars["workloads"]: del vcs_tf_vars["workloads"][wl_name] - with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(vcs_tf_vars, indent=2)) + with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(vcs_tf_vars, indent=2)) # secrets with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "r") as file: @@ -62,8 +75,8 @@ def delete(wl_name: str, verbosity: str): if wl_name in secrets_tf_vars["workloads"]: del secrets_tf_vars["workloads"][wl_name] - with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(secrets_tf_vars, indent=2)) + with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(secrets_tf_vars, indent=2)) # core services with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "r") as file: @@ -72,14 +85,75 @@ def delete(wl_name: str, verbosity: str): if wl_name in services_tf_vars["workloads"]: del services_tf_vars["workloads"][wl_name] - with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(services_tf_vars, indent=2)) + with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(services_tf_vars, indent=2)) # delete ArgoCD manifest - os.remove(LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml") + wl_argo_manifest = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml" + if os.path.exists(wl_argo_manifest): + os.remove(wl_argo_manifest) + + key_path = p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] + + # destroy resources + if destroy_resources: + # to destroy workload resources, we need to clone workload gitops repo + # and call tf destroy while pointing to remote state + tf_destroy_confirmation: bool = click.confirm( + "This will delete all the resources defined in workload IaC. Please confirm to continue") + + if tf_destroy_confirmation: + temp_folder = LOCAL_FOLDER / ".wl_tmp" + wl_gitops_repo_folder = temp_folder / f"{wl_name}-gitops" + + if os.path.exists(wl_gitops_repo_folder): + shutil.rmtree(wl_gitops_repo_folder) + + os.makedirs(wl_gitops_repo_folder) + + try: + wl_gitops_repo = Repo.clone_from( + GHRepo(p.parameters[""], wl_gitops_repo_name).ssh_url, + wl_gitops_repo_folder, + env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) + except GitError as e: + raise click.ClickException("Failed cloning repo") + + cloud_provider_auth_env_vars = prepare_cloud_provider_auth_env_vars(p) + + tf_env_vars = { + **cloud_provider_auth_env_vars, + **{ + "GITHUB_TOKEN": p.get_input_param(GIT_ACCESS_TOKEN), + "GITHUB_OWNER": p.get_input_param(GIT_ORGANIZATION_NAME), + "VAULT_TOKEN": p.internals.get("VAULT_ROOT_TOKEN", None), + "VAULT_ADDR": f'https://{p.parameters.get("", None)}', + } + } + # set envs as required by tf + set_envs(tf_env_vars) + + click.echo("Destroying WL secrets...") + + tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/secrets") + tf_wrapper.init() + tf_wrapper.destroy() + + click.echo("Destroying WL secrets. Done!") + + click.echo("Destroying WL cloud resources...") + + tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/infrastructure") + tf_wrapper.init() + tf_wrapper.destroy() + + click.echo("Destroying WL cloud resources. Done!") + + # remove temp folder + shutil.rmtree(temp_folder) # commit and prepare a PR - ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]}' + ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {key_path}' with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): repo.git.add(all=True) @@ -99,5 +173,3 @@ def delete(wl_name: str, verbosity: str): click.echo("Deleting workload GitOps code. Done!") return True - - diff --git a/tools/cli/common/enums/git_plans.py b/tools/cli/common/enums/git_plans.py new file mode 100644 index 00000000..e5328837 --- /dev/null +++ b/tools/cli/common/enums/git_plans.py @@ -0,0 +1,12 @@ +"""Git provider subscription types enums.""" +import enum + +from common.const.const import DEFAULT_ENUM_VALUE + + +class GitSubscriptionPlans(int, enum.Enum): + """List of standardized Git provider subscription plans.""" + + Free = 1 + Pro = 2 + Enterprise = 3 diff --git a/tools/cli/services/vcs/git_provider_manager.py b/tools/cli/services/vcs/git_provider_manager.py index 154b1307..e09c4fe3 100644 --- a/tools/cli/services/vcs/git_provider_manager.py +++ b/tools/cli/services/vcs/git_provider_manager.py @@ -1,5 +1,7 @@ from abc import ABC, abstractmethod +from common.enums.git_plans import GitSubscriptionPlans + class GitProviderManager(ABC): """Git provider wrapper to standardise Git management.""" @@ -35,3 +37,19 @@ def get_current_user_info(self): :return: Login, Name, Email """ pass + + @abstractmethod + def create_runner_group_snippet(self) -> str: + """ + Create external CI/CD runner group snippet + :return: Snippet + """ + pass + + @abstractmethod + def get_organization_plan(self, organization_name: str) -> [str, GitSubscriptionPlans]: + """ + Get active plan, if present + :return: Subscription plan + """ + pass diff --git a/tools/cli/services/vcs/github/github_manager.py b/tools/cli/services/vcs/github/github_manager.py index 7c01c4ac..774c23c0 100644 --- a/tools/cli/services/vcs/github/github_manager.py +++ b/tools/cli/services/vcs/github/github_manager.py @@ -4,6 +4,7 @@ import requests from common.const.const import FALLBACK_AUTHOR_NAME, FALLBACK_AUTHOR_EMAIL +from common.enums.git_plans import GitSubscriptionPlans from common.tracing_decorator import trace from services.vcs.git_provider_manager import GitProviderManager @@ -91,3 +92,38 @@ def get_current_user_info(self): @trace() def create_tf_module_snippet(self): return 'provider "github" {}' + + @trace() + def create_runner_group_snippet(self) -> str: + return 'group: ' + + @trace() + def get_organization_plan(self, organization_name: str) -> GitSubscriptionPlans: + """ + Get active plan, if present + :return: Plan name + """ + + headers = { + 'Authorization': f'token {self.__token}', + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + } + try: + response = requests.get(f'https://api.github.com/orgs/{organization_name}', headers=headers) + if response.ok: + res = json.loads(response.text) + + plan_name = res["plan"]["name"] + if plan_name == "pro": + return GitSubscriptionPlans.Enterprise + elif plan_name == "team": + return GitSubscriptionPlans.Pro + elif plan_name == "free": + return GitSubscriptionPlans.Free + else: + return GitSubscriptionPlans.Free + else: + raise Exception("Org not found") + except HTTPError as e: + raise e diff --git a/tools/cli/services/vcs/gitlab/gitlab_manager.py b/tools/cli/services/vcs/gitlab/gitlab_manager.py index ffcc9b79..1417550a 100644 --- a/tools/cli/services/vcs/gitlab/gitlab_manager.py +++ b/tools/cli/services/vcs/gitlab/gitlab_manager.py @@ -5,6 +5,7 @@ from requests.exceptions import HTTPError from common.const.const import FALLBACK_AUTHOR_NAME, FALLBACK_AUTHOR_EMAIL +from common.enums.git_plans import GitSubscriptionPlans from common.tracing_decorator import trace from services.vcs.git_provider_manager import GitProviderManager @@ -166,3 +167,15 @@ def create_tf_module_snippet(self) -> str: :return: A string containing the Terraform module snippet. """ return 'provider "gitlab" {}' + + @trace() + def create_runner_group_snippet(self) -> str: + return '' + + @trace() + def get_organization_plan(self, organization_name: str) -> GitSubscriptionPlans: + """ + Get active plan, if present + :return: Plan name + """ + return GitSubscriptionPlans.Free From db81b727904a706d57212a0c90fa6fb3ddfd6dc0 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Wed, 13 Dec 2023 14:06:42 +0100 Subject: [PATCH 09/49] - open PR url in web browser - s3 bucket delete fix - extra meta on delete - runner group fix --- .../actions-runner-controller/application.yaml | 7 +------ .../github/github-runner/runnerdeployment.yaml | 2 ++ tools/cli/commands/destroy.py | 11 ++++++++--- tools/cli/commands/setup.py | 4 +++- tools/cli/commands/workload/create.py | 15 ++++++++++----- tools/cli/commands/workload/delete.py | 17 ++++++++++------- tools/cli/common/command_utils.py | 7 ++++--- tools/cli/services/cloud/aws/aws_sdk.py | 18 ++++++++---------- 8 files changed, 46 insertions(+), 35 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/application.yaml index afd045e1..d09c72af 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/actions-runner-controller/application.yaml @@ -15,12 +15,7 @@ spec: labels: {} replicaCount: 1 - - template: - spec:{ - # - } - + webhookPort: 9443 syncPeriod: 1m defaultScaleDownDelay: 10m diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/runnerdeployment.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/runnerdeployment.yaml index 5811e856..1291e5e0 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/runnerdeployment.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/runners/github/github-runner/runnerdeployment.yaml @@ -13,3 +13,5 @@ spec: serviceAccountName: github-runner dockerdWithinRunnerContainer: true automountServiceAccountToken: true + # set runner group name when GitHub is on non-free plan + # diff --git a/tools/cli/commands/destroy.py b/tools/cli/commands/destroy.py index 91df9e40..2fb7d8c9 100644 --- a/tools/cli/commands/destroy.py +++ b/tools/cli/commands/destroy.py @@ -26,7 +26,6 @@ ), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') def destroy(verbosity: str): """Destroy existing CG DevX installation.""" - click.confirm("This will destroy CG DevX installation. Please confirm to continue", abort=True) # Set up global logger configure_logging(verbosity) @@ -35,9 +34,14 @@ def destroy(verbosity: str): click.echo("CG DevX installation local files not found.") return - click.echo("Destroying CG DevX installation...") p: StateStore = StateStore() + click.confirm( + f'This will destroy cluster "{p.parameters[""]}" and local files at "{LOCAL_FOLDER}". Please confirm to continue', + abort=True) + + click.echo("Destroying CG DevX installation...") + cloud_man, dns_man = init_cloud_provider(p) cloud_provider_auth_env_vars = prepare_cloud_provider_auth_env_vars(p) @@ -113,7 +117,8 @@ def destroy(verbosity: str): unset_envs(tf_env_vars) # delete IaC backend storage bucket - cloud_man.destroy_iac_state_storage(p.internals["TF_BACKEND_STORAGE_NAME"]) + if not cloud_man.destroy_iac_state_storage(p.internals["TF_BACKEND_STORAGE_NAME"]): + click.echo(f'Failed to delete IaC state storage {p.internals["TF_BACKEND_STORAGE_NAME"]}. You should delete it manually.') # delete local data folder shutil.rmtree(LOCAL_FOLDER) diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 6be9cdeb..d8560228 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -148,7 +148,9 @@ def setup( p.internals["GIT_USER_EMAIL"] = git_user_email p.fragments["# "] = git_man.create_tf_module_snippet() - if git_man.get_organization_plan(p.get_input_param(GIT_ORGANIZATION_NAME)) > 1: + git_subscription_plan = git_man.get_organization_plan(p.get_input_param(GIT_ORGANIZATION_NAME)) + p.internals["GIT_SUBSCRIPTION_PLAN"] = git_subscription_plan + if git_subscription_plan > 1: p.fragments["# "] = git_man.create_runner_group_snippet() p.parameters[""] = p.get_input_param(PRIMARY_CLUSTER_NAME) diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 5ef1185b..b24ba68e 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -1,5 +1,6 @@ import json import os +import webbrowser import click from ghrepo import GHRepo @@ -120,12 +121,16 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: repo.remotes.origin.push(repo.active_branch.name) - if not create_pr(p.parameters[""], p.parameters[""], - p.internals["GIT_ACCESS_TOKEN"], - branch_name, main_branch, - f"introduce {wl_name}", - f"Add default secrets, user and default repository structure."): + pr_url = create_pr(p.parameters[""], p.parameters[""], + p.internals["GIT_ACCESS_TOKEN"], + branch_name, main_branch, + f"introduce {wl_name}", + f"Add default secrets, user and default repository structure.") + + if not pr_url: raise click.ClickException("Could not create PR") + else: + webbrowser.open(pr_url, autoraise=False) repo.heads.main.checkout() diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index fc6b0ec7..8a80dc1e 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -1,6 +1,7 @@ import json import os import shutil +import webbrowser import click from ghrepo import GHRepo @@ -20,14 +21,15 @@ @click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) @click.option('--workload-gitops-repository-name', '-wlgrn', 'wl_gitops_repo_name', help='Workload GitOps repository name', type=click.STRING) -@click.option('--destroy-resources', '-wldr', 'destroy_resources', help='Destroy workload resources', is_flag=True, default=False) +@click.option('--destroy-resources', '-wldr', 'destroy_resources', help='Destroy workload resources', is_flag=True, + default=False) @click.option('--verbosity', type=click.Choice( ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False ), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verbosity: str): """Deletes all the workload boilerplate.""" - click.confirm("This will delete workload. Please confirm to continue", abort=True) + click.confirm(f'This will delete the workload "{wl_name}". Please confirm to continue', abort=True) click.echo("Deleting workload GitOps code...") @@ -162,12 +164,13 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb repo.remotes.origin.push(repo.active_branch.name) - if not create_pr(p.parameters[""], p.parameters[""], - p.internals["GIT_ACCESS_TOKEN"], - branch_name, main_branch, - f"remove {wl_name}", - f"Remove default secrets, user and repository structure."): + pr_url = create_pr(p.parameters[""], p.parameters[""], + p.internals["GIT_ACCESS_TOKEN"], branch_name, main_branch, f"remove {wl_name}", + f"Remove default secrets, user and repository structure.") + if not pr_url: raise click.ClickException("Could not create PR") + else: + webbrowser.open(pr_url, autoraise=False) repo.heads.main.checkout() diff --git a/tools/cli/common/command_utils.py b/tools/cli/common/command_utils.py index 1b25c74d..67e5597d 100644 --- a/tools/cli/common/command_utils.py +++ b/tools/cli/common/command_utils.py @@ -150,7 +150,7 @@ def update_gitops_repo(): def create_pr(org_name: str, repo_name: str, token: str, head_branch: str, base_branch: str, title: str, - body: str) -> bool: + body: str) -> str: git_pulls_api = f"https://api.github.com/repos/{org_name}/{repo_name}/pulls" headers = { "Authorization": f"token {token}", @@ -170,6 +170,7 @@ def create_pr(org_name: str, repo_name: str, token: str, head_branch: str, base_ if not res.ok: logger.error("GitHub API Request Failed: {0}".format(res.text)) - return False + return + data = json.loads(res.text) - return True + return data["html_url"] diff --git a/tools/cli/services/cloud/aws/aws_sdk.py b/tools/cli/services/cloud/aws/aws_sdk.py index c95b6ab6..cdce5539 100644 --- a/tools/cli/services/cloud/aws/aws_sdk.py +++ b/tools/cli/services/cloud/aws/aws_sdk.py @@ -252,18 +252,16 @@ def delete_bucket(self, bucket_name: str, region: str = None): if region is None: region = self.region - s3_client = self._session_manager.session.client('s3', region_name=region) + resource = self._session_manager.session.resource("s3", region_name=region) + s3_client = self._session_manager.session.client("s3", region_name=region) - objects = s3_client.get_paginator("list_objects_v2") + bucket = resource.Bucket(bucket_name) + bucket_versioning = resource.BucketVersioning(bucket_name) - objects_iterator = objects.paginate(Bucket=bucket_name) - for res in objects_iterator: - if 'Versions' in res: - objects = [{'Key': obj['Key'], 'VersionId': obj['VersionId']} for obj in res['Versions']] - s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects}) - if 'Contents' in res: - objects = [{'Key': obj['Key']} for obj in res['Contents']] - s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects}) + if bucket_versioning.status == 'Enabled': + bucket.object_versions.delete() + else: + bucket.objects.all().delete() s3_client.delete_bucket(Bucket=bucket_name) except ClientError as e: From 56dc7e6968debfcebe42562f2bd56725fae85fa0 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Thu, 14 Dec 2023 17:33:01 +0100 Subject: [PATCH 10/49] add harbor proxy --- platform/terraform/modules/registry_harbor/proxy.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 platform/terraform/modules/registry_harbor/proxy.tf diff --git a/platform/terraform/modules/registry_harbor/proxy.tf b/platform/terraform/modules/registry_harbor/proxy.tf new file mode 100644 index 00000000..f404f993 --- /dev/null +++ b/platform/terraform/modules/registry_harbor/proxy.tf @@ -0,0 +1,10 @@ +resource "harbor_project" "proxy" { + name = "dockerhub-proxy" + registry_id = harbor_registry.docker.registry_id +} + +resource "harbor_registry" "docker" { + provider_name = "docker-hub" + name = "dockerhub-proxy" + endpoint_url = "https://hub.docker.com" +} \ No newline at end of file From 599bce105c00c1a5df278bc4e9d74794421ea87b Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Thu, 14 Dec 2023 18:05:15 +0100 Subject: [PATCH 11/49] - set tags as locals - change cni plugin creation to fix pod-count - pass tags as vars - appset resource deletion fix --- .../workloads/workload-template.yaml | 6 +++- platform/terraform/hosting_provider/main.tf | 19 +++++++++---- platform/terraform/modules/cloud_aws/eks.tf | 1 + platform/terraform/modules/cloud_aws/main.tf | 28 ++++++++----------- .../terraform/modules/cloud_aws/variables.tf | 10 ++++++- .../terraform/modules/cloud_azure/main.tf | 14 ++++------ .../modules/cloud_azure/variables.tf | 10 ++++++- .../modules/secrets_vault/auth-backends.tf | 7 ----- platform/terraform/secrets/main.tf | 3 -- tools/cli/commands/setup.py | 4 +-- tools/cli/common/enums/git_plans.py | 6 ++-- tools/cli/services/cloud/aws/aws_manager.py | 5 +--- .../services/k8s/delivery_service_manager.py | 8 +++--- .../cli/services/vcs/git_provider_manager.py | 2 +- 14 files changed, 66 insertions(+), 57 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml index dfb85fd7..d9c3d23f 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml @@ -78,4 +78,8 @@ spec: backoff: duration: 5s maxDuration: 5m0s - factor: 2 \ No newline at end of file + factor: 2 + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/platform/terraform/hosting_provider/main.tf b/platform/terraform/hosting_provider/main.tf index bf93b63a..48e197b7 100644 --- a/platform/terraform/hosting_provider/main.tf +++ b/platform/terraform/hosting_provider/main.tf @@ -7,6 +7,13 @@ locals { cluster_name = "" region = "" email = [""] + tags = { + ClusterName = local.cluster_name + ProvisionedBy = "CGDevX" + } + labels = { + provisioned-by = "cg-devx" + } } # Cloud Provider configuration @@ -14,9 +21,11 @@ locals { module "hosting-provider" { - source = "../modules/cloud_" - cluster_name = local.cluster_name - region = local.region - alert_emails = local.email - ssh_public_key = var.ssh_public_key + source = "../modules/cloud_" + cluster_name = local.cluster_name + region = local.region + alert_emails = local.email + ssh_public_key = var.ssh_public_key + tags = local.tags + cluster_node_labels = local.labels } diff --git a/platform/terraform/modules/cloud_aws/eks.tf b/platform/terraform/modules/cloud_aws/eks.tf index 1760e993..4af81309 100644 --- a/platform/terraform/modules/cloud_aws/eks.tf +++ b/platform/terraform/modules/cloud_aws/eks.tf @@ -26,6 +26,7 @@ module "eks" { } vpc-cni = { most_recent = true + before_compute = true service_account_role_arn = module.vpc_cni_irsa.iam_role_arn configuration_values = jsonencode({ env = { diff --git a/platform/terraform/modules/cloud_aws/main.tf b/platform/terraform/modules/cloud_aws/main.tf index 0e48ad35..c278b819 100644 --- a/platform/terraform/modules/cloud_aws/main.tf +++ b/platform/terraform/modules/cloud_aws/main.tf @@ -1,17 +1,15 @@ data "aws_caller_identity" "current" {} data "aws_availability_zones" "available" {} locals { - name = var.cluster_name - cluster_version = var.cluster_version - region = var.region - aws_account = data.aws_caller_identity.current.account_id - vpc_cidr = var.cluster_network_cidr - azs = slice(data.aws_availability_zones.available.names, 0, min(var.az_count, length(data.aws_availability_zones.available.names))) - cluster_node_lables = var.cluster_node_labels - node_group_type = var.node_group_type - tags = { - cgx_name = local.name - } + name = var.cluster_name + cluster_version = var.cluster_version + region = var.region + aws_account = data.aws_caller_identity.current.account_id + vpc_cidr = var.cluster_network_cidr + azs = slice(data.aws_availability_zones.available.names, 0, min(var.az_count, length(data.aws_availability_zones.available.names))) + cluster_node_lables = var.cluster_node_labels + node_group_type = var.node_group_type + tags = var.tags node_labels_substr = join(",", formatlist("%s=%s", keys(var.cluster_node_labels), values(var.cluster_node_labels))) default_node_group_name = "${local.name}-node-group" eks_node_groups = [ @@ -45,8 +43,8 @@ locals { instance_market_options = ((upper(node_group.capacity_type) == "spot") && (local.node_group_type == "SELF")) ? { market_type = "spot" } : {} - capacity_type = upper(node_group.capacity_type) - bootstrap_extra_args = join(",", [ + capacity_type = upper(node_group.capacity_type) + bootstrap_extra_args = join(",", [ "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=${node_group.capacity_type}", local.node_labels_substr, "'" @@ -58,13 +56,9 @@ locals { } #end of locals - ################################################################################ # Supporting Resources ################################################################################ - - - data "aws_ami" "eks_default" { most_recent = true owners = ["amazon"] diff --git a/platform/terraform/modules/cloud_aws/variables.tf b/platform/terraform/modules/cloud_aws/variables.tf index 68579cf1..b370622d 100644 --- a/platform/terraform/modules/cloud_aws/variables.tf +++ b/platform/terraform/modules/cloud_aws/variables.tf @@ -75,11 +75,19 @@ variable "node_groups" { variable "cluster_node_labels" { type = map(any) default = { - ProvisionedBy = "CGDevX" + provisioned-by = "cg-devx" } description = "(Optional) EKS node labels" } +variable "tags" { + type = map(string) + default = { + ProvisionedBy = "CGDevX" + } + description = "(Optional) Specifies the AWS resource tags" +} + variable "alert_emails" { type = list(string) default = [] diff --git a/platform/terraform/modules/cloud_azure/main.tf b/platform/terraform/modules/cloud_azure/main.tf index 54bce070..c952c4a5 100644 --- a/platform/terraform/modules/cloud_azure/main.tf +++ b/platform/terraform/modules/cloud_azure/main.tf @@ -17,14 +17,12 @@ locals { name = var.cluster_name vnet_name = "${var.cluster_name}-vnet" log_analytics_retention_days = 30 - tags = { - cgx_name = var.cluster_name - } - azs = [for i in range(1, var.az_count + 1) : tostring(i)] - default_node_group = var.node_groups[0] - additional_node_pools = try(slice(var.node_groups, 1, length(var.node_groups)), []) - max_pods = 100 - node_admin_username = "azadmin" + tags = var.tags + azs = [for i in range(1, var.az_count + 1) : tostring(i)] + default_node_group = var.node_groups[0] + additional_node_pools = try(slice(var.node_groups, 1, length(var.node_groups)), []) + max_pods = 100 + node_admin_username = "azadmin" } resource "azurerm_resource_group" "rg" { diff --git a/platform/terraform/modules/cloud_azure/variables.tf b/platform/terraform/modules/cloud_azure/variables.tf index 2406edc7..8ffabe7b 100644 --- a/platform/terraform/modules/cloud_azure/variables.tf +++ b/platform/terraform/modules/cloud_azure/variables.tf @@ -62,11 +62,19 @@ variable "node_groups" { variable "cluster_node_labels" { type = map(string) default = { - ProvisionedBy = "CGDevX" + provisioned-by = "cg-devx" } description = "(Optional) Specifies the AKS node labels" } +variable "tags" { + type = map(string) + default = { + ProvisionedBy = "CGDevX" + } + description = "(Optional) Specifies the Azure resource tags" +} + variable "alert_emails" { type = list(string) default = [] diff --git a/platform/terraform/modules/secrets_vault/auth-backends.tf b/platform/terraform/modules/secrets_vault/auth-backends.tf index 02084589..0e579599 100644 --- a/platform/terraform/modules/secrets_vault/auth-backends.tf +++ b/platform/terraform/modules/secrets_vault/auth-backends.tf @@ -1,10 +1,3 @@ -###AWS specific data sources begin -data "aws_eks_cluster" "cluster" { - name = var.cluster_name -} - -###AWS specific data sources end - resource "vault_auth_backend" "k8s" { type = "kubernetes" path = "kubernetes/cgdevx" diff --git a/platform/terraform/secrets/main.tf b/platform/terraform/secrets/main.tf index ccc46763..51da261f 100644 --- a/platform/terraform/secrets/main.tf +++ b/platform/terraform/secrets/main.tf @@ -14,9 +14,6 @@ provider "vault" { skip_tls_verify = "true" } -# Cloud Provider configuration -# - locals { cluster_name = "" provisioned_by = "cgdevx" diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index d8560228..aaf4d821 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -149,8 +149,8 @@ def setup( p.fragments["# "] = git_man.create_tf_module_snippet() git_subscription_plan = git_man.get_organization_plan(p.get_input_param(GIT_ORGANIZATION_NAME)) - p.internals["GIT_SUBSCRIPTION_PLAN"] = git_subscription_plan - if git_subscription_plan > 1: + p.internals["GIT_SUBSCRIPTION_PLAN"] = bool(git_subscription_plan) + if git_subscription_plan > 0: p.fragments["# "] = git_man.create_runner_group_snippet() p.parameters[""] = p.get_input_param(PRIMARY_CLUSTER_NAME) diff --git a/tools/cli/common/enums/git_plans.py b/tools/cli/common/enums/git_plans.py index e5328837..873ed4bd 100644 --- a/tools/cli/common/enums/git_plans.py +++ b/tools/cli/common/enums/git_plans.py @@ -7,6 +7,6 @@ class GitSubscriptionPlans(int, enum.Enum): """List of standardized Git provider subscription plans.""" - Free = 1 - Pro = 2 - Enterprise = 3 + Free = 0 + Pro = 1 + Enterprise = 2 diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index 3f585021..64c2c68e 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -92,10 +92,7 @@ def create_hosting_provider_snippet(self) -> str: return textwrap.dedent('''\ provider "aws" { default_tags { - tags = { - ClusterName = local.cluster_name - ProvisionedBy = "CGDevX" - } + tags = local.tags } }''') diff --git a/tools/cli/services/k8s/delivery_service_manager.py b/tools/cli/services/k8s/delivery_service_manager.py index 5a9235f1..a39eb70a 100644 --- a/tools/cli/services/k8s/delivery_service_manager.py +++ b/tools/cli/services/k8s/delivery_service_manager.py @@ -11,9 +11,9 @@ @exponential_backoff(base_delay=5) -def get_argocd_token(user, password): +def get_argocd_token(user, password, endpoint="localhost:8080"): try: - response = requests.post("https://localhost:8080/api/v1/session", + response = requests.post(f"https://{endpoint}/api/v1/session", verify=False, headers={"Content-Type": "application/json"}, data=json.dumps({"username": user, "password": password}) @@ -28,9 +28,9 @@ def get_argocd_token(user, password): @exponential_backoff(base_delay=5) -def delete_application(app_name, token): +def delete_application(app_name, token, endpoint="localhost:8080"): try: - response = requests.delete(f"https://localhost:8080/api/v1/applications/{app_name}?cascade=true", + response = requests.delete(f"https://{endpoint}/api/v1/applications/{app_name}?cascade=true", verify=False, headers={ "Content-Type": "application/json", diff --git a/tools/cli/services/vcs/git_provider_manager.py b/tools/cli/services/vcs/git_provider_manager.py index e09c4fe3..39d05398 100644 --- a/tools/cli/services/vcs/git_provider_manager.py +++ b/tools/cli/services/vcs/git_provider_manager.py @@ -47,7 +47,7 @@ def create_runner_group_snippet(self) -> str: pass @abstractmethod - def get_organization_plan(self, organization_name: str) -> [str, GitSubscriptionPlans]: + def get_organization_plan(self, organization_name: str) -> GitSubscriptionPlans: """ Get active plan, if present :return: Subscription plan From cc10b038b47e74811341a1c1aae6133007fc259d Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Fri, 15 Dec 2023 15:14:56 +0100 Subject: [PATCH 12/49] fix wl argoproj deletion --- .../workloads/workload-template.yaml | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml index d9c3d23f..69d68c17 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml @@ -2,46 +2,47 @@ apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: - name: + name: '' namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io spec: sourceRepos: - destinations: - - namespace: wl--* + - namespace: 'wl--*' server: '*' clusterResourceWhitelist: - group: '*' kind: '*' roles: - # A role which provides read-only access to all applications in the project - - name: read-only - description: Read-only privileges to project - policies: - - p, proj::read-only, applications, get, /*, allow - groups: - - developers - - name: full-access - description: Full privileges to project for workload admins and developers groups - policies: - - p, proj::full-access, applications, *, /*, allow - groups: - - -admins - - -developers - + # A role which provides read-only access to all applications in the project + - name: read-only + description: Read-only privileges to project + policies: + - p, proj::read-only, applications, get, /*, allow + groups: + - developers + - name: full-access + description: Full privileges to project for workload admins and developers groups + policies: + - p, proj::full-access, applications, *, /*, allow + groups: + - -admins + - -developers --- apiVersion: v1 kind: Secret metadata: - name: -apps + name: '-apps' labels: argocd.argoproj.io/secret-type: repository type: Opaque stringData: # Project scoped - project: - name: - url: + project: '' + name: '' + url: '' --- apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet @@ -59,7 +60,7 @@ spec: metadata: name: '-{{path.basename}}' spec: - project: + project: '' source: repoURL: targetRevision: HEAD From 7db0cedd01eb649c1f29844616355b7fe5e75a66 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Fri, 15 Dec 2023 20:32:52 +0100 Subject: [PATCH 13/49] refactor wl create & destroy commands --- tools/cli/commands/setup.py | 2 +- tools/cli/commands/workload/create.py | 111 ++++------------- tools/cli/commands/workload/delete.py | 116 ++++++------------ tools/cli/common/command_utils.py | 46 ++----- tools/cli/services/platform_gitops.py | 116 ++++++++++++++++++ .../cli/services/vcs/git_provider_manager.py | 15 ++- .../cli/services/vcs/github/github_manager.py | 68 ++++++---- .../cli/services/vcs/gitlab/gitlab_manager.py | 8 +- 8 files changed, 251 insertions(+), 231 deletions(-) create mode 100644 tools/cli/services/platform_gitops.py diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 703b7fcf..6664175a 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -149,7 +149,7 @@ def setup( p.internals["GIT_USER_EMAIL"] = git_user_email p.fragments["# "] = git_man.create_tf_module_snippet() - git_subscription_plan = git_man.get_organization_plan(p.get_input_param(GIT_ORGANIZATION_NAME)) + git_subscription_plan = git_man.get_organization_plan() p.internals["GIT_SUBSCRIPTION_PLAN"] = bool(git_subscription_plan) if git_subscription_plan > 0: p.fragments["# "] = git_man.create_runner_group_snippet() diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index b24ba68e..456324dc 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -1,16 +1,11 @@ -import json -import os import webbrowser import click -from ghrepo import GHRepo -from git import Actor -from common.command_utils import str_to_kebab, update_gitops_repo, create_pr -from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ - LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER, LOCAL_CC_CLUSTER_WORKLOAD_FOLDER +from common.command_utils import str_to_kebab, init_git_provider, check_installation_presence from common.logging_config import configure_logging from common.state_store import StateStore +from services.platform_gitops import PlatformGitOpsRepo @click.command() @@ -30,10 +25,7 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: # Set up global logger configure_logging(verbosity) - if not os.path.exists(LOCAL_FOLDER): - raise click.ClickException("CG DevX metadata does not exist on this machine") - - p: StateStore = StateStore() + check_installation_presence() wl_name = str_to_kebab(wl_name) @@ -49,90 +41,37 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: wl_repo_name = str_to_kebab(wl_repo_name) wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) - if not os.path.exists(LOCAL_GITOPS_FOLDER): - raise click.ClickException("GitOps repo does not exist") + p: StateStore = StateStore() main_branch = "main" - - repo = update_gitops_repo() + git_man = init_git_provider(p) + gor = PlatformGitOpsRepo(git_man, + author_name=p.internals["GIT_USER_NAME"], + author_email=p.internals["GIT_USER_EMAIL"], + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) + # update repo just in case + gor.update() # create new branch branch_name = f"feature/{wl_name}-init" - current = repo.create_head(branch_name) - current.checkout() - - # repos - with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "r") as file: - vcs_tf_vars = json.load(file) - - vcs_tf_vars["workloads"][wl_name] = { - "description": f"CG DevX {wl_name} workload definition", - "repos": {} - } - vcs_tf_vars["workloads"][wl_name]["repos"][wl_repo_name] = {} - vcs_tf_vars["workloads"][wl_name]["repos"][wl_gitops_repo_name] = { - "atlantis_enabled": True, - } - with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(vcs_tf_vars, indent=2)) - - # secrets - with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "r") as file: - secrets_tf_vars = json.load(file) - - secrets_tf_vars["workloads"][wl_name] = {} - - with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(secrets_tf_vars, indent=2)) - - # core services - with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "r") as file: - services_tf_vars = json.load(file) - - services_tf_vars["workloads"][wl_name] = {} - - with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(services_tf_vars, indent=2)) - - # prepare ArgoCD manifest - wl_gitops_repo = GHRepo(p.parameters[""], wl_gitops_repo_name) - params = { - "": wl_name, - "": wl_gitops_repo.ssh_url, - } - - workload_template_file = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / "workload-template.yaml" - with open(workload_template_file, "r") as file: - data = file.read() - for k, v in params.items(): - data = data.replace(k, v) - - workload_file = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml" - with open(workload_file, "w") as file: - file.write(data) - - # commit and prepare a PR - ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]}' - with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + gor.create_branch(branch_name) - repo.git.add(all=True) - author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) - repo.index.commit("initial", author=author, committer=author) + gor.add_workload(wl_name, wl_repo_name, wl_gitops_repo_name) - repo.remotes.origin.push(repo.active_branch.name) - - pr_url = create_pr(p.parameters[""], p.parameters[""], - p.internals["GIT_ACCESS_TOKEN"], - branch_name, main_branch, - f"introduce {wl_name}", - f"Add default secrets, user and default repository structure.") - - if not pr_url: - raise click.ClickException("Could not create PR") - else: + # commit and prepare a PR + gor.upload_changes() + + try: + pr_url = gor.create_pr(p.parameters[""], + branch_name, + main_branch, + f"introduce {wl_name}", + f"Add default secrets, user and default repository structure.") webbrowser.open(pr_url, autoraise=False) + except Exception as e: + raise click.ClickException("Could not create PR") - repo.heads.main.checkout() + gor.switch_to_branch() click.echo("Creating workload GitOps code. Done!") return True diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index 8a80dc1e..a61c1c00 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -1,19 +1,19 @@ -import json +import os import os import shutil import webbrowser import click from ghrepo import GHRepo -from git import Repo, GitError, Actor +from git import Repo, GitError -from common.command_utils import str_to_kebab, update_gitops_repo, prepare_cloud_provider_auth_env_vars, set_envs, \ - create_pr -from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ - LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_FOLDER, LOCAL_CC_CLUSTER_WORKLOAD_FOLDER +from common.command_utils import str_to_kebab, prepare_cloud_provider_auth_env_vars, set_envs, \ + check_installation_presence, init_git_provider +from common.const.common_path import LOCAL_FOLDER from common.const.parameter_names import GIT_ACCESS_TOKEN, GIT_ORGANIZATION_NAME from common.logging_config import configure_logging from common.state_store import StateStore +from services.platform_gitops import PlatformGitOpsRepo from services.tf_wrapper import TfWrapper @@ -36,10 +36,7 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb # Set up global logger configure_logging(verbosity) - if not os.path.exists(LOCAL_FOLDER): - raise click.ClickException("CG DevX metadata does not exist on this machine") - - p: StateStore = StateStore() + check_installation_presence() wl_name = str_to_kebab(wl_name) @@ -48,54 +45,22 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) - if not os.path.exists(LOCAL_GITOPS_FOLDER): - raise click.ClickException("GitOps repo does not exist") - + p: StateStore = StateStore() main_branch = "main" - repo = update_gitops_repo() + git_man = init_git_provider(p) + gor = PlatformGitOpsRepo(git_man, + author_name=p.internals["GIT_USER_NAME"], + author_email=p.internals["GIT_USER_EMAIL"], + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"], ) + # update repo just in case + gor.update() # create new branch branch_name = f"feature/{wl_name}-destroy" - current = repo.create_head(branch_name) - current.checkout() - - # repos - with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "r") as file: - vcs_tf_vars = json.load(file) - - if wl_name in vcs_tf_vars["workloads"]: - del vcs_tf_vars["workloads"][wl_name] - - with open(LOCAL_TF_FOLDER_VCS / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(vcs_tf_vars, indent=2)) - - # secrets - with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "r") as file: - secrets_tf_vars = json.load(file) - - if wl_name in secrets_tf_vars["workloads"]: - del secrets_tf_vars["workloads"][wl_name] + gor.create_branch(branch_name) - with open(LOCAL_TF_FOLDER_SECRETS_MANAGER / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(secrets_tf_vars, indent=2)) - - # core services - with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "r") as file: - services_tf_vars = json.load(file) - - if wl_name in services_tf_vars["workloads"]: - del services_tf_vars["workloads"][wl_name] - - with open(LOCAL_TF_FOLDER_CORE_SERVICES / "terraform.tfvars.json", "w") as file: - file.write(json.dumps(services_tf_vars, indent=2)) - - # delete ArgoCD manifest - wl_argo_manifest = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml" - if os.path.exists(wl_argo_manifest): - os.remove(wl_argo_manifest) - - key_path = p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] + gor.rm_workload(wl_name) # destroy resources if destroy_resources: @@ -112,7 +77,7 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb shutil.rmtree(wl_gitops_repo_folder) os.makedirs(wl_gitops_repo_folder) - + key_path = p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] try: wl_gitops_repo = Repo.clone_from( GHRepo(p.parameters[""], wl_gitops_repo_name).ssh_url, @@ -135,44 +100,39 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb # set envs as required by tf set_envs(tf_env_vars) - click.echo("Destroying WL secrets...") + if os.path.exists(wl_gitops_repo_folder / "terraform"): + click.echo("Destroying WL secrets...") - tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/secrets") - tf_wrapper.init() - tf_wrapper.destroy() + tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/secrets") + tf_wrapper.init() + tf_wrapper.destroy() - click.echo("Destroying WL secrets. Done!") + click.echo("Destroying WL secrets. Done!") - click.echo("Destroying WL cloud resources...") + click.echo("Destroying WL cloud resources...") - tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/infrastructure") - tf_wrapper.init() - tf_wrapper.destroy() + tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/infrastructure") + tf_wrapper.init() + tf_wrapper.destroy() - click.echo("Destroying WL cloud resources. Done!") + click.echo("Destroying WL cloud resources. Done!") # remove temp folder shutil.rmtree(temp_folder) # commit and prepare a PR - ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {key_path}' - with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): - - repo.git.add(all=True) - author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) - repo.index.commit("initial", author=author, committer=author) - - repo.remotes.origin.push(repo.active_branch.name) + gor.upload_changes() - pr_url = create_pr(p.parameters[""], p.parameters[""], - p.internals["GIT_ACCESS_TOKEN"], branch_name, main_branch, f"remove {wl_name}", - f"Remove default secrets, user and repository structure.") - if not pr_url: - raise click.ClickException("Could not create PR") - else: + try: + pr_url = gor.create_pr(p.parameters[""], + branch_name, main_branch, + f"remove {wl_name}", + f"Remove default secrets, user and repository structure.") webbrowser.open(pr_url, autoraise=False) + except Exception as e: + raise click.ClickException("Could not create PR") - repo.heads.main.checkout() + gor.switch_to_branch() click.echo("Deleting workload GitOps code. Done!") return True diff --git a/tools/cli/common/command_utils.py b/tools/cli/common/command_utils.py index 67e5597d..5e69afd6 100644 --- a/tools/cli/common/command_utils.py +++ b/tools/cli/common/command_utils.py @@ -1,20 +1,19 @@ -import json +import os import os import time from re import sub +import click import requests -from git import Repo from requests import HTTPError -from common.const.common_path import LOCAL_GITOPS_FOLDER +from common.const.common_path import LOCAL_GITOPS_FOLDER, LOCAL_FOLDER from common.const.parameter_names import CLOUD_REGION, CLOUD_PROFILE, CLOUD_ACCOUNT_ACCESS_KEY, \ CLOUD_ACCOUNT_ACCESS_SECRET, DNS_REGISTRAR_ACCESS_KEY, DNS_REGISTRAR_ACCESS_SECRET, GIT_ACCESS_TOKEN, \ GIT_ORGANIZATION_NAME from common.enums.cloud_providers import CloudProviders from common.enums.dns_registrars import DnsRegistrars from common.enums.git_providers import GitProviders -from common.logging_config import logger from common.retry_decorator import exponential_backoff from common.state_store import StateStore from services.cloud.aws.aws_manager import AWSManager @@ -139,38 +138,9 @@ def str_to_kebab(string: str): lambda match: ' ' + match.group(0).lower(), string)).split()) -def update_gitops_repo(): - repo = Repo(LOCAL_GITOPS_FOLDER) - # clean stale branches - repo.remotes.origin.fetch(prune=True) - # update repo just in case - repo.heads.main.checkout() - repo.remotes.origin.pull(repo.active_branch) - return repo - - -def create_pr(org_name: str, repo_name: str, token: str, head_branch: str, base_branch: str, title: str, - body: str) -> str: - git_pulls_api = f"https://api.github.com/repos/{org_name}/{repo_name}/pulls" - headers = { - "Authorization": f"token {token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28" - } - payload = { - "title": title, - "body": body, - "head": head_branch, - "base": base_branch - } - res = requests.post( - git_pulls_api, - headers=headers, - data=json.dumps(payload)) - - if not res.ok: - logger.error("GitHub API Request Failed: {0}".format(res.text)) - return - data = json.loads(res.text) +def check_installation_presence(): + if not os.path.exists(LOCAL_FOLDER): + raise click.ClickException("CG DevX metadata does not exist on this machine") - return data["html_url"] + if not os.path.exists(LOCAL_GITOPS_FOLDER): + raise click.ClickException("GitOps repo does not exist") diff --git a/tools/cli/services/platform_gitops.py b/tools/cli/services/platform_gitops.py new file mode 100644 index 00000000..b4b12325 --- /dev/null +++ b/tools/cli/services/platform_gitops.py @@ -0,0 +1,116 @@ +import json +import os + +from ghrepo import GHRepo +from git import Repo, Actor + +from common.const.common_path import LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ + LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_CC_CLUSTER_WORKLOAD_FOLDER +from common.const.const import FALLBACK_AUTHOR_NAME, FALLBACK_AUTHOR_EMAIL +from services.vcs.git_provider_manager import GitProviderManager + + +class PlatformGitOpsRepo: + def __init__(self, git_man: GitProviderManager, key_path: str = None, author_name: str = FALLBACK_AUTHOR_NAME, + author_email: str = FALLBACK_AUTHOR_EMAIL): + self._repo = Repo(LOCAL_GITOPS_FOLDER) + self._git_man = git_man + self._ssh_key_path = key_path + self._author_name = author_name + self._author_email = author_email + + def update(self): + # clean stale branches + self._repo.remotes.origin.fetch(prune=True) + self._repo.heads.main.checkout() + self._repo.remotes.origin.pull(self._repo.active_branch) + + def create_branch(self, branch_name): + current = self._repo.create_head(branch_name) + current.checkout() + + def upload_changes(self): + ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {self._ssh_key_path}' + with self._repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + self._repo.git.add(all=True) + author = Actor(name=self._author_name, email=self._author_email) + self._repo.index.commit("initial", author=author, committer=author) + + self._repo.remotes.origin.push(self._repo.active_branch.name) + + def switch_to_branch(self, branch_name: str = "main"): + self._repo.heads[branch_name].checkout() + # if branch_name: + # pass + # else: + # self._repo.heads.main.checkout() + + def create_pr(self, repo_name: str, head_branch: str, base_branch: str, title: str, body: str) -> str: + return self._git_man.create_pr(repo_name, head_branch, base_branch, title, body) + + def add_workload(self, wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str): + # repos + self._add_wl_vars(LOCAL_TF_FOLDER_VCS, wl_name, { + "description": f"CG DevX {wl_name} workload definition", + "repos": { + wl_repo_name: {}, + wl_gitops_repo_name: { + "atlantis_enabled": True, + } + } + }) + # secrets + self._add_wl_vars(LOCAL_TF_FOLDER_SECRETS_MANAGER, wl_name, {}) + # core services + self._add_wl_vars(LOCAL_TF_FOLDER_CORE_SERVICES, wl_name, {}) + + # prepare ArgoCD manifest + wl_gitops_repo = GHRepo(self._git_man.organization, wl_gitops_repo_name) + params = { + "": wl_name, + "": wl_gitops_repo.ssh_url, + } + + workload_template_file = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / "workload-template.yaml" + with open(workload_template_file, "r") as file: + data = file.read() + for k, v in params.items(): + data = data.replace(k, v) + + workload_file = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml" + with open(workload_file, "w") as file: + file.write(data) + + def rm_workload(self, wl_name: str): + # repos + self._rm_wl_vars(LOCAL_TF_FOLDER_VCS, wl_name) + # secrets + self._rm_wl_vars(LOCAL_TF_FOLDER_SECRETS_MANAGER, wl_name) + # core services + self._rm_wl_vars(LOCAL_TF_FOLDER_CORE_SERVICES, wl_name) + + # delete ArgoCD manifest + wl_argo_manifest = LOCAL_CC_CLUSTER_WORKLOAD_FOLDER / f"{wl_name}.yaml" + if os.path.exists(wl_argo_manifest): + os.remove(wl_argo_manifest) + + @staticmethod + def _add_wl_vars(tf_module_path, wl_name: str, payload: dict = {}): + with open(tf_module_path / "terraform.tfvars.json", "r") as file: + services_tf_vars = json.load(file) + + services_tf_vars["workloads"][wl_name] = payload + + with open(tf_module_path / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(services_tf_vars, indent=2)) + + @staticmethod + def _rm_wl_vars(tf_module_path, wl_name: str): + with open(tf_module_path / "terraform.tfvars.json", "r") as file: + vcs_tf_vars = json.load(file) + + if wl_name in vcs_tf_vars["workloads"]: + del vcs_tf_vars["workloads"][wl_name] + + with open(tf_module_path / "terraform.tfvars.json", "w") as file: + file.write(json.dumps(vcs_tf_vars, indent=2)) diff --git a/tools/cli/services/vcs/git_provider_manager.py b/tools/cli/services/vcs/git_provider_manager.py index 39d05398..d9643d1b 100644 --- a/tools/cli/services/vcs/git_provider_manager.py +++ b/tools/cli/services/vcs/git_provider_manager.py @@ -5,11 +5,14 @@ class GitProviderManager(ABC): """Git provider wrapper to standardise Git management.""" + @property + def organization(self) -> str: + pass @abstractmethod def check_repository_existence(self, name: str = "GitOps"): """ - Check if repository exists + Check if the repository exists :return: True or False """ pass @@ -47,9 +50,17 @@ def create_runner_group_snippet(self) -> str: pass @abstractmethod - def get_organization_plan(self, organization_name: str) -> GitSubscriptionPlans: + def get_organization_plan(self) -> GitSubscriptionPlans: """ Get active plan, if present :return: Subscription plan """ pass + + @abstractmethod + def create_pr(self, repo_name: str, head_branch: str, base_branch: str, title: str, body: str) -> str: + """ + Create a pull request + :return: Pull request URL + """ + pass diff --git a/tools/cli/services/vcs/github/github_manager.py b/tools/cli/services/vcs/github/github_manager.py index 774c23c0..15d9132b 100644 --- a/tools/cli/services/vcs/github/github_manager.py +++ b/tools/cli/services/vcs/github/github_manager.py @@ -20,17 +20,17 @@ def __init__(self, token: str = None, org_name: str = None): "delete_repo", "repo" } + @property + def organization(self) -> str: + return self.__org_name + @trace() def check_repository_existence(self, name: str = "GitOps"): """ - Check if repository exists + Check if the repository exists :return: True or False """ - headers = { - 'Authorization': f'token {self.__token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28', - } + headers = self._generate_headers() try: response = requests.get(f'https://api.github.com/repos/{self.__org_name}/{name}', headers=headers) if response.status_code == requests.codes["not_found"]: @@ -46,11 +46,7 @@ def evaluate_permissions(self): Check if provided credentials have required permissions :return: True or False """ - headers = { - 'Authorization': f'token {self.__token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28', - } + headers = self._generate_headers() try: response = requests.head('https://api.github.com', headers=headers) if "x-oauth-scopes" not in response.headers: @@ -67,11 +63,7 @@ def get_current_user_info(self): Get authenticated user info :return: Login, Name, Email """ - headers = { - 'Authorization': f'token {self.__token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28', - } + headers = self._generate_headers() try: response = requests.get('https://api.github.com/user', headers=headers) res = json.loads(response.text) @@ -98,19 +90,14 @@ def create_runner_group_snippet(self) -> str: return 'group: ' @trace() - def get_organization_plan(self, organization_name: str) -> GitSubscriptionPlans: + def get_organization_plan(self) -> GitSubscriptionPlans: """ Get active plan, if present :return: Plan name """ - - headers = { - 'Authorization': f'token {self.__token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28', - } + headers = self._generate_headers() try: - response = requests.get(f'https://api.github.com/orgs/{organization_name}', headers=headers) + response = requests.get(f'https://api.github.com/orgs/{self.__org_name}', headers=headers) if response.ok: res = json.loads(response.text) @@ -127,3 +114,36 @@ def get_organization_plan(self, organization_name: str) -> GitSubscriptionPlans: raise Exception("Org not found") except HTTPError as e: raise e + + @trace() + def create_pr(self, repo_name: str, head_branch: str, base_branch: str, title: str, body: str) -> str: + git_pulls_api = f"https://api.github.com/repos/{self.__org_name}/{repo_name}/pulls" + headers = self._generate_headers() + payload = { + "title": title, + "body": body, + "head": head_branch, + "base": base_branch + } + try: + res = requests.post( + git_pulls_api, + headers=headers, + data=json.dumps(payload)) + + if not res.ok: + raise Exception("GitHub API Request Failed: {0}".format(res.text)) + + data = json.loads(res.text) + + return data["html_url"] + except HTTPError as e: + raise e + + def _generate_headers(self): + headers = { + "Authorization": f"token {self.__token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + return headers diff --git a/tools/cli/services/vcs/gitlab/gitlab_manager.py b/tools/cli/services/vcs/gitlab/gitlab_manager.py index 1417550a..afdea719 100644 --- a/tools/cli/services/vcs/gitlab/gitlab_manager.py +++ b/tools/cli/services/vcs/gitlab/gitlab_manager.py @@ -29,13 +29,17 @@ def _get_headers(self) -> Dict[str, str]: """ Construct and return headers for GitLab API requests. - :return: Dictionary containing necessary headers for the API call. + :return: Dictionary containing the necessary headers for the API call. """ return { 'Authorization': f'Bearer {self.__token}', 'Accept': 'application/json' } + @property + def organization(self) -> str: + return self.__group_name + @trace() def check_repository_existence(self, name: str = "GitOps") -> bool: """ @@ -173,7 +177,7 @@ def create_runner_group_snippet(self) -> str: return '' @trace() - def get_organization_plan(self, organization_name: str) -> GitSubscriptionPlans: + def get_organization_plan(self) -> GitSubscriptionPlans: """ Get active plan, if present :return: Plan name From 548a4b537b7798678bc95be8fd47a4757ff58dac Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Mon, 18 Dec 2023 17:47:19 +0100 Subject: [PATCH 14/49] refactor: workload bootstrap --- tools/cli/commands/destroy.py | 2 +- tools/cli/commands/setup.py | 4 +- tools/cli/commands/workload/bootstrap.py | 138 ++++++------------ tools/cli/commands/workload/create.py | 4 +- tools/cli/commands/workload/delete.py | 31 ++-- tools/cli/common/const/common_path.py | 1 + tools/cli/common/const/const.py | 4 + tools/cli/common/{ => utils}/command_utils.py | 2 +- tools/cli/services/platform_gitops.py | 16 +- ...anager.py => platform_template_manager.py} | 20 +-- .../services/wl_gitops_template_manager.py | 98 +++++++++++++ tools/cli/services/wl_template_manager.py | 102 +++++++++++++ 12 files changed, 291 insertions(+), 131 deletions(-) rename tools/cli/common/{ => utils}/command_utils.py (98%) rename tools/cli/services/{template_manager.py => platform_template_manager.py} (91%) create mode 100644 tools/cli/services/wl_gitops_template_manager.py create mode 100644 tools/cli/services/wl_template_manager.py diff --git a/tools/cli/commands/destroy.py b/tools/cli/commands/destroy.py index 2fb7d8c9..7955cd22 100644 --- a/tools/cli/commands/destroy.py +++ b/tools/cli/commands/destroy.py @@ -5,7 +5,7 @@ import portforward import urllib3 -from common.command_utils import init_cloud_provider, prepare_cloud_provider_auth_env_vars, set_envs, unset_envs, wait +from common.utils.command_utils import init_cloud_provider, prepare_cloud_provider_auth_env_vars, set_envs, unset_envs, wait from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_HOSTING_PROVIDER, LOCAL_FOLDER, \ LOCAL_STATE_FILE from common.const.namespaces import ARGOCD_NAMESPACE diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 6664175a..41e2c0fd 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -6,7 +6,7 @@ import portforward import yaml -from common.command_utils import init_cloud_provider, init_git_provider, prepare_cloud_provider_auth_env_vars, \ +from common.utils.command_utils import init_cloud_provider, init_git_provider, prepare_cloud_provider_auth_env_vars, \ set_envs, unset_envs, wait, wait_http_endpoint_readiness from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_HOSTING_PROVIDER, \ LOCAL_TF_FOLDER_SECRETS_MANAGER, LOCAL_TF_FOLDER_USERS, LOCAL_TF_FOLDER_CORE_SERVICES @@ -34,7 +34,7 @@ from services.k8s.k8s import KubeClient, write_ca_cert from services.k8s.kctl_wrapper import KctlWrapper from services.keys.key_manager import KeyManager -from services.template_manager import GitOpsTemplateManager +from services.platform_template_manager import GitOpsTemplateManager from services.tf_wrapper import TfWrapper from services.vcs.git_provider_manager import GitProviderManager diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 95548e64..c8861f18 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -1,16 +1,17 @@ import os -import pathlib import shutil import click from ghrepo import GHRepo from git import Repo, GitError, Actor -from common.command_utils import str_to_kebab, init_cloud_provider -from common.const.common_path import LOCAL_GITOPS_FOLDER, LOCAL_FOLDER -from common.logging_config import configure_logging, logger +from common.const.common_path import LOCAL_GITOPS_FOLDER, LOCAL_FOLDER, LOCAL_WORKLOAD_TEMP_FOLDER +from common.const.const import WL_REPOSITORY_URL, WL_GITOPS_REPOSITORY_URL +from common.logging_config import configure_logging from common.state_store import StateStore -from services.template_manager import GitOpsTemplateManager +from common.utils.command_utils import str_to_kebab, init_cloud_provider +from services.wl_gitops_template_manager import WorkloadGitOpsTemplateManager +from services.wl_template_manager import WorkloadTemplateManager @click.command() @@ -70,75 +71,61 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp if not wl_template_branch: wl_template_branch = "main" if not wl_template_url: - wl_template_url = "git@github.com:CloudGeometry/cg-devx-wl-template.git" + wl_template_url = WL_REPOSITORY_URL if not wl_gitops_template_branch: wl_gitops_template_branch = "main" if not wl_gitops_template_url: - wl_gitops_template_url = "git@github.com:CloudGeometry/cg-devx-wl-gitops-template.git" + wl_gitops_template_url = WL_GITOPS_REPOSITORY_URL if not os.path.exists(LOCAL_GITOPS_FOLDER): raise click.ClickException("GitOps repo does not exist") - temp_folder = LOCAL_FOLDER / ".wl_tmp" + if os.path.exists(LOCAL_WORKLOAD_TEMP_FOLDER): + shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) - if os.path.exists(temp_folder): - shutil.rmtree(temp_folder) + os.makedirs(LOCAL_WORKLOAD_TEMP_FOLDER) - os.makedirs(temp_folder) - - key_path = p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] + wl_man = WorkloadTemplateManager( + org_name=p.parameters[""], + repo_name=wl_repo_name, + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) # workload repo - wl_repo_folder = temp_folder / wl_repo_name - os.makedirs(wl_repo_folder) - - try: - wl_repo = Repo.clone_from(GHRepo(p.parameters[""], wl_repo_name).ssh_url, - wl_repo_folder, - env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) - except GitError as e: + if not (wl_man.clone_wl()): raise click.ClickException("Failed cloning repo") - wl_template_repo_folder = temp_folder / GHRepo.parse(wl_template_url).name - os.makedirs(wl_template_repo_folder) - - try: - wl_template_repo = Repo.clone_from(wl_template_url, - wl_template_repo_folder, - branch=wl_template_branch, - env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) - except GitError as e: + if not (wl_man.clone_template()): raise click.ClickException("Failed cloning template repo") - shutil.rmtree(wl_template_repo_folder / ".git") - shutil.copytree(wl_template_repo_folder, wl_repo_folder, dirs_exist_ok=True) - shutil.rmtree(wl_template_repo_folder) - shutil.move(wl_repo_folder / "wl-service-name", wl_repo_folder / wl_svc_name) + wl_man.bootstrap([wl_svc_name]) wl_repo_params = { "": wl_name, "": wl_svc_name, } - for root, dirs, files in os.walk(wl_repo_folder): - for name in files: - if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): - file_path = os.path.join(root, name) - with open(file_path, "r") as file: - data = file.read() - for k, v in wl_repo_params.items(): - data = data.replace(k, v) - with open(file_path, "w") as file: - file.write(data) - - wl_repo.git.add(all=True) - author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) - wl_repo.index.commit("initial", author=author, committer=author) - - wl_repo.remotes.origin.push(wl_repo.active_branch.name) + + wl_man.parametrise(wl_repo_params) + + wl_man.upload(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) + + wl_man.cleanup() # wl gitops repo cloud_man, dns_man = init_cloud_provider(p) + wl_ops_man = WorkloadGitOpsTemplateManager( + org_name=p.parameters[""], + repo_name=wl_gitops_repo_name, + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) + + # workload repo + if not (wl_ops_man.clone_wl()): + raise click.ClickException("Failed cloning repo") + + if not (wl_ops_man.clone_template()): + raise click.ClickException("Failed cloning template repo") + + wl_ops_man.bootstrap() wl_gitops_repo_params = { "": wl_name, @@ -149,56 +136,21 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp "# ": cloud_man.create_k8s_cluster_role_mapping_snippet(), "": "[Put your workload service role mapping]", "# ": cloud_man.create_additional_labels(), - "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], f"workloads/{wl_name}/secrets"), - "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], f"workloads/{wl_name}/hosting_provider"), + "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], + f"workloads/{wl_name}/secrets"), + "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], + f"workloads/{wl_name}/hosting_provider"), "# ": cloud_man.create_hosting_provider_snippet(), } - wl_gitops_repo_folder = temp_folder / wl_gitops_repo_name - os.makedirs(wl_gitops_repo_folder) - - try: - wl_gitops_repo = Repo.clone_from(GHRepo(p.parameters[""], wl_gitops_repo_name).ssh_url, - wl_gitops_repo_folder, - env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) - except GitError as e: - raise click.ClickException("Failed cloning repo") - - wl_gitops_template_repo_folder = temp_folder / GHRepo.parse(wl_gitops_template_url).name - os.makedirs(wl_gitops_template_repo_folder) - - try: - wl_gitops_template_repo = Repo.clone_from(wl_gitops_template_url, - wl_gitops_template_repo_folder, - branch=wl_gitops_template_branch, - env={ - "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) - except GitError as e: - raise click.ClickException("Failed cloning template repo") - - shutil.rmtree(wl_gitops_template_repo_folder / ".git") - shutil.copytree(wl_gitops_template_repo_folder, wl_gitops_repo_folder, dirs_exist_ok=True) - shutil.rmtree(wl_gitops_template_repo_folder) - - for root, dirs, files in os.walk(wl_gitops_repo_folder): - for name in files: - if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): - file_path = os.path.join(root, name) - with open(file_path, "r") as file: - data = file.read() - for k, v in wl_gitops_repo_params.items(): - data = data.replace(k, v) - with open(file_path, "w") as file: - file.write(data) + wl_ops_man.parametrise(wl_gitops_repo_params) - wl_gitops_repo.git.add(all=True) - author = Actor(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) - wl_gitops_repo.index.commit("initial", author=author, committer=author) + wl_ops_man.upload() - wl_gitops_repo.remotes.origin.push(wl_gitops_repo.active_branch.name) + wl_ops_man.cleanup() # remove temp folder - shutil.rmtree(temp_folder) + shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) click.echo("Bootstrapping workload. Done!") return True diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 456324dc..5c82a8ad 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -2,7 +2,7 @@ import click -from common.command_utils import str_to_kebab, init_git_provider, check_installation_presence +from common.utils.command_utils import str_to_kebab, init_git_provider, check_installation_presence from common.logging_config import configure_logging from common.state_store import StateStore from services.platform_gitops import PlatformGitOpsRepo @@ -54,6 +54,8 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: # create new branch branch_name = f"feature/{wl_name}-init" + if gor.branch_exist(branch_name): + raise click.ClickException("Branch already exist, please use different workload name. ") gor.create_branch(branch_name) gor.add_workload(wl_name, wl_repo_name, wl_gitops_repo_name) diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index a61c1c00..21f3509c 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -1,5 +1,4 @@ import os -import os import shutil import webbrowser @@ -7,14 +6,15 @@ from ghrepo import GHRepo from git import Repo, GitError -from common.command_utils import str_to_kebab, prepare_cloud_provider_auth_env_vars, set_envs, \ +from common.utils.command_utils import str_to_kebab, prepare_cloud_provider_auth_env_vars, set_envs, \ check_installation_presence, init_git_provider -from common.const.common_path import LOCAL_FOLDER +from common.const.common_path import LOCAL_FOLDER, LOCAL_WORKLOAD_TEMP_FOLDER from common.const.parameter_names import GIT_ACCESS_TOKEN, GIT_ORGANIZATION_NAME from common.logging_config import configure_logging from common.state_store import StateStore from services.platform_gitops import PlatformGitOpsRepo from services.tf_wrapper import TfWrapper +from services.wl_gitops_template_manager import WorkloadGitOpsTemplateManager @click.command() @@ -58,6 +58,8 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb # create new branch branch_name = f"feature/{wl_name}-destroy" + if gor.branch_exist(branch_name): + raise click.ClickException("Branch already exist, please use different workload name. ") gor.create_branch(branch_name) gor.rm_workload(wl_name) @@ -70,21 +72,12 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb "This will delete all the resources defined in workload IaC. Please confirm to continue") if tf_destroy_confirmation: - temp_folder = LOCAL_FOLDER / ".wl_tmp" - wl_gitops_repo_folder = temp_folder / f"{wl_name}-gitops" - - if os.path.exists(wl_gitops_repo_folder): - shutil.rmtree(wl_gitops_repo_folder) - - os.makedirs(wl_gitops_repo_folder) - key_path = p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] - try: - wl_gitops_repo = Repo.clone_from( - GHRepo(p.parameters[""], wl_gitops_repo_name).ssh_url, - wl_gitops_repo_folder, - env={"GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {key_path}'}) - except GitError as e: - raise click.ClickException("Failed cloning repo") + wl_ops_man = WorkloadGitOpsTemplateManager( + org_name=p.parameters[""], + repo_name=wl_gitops_repo_name, + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) + + wl_gitops_repo_folder = wl_ops_man.clone_wl() cloud_provider_auth_env_vars = prepare_cloud_provider_auth_env_vars(p) @@ -118,7 +111,7 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb click.echo("Destroying WL cloud resources. Done!") # remove temp folder - shutil.rmtree(temp_folder) + shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) # commit and prepare a PR gor.upload_changes() diff --git a/tools/cli/common/const/common_path.py b/tools/cli/common/const/common_path.py index 33b1f93c..76aa66cd 100644 --- a/tools/cli/common/const/common_path.py +++ b/tools/cli/common/const/common_path.py @@ -15,3 +15,4 @@ LOCAL_KCTL_TOOL = LOCAL_TOOLS_FOLDER / "kubectl" LOCAL_STATE_FILE = LOCAL_FOLDER / "state.yaml" LOCAL_CC_CLUSTER_WORKLOAD_FOLDER = LOCAL_GITOPS_FOLDER / "gitops-pipelines/delivery/clusters/cc-cluster/workloads" +LOCAL_WORKLOAD_TEMP_FOLDER = LOCAL_FOLDER / ".wl_tmp" diff --git a/tools/cli/common/const/const.py b/tools/cli/common/const/const.py index 8fa0cb70..dba39262 100644 --- a/tools/cli/common/const/const.py +++ b/tools/cli/common/const/const.py @@ -12,3 +12,7 @@ ARGOCD_REGISTRY_APP_PATH = "gitops-pipelines/delivery/clusters/cc-cluster/core-services" KUBECTL_VERSION = "1.27.4" TERRAFORM_VERSION = "1.6.4" +WL_REPOSITORY_URL = "git@github.com:CloudGeometry/cg-devx-wl-template.git" +WL_REPOSITORY_BRANCH = "main" +WL_GITOPS_REPOSITORY_URL = "git@github.com:CloudGeometry/cg-devx-wl-gitops-template.git" +WL_GITOPS_REPOSITORY_BRANCH = "main" diff --git a/tools/cli/common/command_utils.py b/tools/cli/common/utils/command_utils.py similarity index 98% rename from tools/cli/common/command_utils.py rename to tools/cli/common/utils/command_utils.py index 5e69afd6..1f6c00b4 100644 --- a/tools/cli/common/command_utils.py +++ b/tools/cli/common/utils/command_utils.py @@ -40,7 +40,7 @@ def init_cloud_provider(state: StateStore) -> tuple[CloudProviderManager, DNSMan # check if cloud native DNS registrar is selected if state.dns_registrar == DnsRegistrars.Route53: - # Note!: Route53 is initialised with AWS Cloud Provider + # Note!: Route53 is initialized with AWS Cloud Provider if state.get_input_param(DNS_REGISTRAR_ACCESS_KEY) is None and state.get_input_param( DNS_REGISTRAR_ACCESS_SECRET) is None: # initialize with cloud account permissions diff --git a/tools/cli/services/platform_gitops.py b/tools/cli/services/platform_gitops.py index b4b12325..80fe85fa 100644 --- a/tools/cli/services/platform_gitops.py +++ b/tools/cli/services/platform_gitops.py @@ -7,6 +7,7 @@ from common.const.common_path import LOCAL_GITOPS_FOLDER, LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_SECRETS_MANAGER, \ LOCAL_TF_FOLDER_CORE_SERVICES, LOCAL_CC_CLUSTER_WORKLOAD_FOLDER from common.const.const import FALLBACK_AUTHOR_NAME, FALLBACK_AUTHOR_EMAIL +from common.tracing_decorator import trace from services.vcs.git_provider_manager import GitProviderManager @@ -19,16 +20,23 @@ def __init__(self, git_man: GitProviderManager, key_path: str = None, author_nam self._author_name = author_name self._author_email = author_email + @trace() def update(self): # clean stale branches self._repo.remotes.origin.fetch(prune=True) self._repo.heads.main.checkout() self._repo.remotes.origin.pull(self._repo.active_branch) + @trace() + def branch_exist(self, branch_name): + return branch_name in self._repo.branches + + @trace() def create_branch(self, branch_name): current = self._repo.create_head(branch_name) current.checkout() + @trace() def upload_changes(self): ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {self._ssh_key_path}' with self._repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): @@ -38,16 +46,15 @@ def upload_changes(self): self._repo.remotes.origin.push(self._repo.active_branch.name) + @trace() def switch_to_branch(self, branch_name: str = "main"): self._repo.heads[branch_name].checkout() - # if branch_name: - # pass - # else: - # self._repo.heads.main.checkout() + @trace() def create_pr(self, repo_name: str, head_branch: str, base_branch: str, title: str, body: str) -> str: return self._git_man.create_pr(repo_name, head_branch, base_branch, title, body) + @trace() def add_workload(self, wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str): # repos self._add_wl_vars(LOCAL_TF_FOLDER_VCS, wl_name, { @@ -81,6 +88,7 @@ def add_workload(self, wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str with open(workload_file, "w") as file: file.write(data) + @trace() def rm_workload(self, wl_name: str): # repos self._rm_wl_vars(LOCAL_TF_FOLDER_VCS, wl_name) diff --git a/tools/cli/services/template_manager.py b/tools/cli/services/platform_template_manager.py similarity index 91% rename from tools/cli/services/template_manager.py rename to tools/cli/services/platform_template_manager.py index 785c18c6..5383774d 100644 --- a/tools/cli/services/template_manager.py +++ b/tools/cli/services/platform_template_manager.py @@ -19,17 +19,17 @@ class GitOpsTemplateManager: def __init__(self, gitops_template_url: str = None, gitops_template_branch: str = None, token=None): if gitops_template_url is None: - self.__url = GITOPS_REPOSITORY_URL + self._url = GITOPS_REPOSITORY_URL else: - self.__url = gitops_template_url + self._url = gitops_template_url if gitops_template_branch is None: - self.__branch = GITOPS_REPOSITORY_BRANCH + self._branch = GITOPS_REPOSITORY_BRANCH else: - self.__branch = gitops_template_branch + self._branch = gitops_template_branch # TODO: DEBUG, remove - self.__token = token + self._token = token @trace() def check_repository_existence(self): @@ -37,13 +37,13 @@ def check_repository_existence(self): Check if the repository exists :return: True or False """ - repo = GHRepo.parse(self.__url) + repo = GHRepo.parse(self._url) headers = {} - if self.__token is not None: - headers['Authorization'] = f'token {self.__token}' + if self._token is not None: + headers['Authorization'] = f'token {self._token}' try: - response = requests.get(f'{repo.api_url}/branches/{self.__branch}', + response = requests.get(f'{repo.api_url}/branches/{self._branch}', headers=headers) if response.status_code == requests.codes["not_found"]: return False @@ -66,7 +66,7 @@ def clone(self): os.makedirs(temp_folder) try: - repo = Repo.clone_from(self.__url, temp_folder, progress=ProgressPrinter(), branch=self.__branch) + repo = Repo.clone_from(self._url, temp_folder, progress=ProgressPrinter(), branch=self._branch) except GitError as e: raise e diff --git a/tools/cli/services/wl_gitops_template_manager.py b/tools/cli/services/wl_gitops_template_manager.py new file mode 100644 index 00000000..c7620953 --- /dev/null +++ b/tools/cli/services/wl_gitops_template_manager.py @@ -0,0 +1,98 @@ +import os +import shutil + +from ghrepo import GHRepo +from git import Repo, GitError, Actor + +from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER +from common.const.const import WL_GITOPS_REPOSITORY_URL, WL_GITOPS_REPOSITORY_BRANCH +from common.tracing_decorator import trace + + +class WorkloadGitOpsTemplateManager: + """CG DevX Workload GitOps templates manager.""" + + def __init__(self, org_name: str, repo_name: str, key_path: str, template_url: str = None, + template_branch: str = None): + if template_url is None: + self._url = WL_GITOPS_REPOSITORY_URL + else: + self._url = template_url + + if template_branch is None: + self._branch = WL_GITOPS_REPOSITORY_BRANCH + else: + self._branch = template_branch + + self._git_org_name = org_name + self._repo_name = repo_name + self._key_path = key_path + self._wl_template_repo = None + self._wl_repo = None + + self._wl_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / self._repo_name + self._wl_template_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / GHRepo.parse(self._url).name + + @trace() + def clone_template(self): + if os.path.exists(self._wl_template_repo_folder): + shutil.rmtree(self._wl_template_repo_folder) + + os.makedirs(self._wl_template_repo_folder) + + try: + self._wl_template_repo = Repo.clone_from(self._url, + self._wl_template_repo_folder, + branch=self._branch, + env={ + "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) + return self._wl_template_repo_folder + except GitError as e: + return None + + @trace() + def clone_wl(self): + if os.path.exists(self._wl_repo_folder): + shutil.rmtree(self._wl_repo_folder) + os.makedirs(self._wl_repo_folder) + + try: + self._wl_repo = Repo.clone_from(GHRepo(self._git_org_name, self._repo_name).ssh_url, + self._wl_repo_folder, + env={ + "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) + return self._wl_repo_folder + except GitError as e: + return None + + @trace() + def bootstrap(self): + if os.path.exists(self._wl_template_repo_folder / ".git"): + shutil.rmtree(self._wl_template_repo_folder / ".git") + shutil.copytree(self._wl_template_repo_folder, self._wl_repo_folder, dirs_exist_ok=True) + + @trace() + def parametrise(self, params: dict = {}): + for root, dirs, files in os.walk(self._wl_repo_folder): + for name in files: + if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): + file_path = os.path.join(root, name) + with open(file_path, "r") as file: + data = file.read() + for k, v in params.items(): + data = data.replace(k, v) + with open(file_path, "w") as file: + file.write(data) + + @trace() + def upload(self, name: str = None, email: str = None): + self._wl_repo.git.add(all=True) + author = Actor(name=name, email=email) + self._wl_repo.index.commit("initial", author=author, committer=author) + + self._wl_repo.remotes.origin.push(self._wl_repo.active_branch.name) + + @trace() + def cleanup(self): + if os.path.exists(self._wl_template_repo_folder): + shutil.rmtree(self._wl_template_repo_folder) diff --git a/tools/cli/services/wl_template_manager.py b/tools/cli/services/wl_template_manager.py new file mode 100644 index 00000000..169f7c34 --- /dev/null +++ b/tools/cli/services/wl_template_manager.py @@ -0,0 +1,102 @@ +import os +import shutil + +from ghrepo import GHRepo +from git import Repo, GitError, Actor + +from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER +from common.const.const import WL_REPOSITORY_URL, WL_REPOSITORY_BRANCH +from common.tracing_decorator import trace + + +class WorkloadTemplateManager: + """CG DevX Workload templates manager.""" + + def __init__(self, org_name: str, repo_name: str, key_path: str, template_url: str = None, + template_branch: str = None): + if template_url is None: + self._url = WL_REPOSITORY_URL + else: + self._url = template_url + + if template_branch is None: + self._branch = WL_REPOSITORY_BRANCH + else: + self._branch = template_branch + + self._git_org_name = org_name + self._repo_name = repo_name + self._key_path = key_path + self._wl_template_repo = None + self._wl_repo = None + + self._wl_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / self._repo_name + self._wl_template_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / GHRepo.parse(self._url).name + + @trace() + def clone_template(self): + if os.path.exists(self._wl_template_repo_folder): + shutil.rmtree(self._wl_template_repo_folder) + + os.makedirs(self._wl_template_repo_folder) + + try: + self._wl_template_repo = Repo.clone_from(self._url, + self._wl_template_repo_folder, + branch=self._branch, + env={ + "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) + return self._wl_template_repo_folder + except GitError as e: + return None + + @trace() + def clone_wl(self): + if os.path.exists(self._wl_repo_folder): + shutil.rmtree(self._wl_repo_folder) + os.makedirs(self._wl_repo_folder) + + try: + self._wl_repo = Repo.clone_from(GHRepo(self._git_org_name, self._repo_name).ssh_url, + self._wl_repo_folder, + env={ + "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) + return self._wl_repo_folder + except GitError as e: + return None + + @trace() + def bootstrap(self, services: [str]): + if os.path.exists(self._wl_template_repo_folder / ".git"): + shutil.rmtree(self._wl_template_repo_folder / ".git") + shutil.copytree(self._wl_template_repo_folder, self._wl_repo_folder, dirs_exist_ok=True) + for wl_svc_name in services: + shutil.copytree(self._wl_repo_folder / "wl-service-name", self._wl_repo_folder / wl_svc_name, + dirs_exist_ok=True) + shutil.rmtree(self._wl_repo_folder / "wl-service-name") + + @trace() + def parametrise(self, params: dict = {}): + for root, dirs, files in os.walk(self._wl_repo_folder): + for name in files: + if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): + file_path = os.path.join(root, name) + with open(file_path, "r") as file: + data = file.read() + for k, v in params.items(): + data = data.replace(k, v) + with open(file_path, "w") as file: + file.write(data) + + @trace() + def upload(self, name: str = None, email: str = None): + self._wl_repo.git.add(all=True) + author = Actor(name=name, email=email) + self._wl_repo.index.commit("initial", author=author, committer=author) + + self._wl_repo.remotes.origin.push(self._wl_repo.active_branch.name) + + @trace() + def cleanup(self): + if os.path.exists(self._wl_template_repo_folder): + shutil.rmtree(self._wl_template_repo_folder) From ea9c7ca52cc3c6024928f107d0f02031596b8eaf Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Tue, 19 Dec 2023 13:12:01 +0100 Subject: [PATCH 15/49] feat: support non-free plans in vcs terraform (#33) --- platform/terraform/modules/vcs_github/main.tf | 2 +- .../modules/vcs_github/repository/main.tf | 54 +++++++++++++++++++ .../modules/vcs_github/repository/variable.tf | 7 ++- .../modules/vcs_github/runner_group.tf | 13 +++++ .../terraform/modules/vcs_github/variable.tf | 12 +++++ .../modules/vcs_github/workload_repos.tf | 1 + platform/terraform/vcs/main.tf | 8 ++- platform/terraform/vcs/variable.tf | 1 + 8 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 platform/terraform/modules/vcs_github/runner_group.tf diff --git a/platform/terraform/modules/vcs_github/main.tf b/platform/terraform/modules/vcs_github/main.tf index cc4fc276..94a04ca3 100644 --- a/platform/terraform/modules/vcs_github/main.tf +++ b/platform/terraform/modules/vcs_github/main.tf @@ -4,7 +4,7 @@ terraform { github = { # https://registry.terraform.io/providers/integrations/github/latest/docs source = "integrations/github" - version = "~> 5.33.0" + version = "~> 5.42.0" } } diff --git a/platform/terraform/modules/vcs_github/repository/main.tf b/platform/terraform/modules/vcs_github/repository/main.tf index c4d3a43a..e7f30b7f 100644 --- a/platform/terraform/modules/vcs_github/repository/main.tf +++ b/platform/terraform/modules/vcs_github/repository/main.tf @@ -19,6 +19,56 @@ resource "github_repository" "repo" { } +# Protect the main branch of the repository. Additionally, require +# only allow the engineers team merge to the branch. + +resource "github_branch_protection" "this" { + count = var.branch_protection ? 1 : 0 + repository_id = var.repo_name + + pattern = "main" + enforce_admins = true + allows_deletions = false + allows_force_pushes = false + required_linear_history = true + require_conversation_resolution = true + + # required_status_checks { + # strict = false + # } + + # required_pull_request_reviews { + # dismiss_stale_reviews = true + # restrict_dismissals = true + # dismissal_restrictions = [ + # data.github_user.example.node_id, + # github_team.example.node_id, + # "/exampleuser", + # "exampleorganization/exampleteam", + # ] + # } + + # push_restrictions = [ + # data.github_user.example.node_id, + # "/exampleuser", + # "exampleorganization/exampleteam", + # # you can have more than one type of restriction (teams + users). If you use + # # more than one type, you must use node_ids of each user and each team. + # # github_team.example.node_id + # # github_user.example-2.node_id + # ] + + # force_push_bypassers = [ + # data.github_user.example.node_id, + # "/exampleuser", + # "exampleorganization/exampleteam", + # # you can have more than one type of restriction (teams + users) + # # github_team.example.node_id + # # github_team.example-2.node_id + # ] +} + + output "repo_name" { value = github_repository.repo.name } @@ -34,3 +84,7 @@ output "repo_git_html_url" { output "repo_git_ssh_clone_url" { value = github_repository.repo.ssh_clone_url } + +output "repo_id" { + value = github_repository.repo.repo_id +} diff --git a/platform/terraform/modules/vcs_github/repository/variable.tf b/platform/terraform/modules/vcs_github/repository/variable.tf index 7c489265..ceec64ba 100644 --- a/platform/terraform/modules/vcs_github/repository/variable.tf +++ b/platform/terraform/modules/vcs_github/repository/variable.tf @@ -37,6 +37,11 @@ variable "delete_branch_on_merge" { default = true } +variable "branch_protection" { + type = bool + default = true +} + variable "template" { type = map(string) description = "Template Repository object for Repository creation" @@ -58,5 +63,3 @@ variable "atlantis_repo_webhook_secret" { default = "" sensitive = true } - - diff --git a/platform/terraform/modules/vcs_github/runner_group.tf b/platform/terraform/modules/vcs_github/runner_group.tf new file mode 100644 index 00000000..3b271d4b --- /dev/null +++ b/platform/terraform/modules/vcs_github/runner_group.tf @@ -0,0 +1,13 @@ +locals { + self_hosted_runners_repos = concat( + [for workload in module.workload_repos : workload.repo_id], + [module.gitops-repo.repo_id] + ) +} + +resource "github_actions_runner_group" "this" { + count = var.vcs_subscription_plan ? 1 : 0 + name = var.vcs_owner + visibility = "selected" + selected_repository_ids = local.self_hosted_runners_repos +} diff --git a/platform/terraform/modules/vcs_github/variable.tf b/platform/terraform/modules/vcs_github/variable.tf index af7018bd..12f96bc1 100644 --- a/platform/terraform/modules/vcs_github/variable.tf +++ b/platform/terraform/modules/vcs_github/variable.tf @@ -19,6 +19,17 @@ variable "vcs_bot_ssh_public_key" { default = "" } +variable "vcs_subscription_plan" { + description = "True for advanced github/gitlab plan. False for free tier" + type = bool + default = false +} + +variable "vcs_owner" { + type = string + default = "" +} + variable "workloads" { description = "workloads configuration" type = map(object({ @@ -31,6 +42,7 @@ variable "workloads" { has_issues = optional(bool, false) default_branch_name = optional(string, "main") delete_branch_on_merge = optional(bool, true) + branch_protection = optional(bool, true) atlantis_enabled = optional(bool, false) })) })) diff --git a/platform/terraform/modules/vcs_github/workload_repos.tf b/platform/terraform/modules/vcs_github/workload_repos.tf index 987bd1ca..19b01681 100644 --- a/platform/terraform/modules/vcs_github/workload_repos.tf +++ b/platform/terraform/modules/vcs_github/workload_repos.tf @@ -15,6 +15,7 @@ module "workload_repos" { has_issues = each.value.has_issues default_branch_name = each.value.default_branch_name delete_branch_on_merge = each.value.delete_branch_on_merge + branch_protection = each.value.branch_protection atlantis_enabled = each.value.atlantis_enabled atlantis_url = var.atlantis_url atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret diff --git a/platform/terraform/vcs/main.tf b/platform/terraform/vcs/main.tf index 2e82ff2a..1e894a04 100644 --- a/platform/terraform/vcs/main.tf +++ b/platform/terraform/vcs/main.tf @@ -8,8 +8,10 @@ terraform { locals { - gitops_repo_name = "" - atlantis_url = "https:///events" + gitops_repo_name = "" + atlantis_url = "https:///events" + vcs_owner = "" + vcs_subscription_plan = # bool true(paid plans) / false (free tier) } @@ -21,4 +23,6 @@ module "vcs" { atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key workloads = var.workloads + vcs_owner = local.vcs_owner + vcs_subscription_plan = local.vcs_subscription_plan } diff --git a/platform/terraform/vcs/variable.tf b/platform/terraform/vcs/variable.tf index c097238b..284bab81 100644 --- a/platform/terraform/vcs/variable.tf +++ b/platform/terraform/vcs/variable.tf @@ -20,6 +20,7 @@ variable "workloads" { has_issues = optional(bool, false) default_branch_name = optional(string, "main") delete_branch_on_merge = optional(bool, true) + branch_protection = optional(bool, true) atlantis_enabled = optional(bool, false) })) })) From b4a9d638f59267502e882bced61b1c469fcd0ea8 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Tue, 19 Dec 2023 14:59:19 +0100 Subject: [PATCH 16/49] feat: add harbor proxy project to parameters set --- platform/terraform/core_services/main.tf | 2 +- platform/terraform/core_services/output.tf | 4 ++++ .../terraform/modules/registry_harbor/output.tf | 4 ++++ tools/cli/commands/setup.py | 13 +++++++------ 4 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 platform/terraform/core_services/output.tf diff --git a/platform/terraform/core_services/main.tf b/platform/terraform/core_services/main.tf index 94c6a2bb..72e566f2 100644 --- a/platform/terraform/core_services/main.tf +++ b/platform/terraform/core_services/main.tf @@ -1,6 +1,6 @@ terraform { # Remote backend configuration - # + # required_providers { harbor = { diff --git a/platform/terraform/core_services/output.tf b/platform/terraform/core_services/output.tf new file mode 100644 index 00000000..a4214215 --- /dev/null +++ b/platform/terraform/core_services/output.tf @@ -0,0 +1,4 @@ +# harbor output +output "dockerhub_proxy_name" { + value = module.registry.dockerhub_proxy_name +} diff --git a/platform/terraform/modules/registry_harbor/output.tf b/platform/terraform/modules/registry_harbor/output.tf index 32ef0748..b8e5b6ba 100644 --- a/platform/terraform/modules/registry_harbor/output.tf +++ b/platform/terraform/modules/registry_harbor/output.tf @@ -2,3 +2,7 @@ output "main_robot_full_name" { value = resource.harbor_robot_account.main.full_name } + +output "dockerhub_proxy_name" { + value = harbor_project.proxy.name +} diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 41e2c0fd..759b0dcc 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -6,8 +6,6 @@ import portforward import yaml -from common.utils.command_utils import init_cloud_provider, init_git_provider, prepare_cloud_provider_auth_env_vars, \ - set_envs, unset_envs, wait, wait_http_endpoint_readiness from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_HOSTING_PROVIDER, \ LOCAL_TF_FOLDER_SECRETS_MANAGER, LOCAL_TF_FOLDER_USERS, LOCAL_TF_FOLDER_CORE_SERVICES from common.const.const import GITOPS_REPOSITORY_URL, GITOPS_REPOSITORY_BRANCH, KUBECTL_VERSION, PLATFORM_USER_NAME, \ @@ -25,6 +23,8 @@ from common.logging_config import configure_logging from common.state_store import StateStore from common.tracing_decorator import trace +from common.utils.command_utils import init_cloud_provider, init_git_provider, prepare_cloud_provider_auth_env_vars, \ + set_envs, unset_envs, wait, wait_http_endpoint_readiness from common.utils.generators import random_string_generator from services.cloud.cloud_provider_manager import CloudProviderManager from services.dependency_manager import DependencyManager @@ -121,7 +121,7 @@ def setup( # validate parameters if not p.validate_input_params(validator=setup_param_validator): - return + raise click.ClickException("Input parameters are incorrect") # save checkpoint p.save_checkpoint() @@ -150,7 +150,7 @@ def setup( p.fragments["# "] = git_man.create_tf_module_snippet() git_subscription_plan = git_man.get_organization_plan() - p.internals["GIT_SUBSCRIPTION_PLAN"] = bool(git_subscription_plan) + p.parameters[""] = bool(git_subscription_plan) if git_subscription_plan > 0: p.fragments["# "] = git_man.create_runner_group_snippet() p.parameters[""] = p.get_input_param(PRIMARY_CLUSTER_NAME) @@ -236,8 +236,8 @@ def setup( "secrets") p.fragments["# "] = cloud_man.create_iac_backend_snippet(tf_backend_storage, "users") - p.fragments["# "] = cloud_man.create_iac_backend_snippet(tf_backend_storage, - "registry") + p.fragments["# "] = cloud_man.create_iac_backend_snippet(tf_backend_storage, + "core_services") p.fragments["# "] = cloud_man.create_hosting_provider_snippet() @@ -770,6 +770,7 @@ def setup( "code_quality_admin_password": p.internals["CODE_QUALITY_PASSWORD"] }) core_services_out = tf_wrapper.output() + p.parameters[""] = core_services_out["dockerhub_proxy_name"] # unset envs as no longer needed unset_envs(core_services_tf_env_vars) From d85cfa599c507e428e14cc7355174fc9d18bf7d6 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Tue, 19 Dec 2023 16:43:46 +0100 Subject: [PATCH 17/49] - workload bootstrap fixes - git plan tf param type change --- tools/cli/commands/setup.py | 5 +-- tools/cli/commands/workload/bootstrap.py | 31 +++++++------------ tools/cli/services/platform_gitops.py | 8 +++-- .../services/wl_gitops_template_manager.py | 2 ++ tools/cli/services/wl_template_manager.py | 2 ++ 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 759b0dcc..e45d1665 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -123,9 +123,6 @@ def setup( if not p.validate_input_params(validator=setup_param_validator): raise click.ClickException("Input parameters are incorrect") - # save checkpoint - p.save_checkpoint() - cloud_man, dns_man = init_cloud_provider(p) p.parameters[""] = cloud_man.region @@ -150,7 +147,7 @@ def setup( p.fragments["# "] = git_man.create_tf_module_snippet() git_subscription_plan = git_man.get_organization_plan() - p.parameters[""] = bool(git_subscription_plan) + p.parameters[""] = str(bool(git_subscription_plan)).lower() if git_subscription_plan > 0: p.fragments["# "] = git_man.create_runner_group_snippet() p.parameters[""] = p.get_input_param(PRIMARY_CLUSTER_NAME) diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index c8861f18..5f4a77a3 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -9,7 +9,7 @@ from common.const.const import WL_REPOSITORY_URL, WL_GITOPS_REPOSITORY_URL from common.logging_config import configure_logging from common.state_store import StateStore -from common.utils.command_utils import str_to_kebab, init_cloud_provider +from common.utils.command_utils import str_to_kebab, init_cloud_provider, check_installation_presence from services.wl_gitops_template_manager import WorkloadGitOpsTemplateManager from services.wl_template_manager import WorkloadTemplateManager @@ -42,8 +42,7 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp # Set up global logger configure_logging(verbosity) - if not os.path.exists(LOCAL_FOLDER): - raise click.ClickException("CG DevX metadata does not exist on this machine") + check_installation_presence() p: StateStore = StateStore() @@ -68,19 +67,6 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp wl_repo_name = str_to_kebab(wl_repo_name) wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) - if not wl_template_branch: - wl_template_branch = "main" - if not wl_template_url: - wl_template_url = WL_REPOSITORY_URL - - if not wl_gitops_template_branch: - wl_gitops_template_branch = "main" - if not wl_gitops_template_url: - wl_gitops_template_url = WL_GITOPS_REPOSITORY_URL - - if not os.path.exists(LOCAL_GITOPS_FOLDER): - raise click.ClickException("GitOps repo does not exist") - if os.path.exists(LOCAL_WORKLOAD_TEMP_FOLDER): shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) @@ -89,7 +75,10 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp wl_man = WorkloadTemplateManager( org_name=p.parameters[""], repo_name=wl_repo_name, - key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"], + template_url=wl_template_url, + template_branch=wl_template_branch + ) # workload repo if not (wl_man.clone_wl()): @@ -98,6 +87,7 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp if not (wl_man.clone_template()): raise click.ClickException("Failed cloning template repo") + # make wl_svc_name multiple value param wl_man.bootstrap([wl_svc_name]) wl_repo_params = { @@ -116,7 +106,10 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp wl_ops_man = WorkloadGitOpsTemplateManager( org_name=p.parameters[""], repo_name=wl_gitops_repo_name, - key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"], + template_url=wl_gitops_template_url, + template_branch=wl_gitops_template_branch + ) # workload repo if not (wl_ops_man.clone_wl()): @@ -145,7 +138,7 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp wl_ops_man.parametrise(wl_gitops_repo_params) - wl_ops_man.upload() + wl_ops_man.upload(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) wl_ops_man.cleanup() diff --git a/tools/cli/services/platform_gitops.py b/tools/cli/services/platform_gitops.py index 80fe85fa..21d0016f 100644 --- a/tools/cli/services/platform_gitops.py +++ b/tools/cli/services/platform_gitops.py @@ -67,9 +67,13 @@ def add_workload(self, wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str } }) # secrets - self._add_wl_vars(LOCAL_TF_FOLDER_SECRETS_MANAGER, wl_name, {}) + self._add_wl_vars(LOCAL_TF_FOLDER_SECRETS_MANAGER, wl_name, { + "description": f"CG DevX {wl_name} workload definition" + }) # core services - self._add_wl_vars(LOCAL_TF_FOLDER_CORE_SERVICES, wl_name, {}) + self._add_wl_vars(LOCAL_TF_FOLDER_CORE_SERVICES, wl_name, { + "description": f"CG DevX {wl_name} workload definition" + }) # prepare ArgoCD manifest wl_gitops_repo = GHRepo(self._git_man.organization, wl_gitops_repo_name) diff --git a/tools/cli/services/wl_gitops_template_manager.py b/tools/cli/services/wl_gitops_template_manager.py index c7620953..e244aadf 100644 --- a/tools/cli/services/wl_gitops_template_manager.py +++ b/tools/cli/services/wl_gitops_template_manager.py @@ -96,3 +96,5 @@ def upload(self, name: str = None, email: str = None): def cleanup(self): if os.path.exists(self._wl_template_repo_folder): shutil.rmtree(self._wl_template_repo_folder) + if os.path.exists(self._wl_repo_folder): + shutil.rmtree(self._wl_repo_folder) diff --git a/tools/cli/services/wl_template_manager.py b/tools/cli/services/wl_template_manager.py index 169f7c34..aa395618 100644 --- a/tools/cli/services/wl_template_manager.py +++ b/tools/cli/services/wl_template_manager.py @@ -100,3 +100,5 @@ def upload(self, name: str = None, email: str = None): def cleanup(self): if os.path.exists(self._wl_template_repo_folder): shutil.rmtree(self._wl_template_repo_folder) + if os.path.exists(self._wl_repo_folder): + shutil.rmtree(self._wl_repo_folder) From e2367aba62fbce58b14bbf8c1b7846f44c133a60 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Tue, 19 Dec 2023 17:33:36 +0100 Subject: [PATCH 18/49] fix: runner group from proper variable --- platform/terraform/modules/vcs_github/runner_group.tf | 2 +- platform/terraform/modules/vcs_github/variable.tf | 5 +++++ platform/terraform/vcs/main.tf | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/platform/terraform/modules/vcs_github/runner_group.tf b/platform/terraform/modules/vcs_github/runner_group.tf index 3b271d4b..b028cf67 100644 --- a/platform/terraform/modules/vcs_github/runner_group.tf +++ b/platform/terraform/modules/vcs_github/runner_group.tf @@ -7,7 +7,7 @@ locals { resource "github_actions_runner_group" "this" { count = var.vcs_subscription_plan ? 1 : 0 - name = var.vcs_owner + name = var.cluster_name visibility = "selected" selected_repository_ids = local.self_hosted_runners_repos } diff --git a/platform/terraform/modules/vcs_github/variable.tf b/platform/terraform/modules/vcs_github/variable.tf index 12f96bc1..e7264444 100644 --- a/platform/terraform/modules/vcs_github/variable.tf +++ b/platform/terraform/modules/vcs_github/variable.tf @@ -30,6 +30,11 @@ variable "vcs_owner" { default = "" } +variable "cluster_name" { + type = string + default = "" +} + variable "workloads" { description = "workloads configuration" type = map(object({ diff --git a/platform/terraform/vcs/main.tf b/platform/terraform/vcs/main.tf index 1e894a04..49b7e173 100644 --- a/platform/terraform/vcs/main.tf +++ b/platform/terraform/vcs/main.tf @@ -10,6 +10,7 @@ terraform { locals { gitops_repo_name = "" atlantis_url = "https:///events" + cluster_name = "" vcs_owner = "" vcs_subscription_plan = # bool true(paid plans) / false (free tier) } @@ -23,6 +24,7 @@ module "vcs" { atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key workloads = var.workloads + cluster_name - local.cluster_name vcs_owner = local.vcs_owner vcs_subscription_plan = local.vcs_subscription_plan } From 20c2e7eeac39336090fb0822fd23e75be522d690 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Tue, 19 Dec 2023 17:33:36 +0100 Subject: [PATCH 19/49] fix: runner group from proper variable --- platform/terraform/vcs/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/terraform/vcs/main.tf b/platform/terraform/vcs/main.tf index 49b7e173..860e6cfe 100644 --- a/platform/terraform/vcs/main.tf +++ b/platform/terraform/vcs/main.tf @@ -24,7 +24,7 @@ module "vcs" { atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key workloads = var.workloads - cluster_name - local.cluster_name + cluster_name = local.cluster_name vcs_owner = local.vcs_owner vcs_subscription_plan = local.vcs_subscription_plan } From 32bb771690ddb24ca8de595341b31a6e5930c4f1 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Tue, 19 Dec 2023 19:11:01 +0100 Subject: [PATCH 20/49] feat: add more proxy projects --- platform/terraform/core_services/output.tf | 9 ++++ .../modules/registry_harbor/output.tf | 14 +++++- .../modules/registry_harbor/proxy.tf | 49 +++++++++++++++++-- tools/cli/commands/setup.py | 3 ++ 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/platform/terraform/core_services/output.tf b/platform/terraform/core_services/output.tf index a4214215..02ec6f2a 100644 --- a/platform/terraform/core_services/output.tf +++ b/platform/terraform/core_services/output.tf @@ -2,3 +2,12 @@ output "dockerhub_proxy_name" { value = module.registry.dockerhub_proxy_name } +output "gcr_proxy_name" { + value = module.registry.gcr_proxy_name +} +output "k8s_gcr_proxy_name" { + value = module.registry.k8s_gcr_proxy_name +} +output "quay_proxy_name" { + value = module.registry.quay_proxy_name +} diff --git a/platform/terraform/modules/registry_harbor/output.tf b/platform/terraform/modules/registry_harbor/output.tf index b8e5b6ba..2ec23df0 100644 --- a/platform/terraform/modules/registry_harbor/output.tf +++ b/platform/terraform/modules/registry_harbor/output.tf @@ -4,5 +4,17 @@ output "main_robot_full_name" { } output "dockerhub_proxy_name" { - value = harbor_project.proxy.name + value = harbor_project.docker-hub-proxy.name +} + +output "gcr_proxy_name" { + value = harbor_project.gcr-proxy.name +} + +output "k8s_gcr_proxy_name" { + value = harbor_project.k8s-gcr-proxy.name +} + +output "quay_proxy_name" { + value = harbor_project.quay-proxy.name } diff --git a/platform/terraform/modules/registry_harbor/proxy.tf b/platform/terraform/modules/registry_harbor/proxy.tf index f404f993..5c6c0e35 100644 --- a/platform/terraform/modules/registry_harbor/proxy.tf +++ b/platform/terraform/modules/registry_harbor/proxy.tf @@ -1,10 +1,51 @@ -resource "harbor_project" "proxy" { - name = "dockerhub-proxy" - registry_id = harbor_registry.docker.registry_id +# docker-hub proxy +resource "harbor_project" "docker-hub-proxy" { + name = "dockerhub-proxy" + registry_id = harbor_registry.docker-hub.registry_id + force_destroy = true } -resource "harbor_registry" "docker" { +resource "harbor_registry" "docker-hub" { provider_name = "docker-hub" name = "dockerhub-proxy" endpoint_url = "https://hub.docker.com" +} + +# gcr.io proxy +resource "harbor_project" "gcr-proxy" { + name = "gcr-proxy" + registry_id = harbor_registry.gcr.registry_id + force_destroy = true +} + +resource "harbor_registry" "gcr" { + provider_name = "docker-registry" + name = "gcr-proxy" + endpoint_url = "https://gcr.io" +} + +# k8s.gcr.io proxy +resource "harbor_project" "k8s-gcr-proxy" { + name = "k8s-gcr-proxy" + registry_id = harbor_registry.k8s-gcr.registry_id + force_destroy = true +} + +resource "harbor_registry" "k8s-gcr" { + provider_name = "docker-registry" + name = "k8s-gcr-proxy" + endpoint_url = "https://k8s.gcr.io" +} + +# quay.io proxy +resource "harbor_project" "quay-proxy" { + name = "quay-proxy" + registry_id = harbor_registry.quay.registry_id + force_destroy = true +} + +resource "harbor_registry" "quay" { + provider_name = "quay" + name = "quay-proxy" + endpoint_url = "https://quay.io" } \ No newline at end of file diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index e45d1665..a972d1f2 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -768,6 +768,9 @@ def setup( }) core_services_out = tf_wrapper.output() p.parameters[""] = core_services_out["dockerhub_proxy_name"] + p.parameters[""] = core_services_out["gcr_proxy_name"] + p.parameters[""] = core_services_out["k8s_gcr_proxy_name"] + p.parameters[""] = core_services_out["quay_proxy_name"] # unset envs as no longer needed unset_envs(core_services_tf_env_vars) From 04ec69c52b1a34b78eff88a620240016325a931c Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Wed, 20 Dec 2023 15:01:24 +0100 Subject: [PATCH 21/49] cgange user to groups in pull request description --- tools/cli/commands/workload/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 5c82a8ad..4e326539 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -68,7 +68,7 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: branch_name, main_branch, f"introduce {wl_name}", - f"Add default secrets, user and default repository structure.") + f"Add default secrets, groups and default repository structure.") webbrowser.open(pr_url, autoraise=False) except Exception as e: raise click.ClickException("Could not create PR") From 4d4d21cecb0ec02624055266018c79c145284f97 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Wed, 20 Dec 2023 15:05:34 +0100 Subject: [PATCH 22/49] docs: Update workload management README.md --- tools/cli/commands/workload/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cli/commands/workload/README.md b/tools/cli/commands/workload/README.md index fe6b5601..e0b704e9 100644 --- a/tools/cli/commands/workload/README.md +++ b/tools/cli/commands/workload/README.md @@ -95,7 +95,7 @@ and branches. Using command arguments ```bash -cgdevxcli workload create --workload-name your-workload-name \ +cgdevxcli workload bootstrap --workload-name your-workload-name \ --workload-repository-name your-workload-repository-name --workload-gitops-repository-name your-workload-gitops-repository-name --workload-service-name your-first-service-name From eee3d1711f821eabccac95d39b421c4029cc089b Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Wed, 20 Dec 2023 15:22:43 +0100 Subject: [PATCH 23/49] fix: vcs non-free plan depends_on for runner group --- platform/terraform/modules/vcs_github/repository/main.tf | 2 +- platform/terraform/modules/vcs_github/runner_group.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/platform/terraform/modules/vcs_github/repository/main.tf b/platform/terraform/modules/vcs_github/repository/main.tf index e7f30b7f..a7e8a118 100644 --- a/platform/terraform/modules/vcs_github/repository/main.tf +++ b/platform/terraform/modules/vcs_github/repository/main.tf @@ -24,7 +24,7 @@ resource "github_repository" "repo" { resource "github_branch_protection" "this" { count = var.branch_protection ? 1 : 0 - repository_id = var.repo_name + repository_id = github_repository.repo.node_id pattern = "main" enforce_admins = true diff --git a/platform/terraform/modules/vcs_github/runner_group.tf b/platform/terraform/modules/vcs_github/runner_group.tf index b028cf67..c8661a71 100644 --- a/platform/terraform/modules/vcs_github/runner_group.tf +++ b/platform/terraform/modules/vcs_github/runner_group.tf @@ -6,6 +6,7 @@ locals { } resource "github_actions_runner_group" "this" { + depends_on = [module.workload_repos] count = var.vcs_subscription_plan ? 1 : 0 name = var.cluster_name visibility = "selected" From f40ce47b23047f02f6eafe5fa035a9e89ddf1531 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Wed, 20 Dec 2023 16:40:46 +0100 Subject: [PATCH 24/49] fix: atlantis unable automerge for protected branches --- platform/terraform/modules/vcs_github/repository/main.tf | 8 ++++++++ .../terraform/modules/vcs_github/repository/variable.tf | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/platform/terraform/modules/vcs_github/repository/main.tf b/platform/terraform/modules/vcs_github/repository/main.tf index a7e8a118..20702e2a 100644 --- a/platform/terraform/modules/vcs_github/repository/main.tf +++ b/platform/terraform/modules/vcs_github/repository/main.tf @@ -7,6 +7,14 @@ resource "github_repository" "repo" { archive_on_destroy = var.archive_on_destroy has_issues = var.has_issues delete_branch_on_merge = var.delete_branch_on_merge + allow_merge_commit = var.allow_merge_commit + # atlantis currently doesn't support branch protection with required linear historyw when repo settings allowed merge commits + # need to monitor these issues + # https://github.com/runatlantis/atlantis/issues/1176 + # https://github.com/runatlantis/atlantis/pull/3211 + # https://github.com/runatlantis/atlantis/pull/3276 + # https://github.com/runatlantis/atlantis/pull/3321 + dynamic "template" { for_each = length(var.template) != 0 ? [var.template] : [] diff --git a/platform/terraform/modules/vcs_github/repository/variable.tf b/platform/terraform/modules/vcs_github/repository/variable.tf index ceec64ba..cc4260c7 100644 --- a/platform/terraform/modules/vcs_github/repository/variable.tf +++ b/platform/terraform/modules/vcs_github/repository/variable.tf @@ -42,6 +42,11 @@ variable "branch_protection" { default = true } +variable "allow_merge_commit" { + type = bool + default = false +} + variable "template" { type = map(string) description = "Template Repository object for Repository creation" From b63c2636a78442e85fc43f17533e6510a5d6cc39 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Wed, 20 Dec 2023 16:50:20 +0100 Subject: [PATCH 25/49] fix: change user to groups in workload pull request desctiption --- tools/cli/commands/workload/delete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index 21f3509c..07d50942 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -120,7 +120,7 @@ def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verb pr_url = gor.create_pr(p.parameters[""], branch_name, main_branch, f"remove {wl_name}", - f"Remove default secrets, user and repository structure.") + f"Remove default secrets, groups and repository structure.") webbrowser.open(pr_url, autoraise=False) except Exception as e: raise click.ClickException("Could not create PR") From 7bfcf1b4a8d0935a255a4e84a869cd4682ed8293 Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Fri, 29 Dec 2023 15:53:07 +0200 Subject: [PATCH 26/49] update: replaced argo-workflows with the actual one. --- .../argo-workflows/application.yaml | 16 ++-- ...l => argo-cluster-workflow-templates.yaml} | 4 +- ...rgo-workflow-controller-secret-reader.yaml | 27 +++++++ .../argo-workflow-templates.yaml | 22 ++++++ .../workflowtemplates/black-wft.yaml | 15 ---- .../workflowtemplates/crane-wft.yaml | 17 ----- .../workflowtemplates/git-clone-wft.yaml | 38 ---------- .../workflowtemplates/kaniko-wft.yaml | 17 ----- .../minimalistic-app-dag-wft.yaml | 74 ------------------- .../workflowtemplates/sillypy-dag-wft.yaml | 72 ------------------ .../workflowtemplates/tslint-wft.yaml | 15 ---- .../argo-workflows/externalsecret.yaml | 25 +++++++ .../components/argo-workflows/wait.yaml | 4 +- .../workflow-templates/black-wft.yaml | 15 ++++ .../workflow-templates/crane-wft.yaml | 18 +++++ .../workflow-templates/eslint-wft.yaml | 15 ++++ .../workflow-templates/git-clone-wft.yaml | 41 ++++++++++ .../workflow-templates/kaniko-wft.yaml | 25 +++++++ .../trivy-fs-scan-wft.yaml | 8 +- .../workflow-templates/tslint-wft.yaml | 15 ++++ 20 files changed, 220 insertions(+), 263 deletions(-) rename platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/{argo-workflows-templates.yaml => argo-cluster-workflow-templates.yaml} (83%) create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-controller-secret-reader.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml delete mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml rename platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/{cluster-workflow-templates/workflowtemplates => workflow-templates}/trivy-fs-scan-wft.yaml (55%) create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml index ef097559..5776fff1 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml @@ -13,6 +13,8 @@ spec: helm: values: |- nameOverride: argo + images: + pullPolicy: IfNotPresent executor: resources: requests: @@ -27,7 +29,7 @@ spec: create: true name: argo-server annotations: - : "" + eks.amazonaws.com/role-arn: "arn:aws:iam::728538925474:role/cg-devx-delta-ci-role" extraArgs: - --secure - --auth-mode=sso @@ -37,24 +39,24 @@ spec: annotations: cert-manager.io/cluster-issuer: "letsencrypt-prod" hosts: - - + - argo.delta.cgdevx.io paths: - / pathType: Prefix tls: - secretName: argo-tls hosts: - - + - argo.delta.cgdevx.io sso: enabled: true - issuer: https:// + issuer: https://vault.delta.cgdevx.io/v1/identity/oidc/provider/cgdevx clientId: name: argo-secrets key: client-id clientSecret: name: argo-secrets key: client-secret - redirectUrl: https:// + redirectUrl: https://argo.delta.cgdevx.io/oauth2/callback scopes: - email - openid @@ -70,7 +72,7 @@ spec: archiveLogs: false s3: insecure: false - bucket: + bucket: cg-devx-delta-artifacts-repository-cjm7m0hm keyFormat: "argo-workflows/artifacts\ /{{workflow.creationTimestamp.Y}}\ /{{workflow.creationTimestamp.m}}\ @@ -79,7 +81,7 @@ spec: /{{workflow.name}}\ /{{pod.name}}" endpoint: s3.amazonaws.com - region: + region: eu-west-2 useSDKCreds: true encryptionOptions: enableEncryption: false diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflows-templates.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-cluster-workflow-templates.yaml similarity index 83% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflows-templates.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-cluster-workflow-templates.yaml index b2c5bf18..59858d48 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflows-templates.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-cluster-workflow-templates.yaml @@ -1,14 +1,14 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: argo-workflow-templates + name: argo-cluster-workflow-templates namespace: argocd annotations: argocd.argoproj.io/sync-wave: '30' spec: project: core source: - repoURL: + repoURL: git@github.com:CGDevX-Demo/cg-devx-delta.git path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates targetRevision: HEAD destination: diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-controller-secret-reader.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-controller-secret-reader.yaml new file mode 100644 index 00000000..25b95056 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-controller-secret-reader.yaml @@ -0,0 +1,27 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: argo-secret-reader + namespace: argo + annotations: + argocd.argoproj.io/sync-wave: '0' +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: argo-secret-reader + namespace: argo + annotations: + argocd.argoproj.io/sync-wave: '0' +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: argo-secret-reader +subjects: +- kind: ServiceAccount + name: argo-workflow-controller + namespace: argo diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml new file mode 100644 index 00000000..518373a3 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: argo-workflow-templates + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: '30' +spec: + project: core + source: + repoURL: git@github.com:CGDevX-Demo/cg-devx-delta.git + path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates + targetRevision: HEAD + destination: + server: https://kubernetes.default.svc + namespace: argo + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml deleted file mode 100644 index a9dd0fa2..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/black-wft.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: black-wft -spec: - templates: - - name: black - container: - image: pyfound/black - imagePullPolicy: IfNotPresent - command: ["black", "--check", "-v", "/{{workflow.parameters.workload}}"] - volumeMounts: - - name: build - mountPath: "{{workflow.parameters.workload}}" - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml deleted file mode 100644 index 9d7da839..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/crane-wft.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: crane-wft -spec: - templates: - - name: crane - container: - image: gcr.io/go-containerregistry/crane - imagePullPolicy: IfNotPresent - args: ["push", "/workspace/{{workflow.parameters.workload}}.tar", "{{workflow.parameters.image}}:{{workflow.parameters.tag}}"] - volumeMounts: - - name: kaniko-secret - mountPath: /home/nonroot/.docker - - name: build - mountPath: /workspace - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml deleted file mode 100644 index f080a248..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/git-clone-wft.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: git-clone-wft -spec: - templates: - - name: git-clone - inputs: - artifacts: - - name: git-src - path: /src - git: - repo: "{{workflow.parameters.repo}}" - # revision: "{{workflow.parameters.revision}}" - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - script: - image: python:latest #subj to change - env: - - name: WORKLOAD - value: "{{workflow.parameters.workload}}" - command: [sh] - source: | - pwd - git status && ls -lra && cat dockerfile - # mkdir /build/${WORKLOAD} - #cp -r * /build/${WORKLOAD} - cp -rv src/$WORKLOAD /build/ - cd /build/ - pwd - ls -l - echo "I've been here" | tee test.txt - volumeMounts: - - mountPath: "/build" - name: build - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml deleted file mode 100644 index 2a39ec54..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/kaniko-wft.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: kaniko-wft -spec: - templates: - - name: kaniko - container: - image: gcr.io/kaniko-project/executor - imagePullPolicy: IfNotPresent - args: ["--dockerfile=/workspace/dockerfile", "--no-push", "--tar-path=/workspace/{{workflow.parameters.workload}}.tar"] - volumeMounts: - - name: kaniko-secret - mountPath: /kaniko/.docker - - name: build - mountPath: /workspace - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml deleted file mode 100644 index a086d716..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/minimalistic-app-dag-wft.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: minimalistic-app-dag-wft -spec: - arguments: - parameters: - - name: image - value: harbor.delta.cgdevx.io/minimalistic-app-test/minimalistic-app - - name: tag - value: "0.3" - - name: workload - value: minimalistic-app-test - - name: wl-service-name - value: minimalistic-app - - name: revision - value: "v1.0" - - name: repo - value: git@github.com:CloudGeometry/cg-devx-minimalistic-demo-app.git - volumeClaimTemplates: - - metadata: - name: build - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 4Gi - entrypoint: ci-sequence - onExit: exit-handler - volumes: - # - name: build - # persistentVolumeClaim: -#claimName: build-volume-claim - - name: kaniko-secret - secret: - secretName: harbcred - items: - - key: .dockerconfigjson - path: config.json - - templates: - - name: ci-sequence - dag: - tasks: - - name: git-clone - templateRef: - name: git-clone-wft - template: git-clone - - name: linter - dependencies: [git-clone] - templateRef: - name: tslint-wft - template: tslint - - name: kaniko - dependencies: [git-clone] - templateRef: - name: kaniko-wft - template: kaniko - - name: trivy-fs-scan - dependencies: [git-clone] - templateRef: - name: trivy-fs-scan-wft - template: trivy-fs-scan - - name: crane - dependencies: [git-clone,linter,kaniko,trivy-fs-scan] - templateRef: - name: crane-wft - template: crane - - name: exit-handler - container: - image: alpine:latest - command: [sh, -c] - args: ["echo {{workflow.name}} {{workflow.status}} {{workflow.duration}}"] - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml deleted file mode 100644 index 3d5c3c81..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/sillypy-dag-wft.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: sillypy-dag-template -spec: - arguments: - parameters: - - name: image - value: harbor.aws-test.cgdevx.io/sillypy/sillypy - - name: tag - value: "0.3" - - name: workload - value: sillypy - - name: revision - value: "v1.0" - - name: repo - value: git@github.com:CGDevX-Demo/workload-sillypy.git - volumeClaimTemplates: - - metadata: - name: build - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 4Gi - entrypoint: ci-sequence - onExit: exit-handler - volumes: - # - name: build - # persistentVolumeClaim: -#claimName: build-volume-claim - - name: kaniko-secret - secret: - secretName: harbcred - items: - - key: .dockerconfigjson - path: config.json - - templates: - - name: ci-sequence - dag: - tasks: - - name: git-clone - templateRef: - name: git-clone-wft - template: git-clone - - name: linter - dependencies: [git-clone] - templateRef: - name: black-wft - template: black - - name: kaniko - dependencies: [linter] - templateRef: - name: kaniko-wft - template: kaniko - - name: trivy-fs-scan - dependencies: [kaniko] - templateRef: - name: trivy-fs-scan-wft - template: trivy-fs-scan - - name: crane - dependencies: [git-clone,linter,kaniko,trivy-fs-scan] - templateRef: - name: crane-wft - template: crane - - name: exit-handler - container: - image: alpine:latest - command: [sh, -c] - args: ["echo {{workflow.name}} {{workflow.status}} {{workflow.duration}}"] - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml deleted file mode 100644 index 3812899d..00000000 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/tslint-wft.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: tslint-wft -spec: - templates: - - name: tslint - container: - image: registry.gitlab.com/pipeline-components/tslint:latest - imagePullPolicy: IfNotPresent - command: ["tslint", "/{{workflow.parameters.workload}}/src/**/*.ts"] - volumeMounts: - - name: build - mountPath: "{{workflow.parameters.workload}}" - diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/externalsecret.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/externalsecret.yaml index a276c8c4..68d297ba 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/externalsecret.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/externalsecret.yaml @@ -57,3 +57,28 @@ spec: key: registry-auth property: auth secretKey: config.json +--- +apiVersion: 'external-secrets.io/v1beta1' +kind: ExternalSecret +metadata: + name: docker-config + annotations: + argocd.argoproj.io/sync-wave: '0' +spec: + target: + template: + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: "{{ .dockerconfig | toString }}" + name: docker-config + creationPolicy: Owner + secretStoreRef: + kind: ClusterSecretStore + name: vault-kv-secret + refreshInterval: 10s + data: + - secretKey: "dockerconfig" + remoteRef: + property: dockerconfig + key: dockerconfigjson + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml index 12ffd117..d5f7237a 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml @@ -55,7 +55,7 @@ spec: - app.kubernetes.io/name=argo-workflow-controller - --timeout=90s - --watch=true - image: registry.hub.docker.com/bitnami/kubectl: + image: registry.hub.docker.com/bitnami/kubectl:1.27.4 imagePullPolicy: IfNotPresent name: wait restartPolicy: OnFailure @@ -83,7 +83,7 @@ spec: - app.kubernetes.io/name=argo-server - --timeout=90s - --watch=true - image: registry.hub.docker.com/bitnami/kubectl: + image: registry.hub.docker.com/bitnami/kubectl:1.27.4 imagePullPolicy: IfNotPresent name: wait restartPolicy: OnFailure diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml new file mode 100644 index 00000000..59cfac10 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: black-wft +spec: + templates: + - name: black + container: + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pyfound/black" + imagePullPolicy: IfNotPresent + command: ["black", "--check", "-v", "/workspace/{{workflow.parameters.wl-service-name}}"] + volumeMounts: + - name: build + mountPath: /workspace + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml new file mode 100644 index 00000000..f822d4e4 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml @@ -0,0 +1,18 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: crane-wft +spec: + templates: + - name: crane + container: + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.gcr-proxy-prefix}}/go-containerregistry/crane" + imagePullPolicy: IfNotPresent + args: ["push", "/workspace/{{workflow.parameters.wl-service-name}}.tar", + "{{workflow.parameters.registry}}/{{workflow.parameters.workload-name}}/{{workflow.parameters.wl-service-name}}:{{workflow.parameters.tag}}"] + volumeMounts: + - name: kaniko-secret + mountPath: /home/nonroot/.docker + - name: build + mountPath: /workspace + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml new file mode 100644 index 00000000..e5ef758e --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: eslint-wft +spec: + templates: + - name: eslint + container: + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pipeline-components/eslint:latest" + imagePullPolicy: IfNotPresent + command: ["eslint", "$( [[ -e /build/{{workflow.parameters.wl-service-name}}/.eslintrc ]] || echo '--no-eslintrc' )", "/build/{{workflow.parameters.wl-service-name}}/src/**/*.ts"] + volumeMounts: + - name: build + mountPath: /build + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml new file mode 100644 index 00000000..e1268712 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: git-clone-wft +spec: + templates: + - name: git-clone + inputs: + artifacts: + - name: git-src + path: /src + git: + repo: "{{workflow.parameters.repo}}" + revision: "{{workflow.parameters.tag}}" + sshPrivateKeySecret: + name: ci-secrets + key: SSH_PRIVATE_KEY + depth: 1 + + script: + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/alpine/git" + imagePullPolicy: IfNotPresent + env: + - name: WLSERVICE + value: "{{workflow.parameters.wl-service-name}}" + - name: MIRROR_PREFIX + value: "{{workflow.parameters.dockerhub-proxy-prefix}}" + command: [sh] + source: | + echo "WLSERVICE: $WLSERVICE" + cp -rv /src/$WLSERVICE /build/ + if [[ $MIRROR_PREFIX ]] + then + cd /build/$WLSERVICE + sed -r -i.orig "s/FROM\s+(\S+(\/)\S+)/FROM $MIRROR_PREFIX\/\1/;s/FROM\s+(\S+)/FROM $MIRROR_PREFIX\/library\/\1/" Dockerfile + fi + cat /src/$WLSERVICE/?ockerfile + volumeMounts: + - mountPath: "/build" + name: build + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml new file mode 100644 index 00000000..439e10db --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml @@ -0,0 +1,25 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: kaniko-wft +spec: + templates: + - name: kaniko + container: + env: + - name: WL-SERVICE + value: "{{workflow.parameters.wl-service-name}}" + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.gcr-proxy-prefix}}/kaniko-project/executor" + imagePullPolicy: IfNotPresent + args: ["--dockerfile=Dockerfile", + "--context=dir:///workspace/{{workflow.parameters.wl-service-name}}/", + "--no-push", "--tar-path=/workspace/{{workflow.parameters.wl-service-name}}.tar", + "--registry-mirror={{workflow.parameters.registry-mirror}}", + "--skip-default-registry-fallback" + ] + volumeMounts: + - name: kaniko-secret + mountPath: /kaniko/.docker + - name: build + mountPath: /workspace + diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/trivy-fs-scan-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml similarity index 55% rename from platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/trivy-fs-scan-wft.yaml rename to platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml index d55d228e..f844e3d4 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates/workflowtemplates/trivy-fs-scan-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml @@ -9,12 +9,12 @@ spec: templates: - name: trivy-fs-scan container: - image: aquasec/trivy + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/aquasec/trivy" + imagePullPolicy: IfNotPresent args: - fs - - /workdir - # - /workdir/{{workflow.parameters.workload}} + - /build/{{workflow.parameters.wl-service-name}} volumeMounts: - name: build - mountPath: /workdir + mountPath: /build diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml new file mode 100644 index 00000000..4b8a7dac --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: tslint-wft +spec: + templates: + - name: tslint + container: + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pipelinecomponents/tslint" + imagePullPolicy: IfNotPresent + command: ["tslint", "-c", "/build/{{workflow.parameters.wl-service-name}}/tslint.json", "/build/{{workflow.parameters.wl-service-name}}/src/**/*.ts"] + volumeMounts: + - name: build + mountPath: /build + From ee4568599e371675ba47f6f63623c6305f91fb6d Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Fri, 29 Dec 2023 18:21:04 +0200 Subject: [PATCH 27/49] fix: put back files with parametrization, argo-workflow-templates.yaml instead of argo-workflows-templates.yaml --- .../components/argo-workflows/application.yaml | 16 +++++++--------- .../argo-cluster-workflow-templates.yaml | 2 +- .../argo-workflows/argo-workflow-templates.yaml | 2 +- .../components/argo-workflows/wait.yaml | 4 ++-- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml index 5776fff1..ef097559 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml @@ -13,8 +13,6 @@ spec: helm: values: |- nameOverride: argo - images: - pullPolicy: IfNotPresent executor: resources: requests: @@ -29,7 +27,7 @@ spec: create: true name: argo-server annotations: - eks.amazonaws.com/role-arn: "arn:aws:iam::728538925474:role/cg-devx-delta-ci-role" + : "" extraArgs: - --secure - --auth-mode=sso @@ -39,24 +37,24 @@ spec: annotations: cert-manager.io/cluster-issuer: "letsencrypt-prod" hosts: - - argo.delta.cgdevx.io + - paths: - / pathType: Prefix tls: - secretName: argo-tls hosts: - - argo.delta.cgdevx.io + - sso: enabled: true - issuer: https://vault.delta.cgdevx.io/v1/identity/oidc/provider/cgdevx + issuer: https:// clientId: name: argo-secrets key: client-id clientSecret: name: argo-secrets key: client-secret - redirectUrl: https://argo.delta.cgdevx.io/oauth2/callback + redirectUrl: https:// scopes: - email - openid @@ -72,7 +70,7 @@ spec: archiveLogs: false s3: insecure: false - bucket: cg-devx-delta-artifacts-repository-cjm7m0hm + bucket: keyFormat: "argo-workflows/artifacts\ /{{workflow.creationTimestamp.Y}}\ /{{workflow.creationTimestamp.m}}\ @@ -81,7 +79,7 @@ spec: /{{workflow.name}}\ /{{pod.name}}" endpoint: s3.amazonaws.com - region: eu-west-2 + region: useSDKCreds: true encryptionOptions: enableEncryption: false diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-cluster-workflow-templates.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-cluster-workflow-templates.yaml index 59858d48..822103e7 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-cluster-workflow-templates.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-cluster-workflow-templates.yaml @@ -8,7 +8,7 @@ metadata: spec: project: core source: - repoURL: git@github.com:CGDevX-Demo/cg-devx-delta.git + repoURL: path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/cluster-workflow-templates targetRevision: HEAD destination: diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml index 518373a3..d51af162 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/argo-workflow-templates.yaml @@ -8,7 +8,7 @@ metadata: spec: project: core source: - repoURL: git@github.com:CGDevX-Demo/cg-devx-delta.git + repoURL: path: gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates targetRevision: HEAD destination: diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml index d5f7237a..12ffd117 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/wait.yaml @@ -55,7 +55,7 @@ spec: - app.kubernetes.io/name=argo-workflow-controller - --timeout=90s - --watch=true - image: registry.hub.docker.com/bitnami/kubectl:1.27.4 + image: registry.hub.docker.com/bitnami/kubectl: imagePullPolicy: IfNotPresent name: wait restartPolicy: OnFailure @@ -83,7 +83,7 @@ spec: - app.kubernetes.io/name=argo-server - --timeout=90s - --watch=true - image: registry.hub.docker.com/bitnami/kubectl:1.27.4 + image: registry.hub.docker.com/bitnami/kubectl: imagePullPolicy: IfNotPresent name: wait restartPolicy: OnFailure From aff7119ceb1f59f4e5f46fcdb1f72553e63feaf6 Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Fri, 29 Dec 2023 18:45:15 +0200 Subject: [PATCH 28/49] argo workflows application.yaml images.pullPolicy --- .../core-services/components/argo-workflows/application.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml index ef097559..6e4efbc6 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/application.yaml @@ -13,6 +13,8 @@ spec: helm: values: |- nameOverride: argo + images: + pullPolicy: IfNotPresent executor: resources: requests: From cba3e6c219797791a9abef5f2b3a71744a32e9f1 Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Fri, 29 Dec 2023 23:54:02 +0200 Subject: [PATCH 29/49] feat: version-changer-wft --- .../version-changer-wft.yaml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml new file mode 100644 index 00000000..4daffdc1 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml @@ -0,0 +1,69 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: version-changer-wft + namespace: argo +spec: + entrypoint: version-changer + templates: + - name: version-changer + inputs: + artifacts: + - name: git-src + path: /src + git: + repo: "{{workflow.parameters.repo}}" + #revision: "{{workflow.parameters.tag}}" + sshPrivateKeySecret: + name: ci-secrets + key: SSH_PRIVATE_KEY + depth: 1 + + script: + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/alpine/git" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: ssh-key-vol + mountPath: "/etc/ssh-key" + env: + - name: WLSERVICE_LIST + value: "{{workflow.parameters.wl-service-list}}" + - name: ENV_PATH + value: "{{workflow.parameters.env-path}}" + - name: ENV_NAME + value: "{{workflow.parameters.env-name}}" + - name: NEW_TAG + value: "{{workflow.parameters.tag}}" + - name: MIRROR_PREFIX + value: "{{workflow.parameters.dockerhub-proxy-prefix}}" + command: [sh] + #gitops/environments/envs/dev/version.yaml + source: | + cd /src/${ENV_PATH}/${ENV_NAME} + ls -lr + if [ -e version.yaml ] + then + VERSION_YAML_CHANGED='' + for SVC_NAME in $WLSERVICE_LIST + do + echo $SVC_NAME + sed -i -r "s/(^\s*image:.+$SVC_NAME):(.+)$/\1:$NEW_TAG\"/" version.yaml + done + cat version.yaml + mkdir ~/.ssh + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + cp /etc/ssh-key/SSH_PRIVATE_KEY ~/.ssh/id_rsa + ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub + ls -l ~/.ssh + git config --global user.email "commiter@example.com" + git config --global user.name "CG Commiter" + git add . && \ + git commit -m "Tag updated for $WLSERVICE_LIST to $NEW_TAG" \ + && git push + fi + if [ $? -gt 0 ] + then + echo "Please read error explanation above." + else + echo "Commit and Push successful." + fi From b0650fa400471eed5858dfead1f6d02d92eb6298 Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Tue, 2 Jan 2024 01:20:28 +0200 Subject: [PATCH 30/49] feat: eslint with plugins wft --- .../workflow-templates/eslint-wft.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml index e5ef758e..c85d8870 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml @@ -5,10 +5,19 @@ metadata: spec: templates: - name: eslint - container: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pipeline-components/eslint:latest" + script: + image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pipelinecomponents/eslint:latest" imagePullPolicy: IfNotPresent - command: ["eslint", "$( [[ -e /build/{{workflow.parameters.wl-service-name}}/.eslintrc ]] || echo '--no-eslintrc' )", "/build/{{workflow.parameters.wl-service-name}}/src/**/*.ts"] + env: + - name: WLSERVICE + value: "{{workflow.parameters.wl-service-name}}" + command: [sh] + source: | + echo $WLSERVICE + cd /build/$WLSERVICE + ls src/**/*.ts* > /dev/null 2>&1 || { echo "Nothing to lint, exiting"; exit 1;} + npm -s --prefix /app install eslint-config-standard-with-typescript eslint-plugin-prettier eslint-config-prettier eslint-plugin-only-warn + eslint --plugin only-warn 'src/**/*.ts?' volumeMounts: - name: build mountPath: /build From 8ba9c9a3a168a8dd3b67c5ea7cddcefce364f3a9 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Tue, 2 Jan 2024 18:55:11 +0100 Subject: [PATCH 31/49] fix: harbor robot permission and db restarts --- .../components/harbor/harbor.yaml | 8 ++ .../modules/registry_harbor/robot.tf | 89 +++++++++++++++---- 2 files changed, 79 insertions(+), 18 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/harbor/harbor.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/harbor/harbor.yaml index a4bcc702..d9334f13 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/harbor/harbor.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/harbor/harbor.yaml @@ -38,6 +38,14 @@ spec: kubernetes.io/ingress.class: nginx database: type: internal + internal: + resources: + limits: + cpu: 500m + memory: 800Mi + requests: + cpu: 500m + memory: 800Mi registry: # set the service account to be used, default if left empty serviceAccountName: "" diff --git a/platform/terraform/modules/registry_harbor/robot.tf b/platform/terraform/modules/registry_harbor/robot.tf index f6714449..c000201a 100644 --- a/platform/terraform/modules/registry_harbor/robot.tf +++ b/platform/terraform/modules/registry_harbor/robot.tf @@ -4,24 +4,77 @@ resource "harbor_robot_account" "main" { level = "system" secret = var.registry_main_robot_password #modify permissions to required later + permissions { - access { - action = "create" - resource = "labels" - } - kind = "system" - namespace = "/" - } - permissions { - access { - action = "pull" - resource = "repository" - } - access { - action = "push" - resource = "repository" - } kind = "project" namespace = "*" - } -} + + access { + action = "create" + resource = "scan" + } + access { + action = "create" + effect = "allow" + resource = "artifact-label" + } + access { + action = "create" + effect = "allow" + resource = "tag" + } + access { + action = "delete" + effect = "allow" + resource = "tag" + } + access { + action = "list" + resource = "tag" + } + access { + action = "list" + effect = "allow" + resource = "artifact" + } + access { + action = "list" + effect = "allow" + resource = "repository" + } + access { + action = "pull" + effect = "allow" + resource = "repository" + } + access { + action = "push" + effect = "allow" + resource = "repository" + } + access { + action = "read" + effect = "allow" + resource = "artifact" + } + access { + action = "stop" + resource = "scan" + } + access { + action = "delete" + effect = "allow" + resource = "artifact-label" + } + access { + action = "delete" + effect = "allow" + resource = "repository" + } + access { + action = "delete" + effect = "allow" + resource = "artifact" + } + } +} \ No newline at end of file From c504dc749c7b5ed708d7fe1e590b582353049ad0 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Wed, 3 Jan 2024 13:32:39 +0100 Subject: [PATCH 32/49] - refactor wft - add additional wft parameters - fix aws state bucket protection - fix vcs branch protection --- .../workflow-templates/black-wft.yaml | 2 +- .../workflow-templates/crane-wft.yaml | 2 +- .../workflow-templates/eslint-wft.yaml | 8 +-- .../workflow-templates/git-clone-wft.yaml | 20 +++---- .../workflow-templates/kaniko-wft.yaml | 6 +-- .../workflow-templates/trivy-fs-scan-wft.yaml | 2 +- .../workflow-templates/tslint-wft.yaml | 2 +- .../version-changer-wft.yaml | 14 ++--- .../workloads/workload-template.yaml | 4 -- .../modules/vcs_github/gitops_repo.tf | 2 +- .../modules/vcs_github/repository/main.tf | 2 +- .../modules/vcs_github/repository/variable.tf | 6 +++ tools/cli/commands/README.md | 12 ++++- tools/cli/commands/destroy.py | 51 +++++++++++------- tools/cli/commands/setup.py | 9 ++-- tools/cli/commands/workload/README.md | 7 ++- tools/cli/commands/workload/bootstrap.py | 13 +++-- tools/cli/common/state_store.py | 54 +++++++++---------- tools/cli/services/cloud/aws/aws_sdk.py | 19 +++++-- tools/cli/services/platform_gitops.py | 18 ++++--- 20 files changed, 151 insertions(+), 102 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml index 59cfac10..78242707 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/black-wft.yaml @@ -6,7 +6,7 @@ spec: templates: - name: black container: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pyfound/black" + image: "{{workflow.parameters.dockerhub-registry-proxy}}/pyfound/black" imagePullPolicy: IfNotPresent command: ["black", "--check", "-v", "/workspace/{{workflow.parameters.wl-service-name}}"] volumeMounts: diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml index f822d4e4..08b0ba42 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/crane-wft.yaml @@ -6,7 +6,7 @@ spec: templates: - name: crane container: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.gcr-proxy-prefix}}/go-containerregistry/crane" + image: "{{workflow.parameters.gcr-registry-proxy}}/go-containerregistry/crane" imagePullPolicy: IfNotPresent args: ["push", "/workspace/{{workflow.parameters.wl-service-name}}.tar", "{{workflow.parameters.registry}}/{{workflow.parameters.workload-name}}/{{workflow.parameters.wl-service-name}}:{{workflow.parameters.tag}}"] diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml index c85d8870..556d96e3 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/eslint-wft.yaml @@ -6,15 +6,15 @@ spec: templates: - name: eslint script: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pipelinecomponents/eslint:latest" + image: "{{workflow.parameters.dockerhub-registry-proxy}}/pipelinecomponents/eslint:latest" imagePullPolicy: IfNotPresent env: - - name: WLSERVICE + - name: WL_SERVICE value: "{{workflow.parameters.wl-service-name}}" command: [sh] source: | - echo $WLSERVICE - cd /build/$WLSERVICE + echo $WL_SERVICE + cd /build/$WL_SERVICE ls src/**/*.ts* > /dev/null 2>&1 || { echo "Nothing to lint, exiting"; exit 1;} npm -s --prefix /app install eslint-config-standard-with-typescript eslint-plugin-prettier eslint-config-prettier eslint-plugin-only-warn eslint --plugin only-warn 'src/**/*.ts?' diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml index e1268712..ae8804cb 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml @@ -18,23 +18,23 @@ spec: depth: 1 script: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/alpine/git" + image: "{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git" imagePullPolicy: IfNotPresent env: - - name: WLSERVICE + - name: WL_SERVICE value: "{{workflow.parameters.wl-service-name}}" - - name: MIRROR_PREFIX - value: "{{workflow.parameters.dockerhub-proxy-prefix}}" + - name: REGISTRY_MIRROR + value: "{{workflow.parameters.dockerhub-registry-proxy}}" command: [sh] source: | - echo "WLSERVICE: $WLSERVICE" - cp -rv /src/$WLSERVICE /build/ - if [[ $MIRROR_PREFIX ]] + echo "WL_SERVICE: $WL_SERVICE" + cp -rv /src/$WL_SERVICE /build/ + if [[ $REGISTRY_MIRROR ]] then - cd /build/$WLSERVICE - sed -r -i.orig "s/FROM\s+(\S+(\/)\S+)/FROM $MIRROR_PREFIX\/\1/;s/FROM\s+(\S+)/FROM $MIRROR_PREFIX\/library\/\1/" Dockerfile + cd /build/$WL_SERVICE + sed -r -i.orig "s/FROM\s+(\S+(\/)\S+)/FROM $REGISTRY_MIRROR\/\1/;s/FROM\s+(\S+)/FROM $REGISTRY_MIRROR\/library\/\1/" Dockerfile fi - cat /src/$WLSERVICE/?ockerfile + cat /src/$WL_SERVICE/?ockerfile volumeMounts: - mountPath: "/build" name: build diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml index 439e10db..e25f53b3 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/kaniko-wft.yaml @@ -7,14 +7,14 @@ spec: - name: kaniko container: env: - - name: WL-SERVICE + - name: WL_SERVICE value: "{{workflow.parameters.wl-service-name}}" - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.gcr-proxy-prefix}}/kaniko-project/executor" + image: "{{workflow.parameters.gcr-registry-proxy}}/kaniko-project/executor" imagePullPolicy: IfNotPresent args: ["--dockerfile=Dockerfile", "--context=dir:///workspace/{{workflow.parameters.wl-service-name}}/", "--no-push", "--tar-path=/workspace/{{workflow.parameters.wl-service-name}}.tar", - "--registry-mirror={{workflow.parameters.registry-mirror}}", + "--registry-mirror={{workflow.parameters.registry}}", "--skip-default-registry-fallback" ] volumeMounts: diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml index f844e3d4..628578d5 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/trivy-fs-scan-wft.yaml @@ -9,7 +9,7 @@ spec: templates: - name: trivy-fs-scan container: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/aquasec/trivy" + image: "{{workflow.parameters.dockerhub-registry-proxy}}/aquasec/trivy" imagePullPolicy: IfNotPresent args: - fs diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml index 4b8a7dac..4598480f 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/tslint-wft.yaml @@ -6,7 +6,7 @@ spec: templates: - name: tslint container: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/pipelinecomponents/tslint" + image: "{{workflow.parameters.dockerhub-registry-proxy}}/pipelinecomponents/tslint" imagePullPolicy: IfNotPresent command: ["tslint", "-c", "/build/{{workflow.parameters.wl-service-name}}/tslint.json", "/build/{{workflow.parameters.wl-service-name}}/src/**/*.ts"] volumeMounts: diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml index 4daffdc1..8176be97 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml @@ -20,13 +20,13 @@ spec: depth: 1 script: - image: "{{workflow.parameters.registry-mirror}}/{{workflow.parameters.dockerhub-proxy-prefix}}/alpine/git" + image: "{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git" imagePullPolicy: IfNotPresent volumeMounts: - name: ssh-key-vol mountPath: "/etc/ssh-key" env: - - name: WLSERVICE_LIST + - name: WL_SERVICE_LIST value: "{{workflow.parameters.wl-service-list}}" - name: ENV_PATH value: "{{workflow.parameters.env-path}}" @@ -35,7 +35,7 @@ spec: - name: NEW_TAG value: "{{workflow.parameters.tag}}" - name: MIRROR_PREFIX - value: "{{workflow.parameters.dockerhub-proxy-prefix}}" + value: "{{workflow.parameters.dockerhub-registry-proxy}}" command: [sh] #gitops/environments/envs/dev/version.yaml source: | @@ -44,7 +44,7 @@ spec: if [ -e version.yaml ] then VERSION_YAML_CHANGED='' - for SVC_NAME in $WLSERVICE_LIST + for SVC_NAME in $WL_SERVICE_LIST do echo $SVC_NAME sed -i -r "s/(^\s*image:.+$SVC_NAME):(.+)$/\1:$NEW_TAG\"/" version.yaml @@ -55,10 +55,10 @@ spec: cp /etc/ssh-key/SSH_PRIVATE_KEY ~/.ssh/id_rsa ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub ls -l ~/.ssh - git config --global user.email "commiter@example.com" - git config --global user.name "CG Commiter" + git config --global user.email "" + git config --global user.name "" git add . && \ - git commit -m "Tag updated for $WLSERVICE_LIST to $NEW_TAG" \ + git commit -m "Tag updated for $WL_SERVICE_LIST to $NEW_TAG" \ && git push fi if [ $? -gt 0 ] diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml index 69d68c17..0aa59716 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/workloads/workload-template.yaml @@ -80,7 +80,3 @@ spec: duration: 5s maxDuration: 5m0s factor: 2 - syncPolicy: - automated: - prune: true - selfHeal: true diff --git a/platform/terraform/modules/vcs_github/gitops_repo.tf b/platform/terraform/modules/vcs_github/gitops_repo.tf index 1770e04c..b7456ab2 100644 --- a/platform/terraform/modules/vcs_github/gitops_repo.tf +++ b/platform/terraform/modules/vcs_github/gitops_repo.tf @@ -6,7 +6,7 @@ module "gitops-repo" { atlantis_enabled = true atlantis_url = var.atlantis_url atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret - + vcs_subscription_plan = var.vcs_subscription_plan } output "gitops_repo_git_clone_url" { diff --git a/platform/terraform/modules/vcs_github/repository/main.tf b/platform/terraform/modules/vcs_github/repository/main.tf index 20702e2a..a4bf58ea 100644 --- a/platform/terraform/modules/vcs_github/repository/main.tf +++ b/platform/terraform/modules/vcs_github/repository/main.tf @@ -31,7 +31,7 @@ resource "github_repository" "repo" { # only allow the engineers team merge to the branch. resource "github_branch_protection" "this" { - count = var.branch_protection ? 1 : 0 + count = var.branch_protection && var.vcs_subscription_plan ? 1 : 0 repository_id = github_repository.repo.node_id pattern = "main" diff --git a/platform/terraform/modules/vcs_github/repository/variable.tf b/platform/terraform/modules/vcs_github/repository/variable.tf index cc4260c7..a3c37743 100644 --- a/platform/terraform/modules/vcs_github/repository/variable.tf +++ b/platform/terraform/modules/vcs_github/repository/variable.tf @@ -68,3 +68,9 @@ variable "atlantis_repo_webhook_secret" { default = "" sensitive = true } + +variable "vcs_subscription_plan" { + description = "True for advanced github/gitlab plan. False for free tier" + type = bool + default = false +} diff --git a/tools/cli/commands/README.md b/tools/cli/commands/README.md index 201e6fcc..6dab5bda 100644 --- a/tools/cli/commands/README.md +++ b/tools/cli/commands/README.md @@ -112,7 +112,15 @@ process. - Remote backend storage (e.g., AWS S3) used for IaC - All local files created by CG DevX CLI -**NOTE!**: this process is irreversible +> **NOTE!**: this process is irreversible + +> **NOTE!**: This operation will delete all workload repositories if you have them. +> If workloads have any out of the cluster (cloud provider) resources, they will become orphaned, +> and should be deleted manually. +> It is highly recommended prior to destroying your installation to delete all active workloads first also deleting all +> the resources. +> Please see more on `workload delete` command with `--destroy-resources` flag [here](workload/README.md#delete). + **Arguments**: @@ -134,3 +142,5 @@ and then all other resources created by our automation. The cleanup process could still fail. If you have any issues, please try restarting the process. If it fails to delete your K8s cluster, please try deleting Load Balancer(s) manually and restart the process. +For GitHub, external action runners should be removed prior to repository deletion. +If it fails to delete your GitOps repo - please check and remove runners and restart the process. \ No newline at end of file diff --git a/tools/cli/commands/destroy.py b/tools/cli/commands/destroy.py index 7955cd22..cf0fc039 100644 --- a/tools/cli/commands/destroy.py +++ b/tools/cli/commands/destroy.py @@ -1,19 +1,19 @@ -import os import shutil import click import portforward import urllib3 -from common.utils.command_utils import init_cloud_provider, prepare_cloud_provider_auth_env_vars, set_envs, unset_envs, wait -from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_HOSTING_PROVIDER, LOCAL_FOLDER, \ - LOCAL_STATE_FILE +from common.const.common_path import LOCAL_TF_FOLDER_VCS, LOCAL_TF_FOLDER_HOSTING_PROVIDER, LOCAL_FOLDER from common.const.namespaces import ARGOCD_NAMESPACE from common.const.parameter_names import GIT_ACCESS_TOKEN, GIT_ORGANIZATION_NAME from common.logging_config import configure_logging from common.state_store import StateStore +from common.utils.command_utils import init_cloud_provider, prepare_cloud_provider_auth_env_vars, set_envs, unset_envs, \ + wait, init_git_provider, check_installation_presence from services.k8s.delivery_service_manager import DeliveryServiceManager, get_argocd_token, delete_application from services.k8s.k8s import KubeClient +from services.platform_gitops import PlatformGitOpsRepo from services.tf_wrapper import TfWrapper urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -30,15 +30,23 @@ def destroy(verbosity: str): # Set up global logger configure_logging(verbosity) - if not os.path.exists(LOCAL_STATE_FILE): - click.echo("CG DevX installation local files not found.") - return + check_installation_presence() + + # TODO: check if we could move vcs destroy to the last step p: StateStore = StateStore() + git_man = init_git_provider(p) + gor = PlatformGitOpsRepo(git_man, + author_name=p.internals["GIT_USER_NAME"], + author_email=p.internals["GIT_USER_EMAIL"], + key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) + + gor.update() + click.confirm( - f'This will destroy cluster "{p.parameters[""]}" and local files at "{LOCAL_FOLDER}". Please confirm to continue', - abort=True) + f'This will destroy cluster "{p.parameters[""]}" and local files at "{LOCAL_FOLDER}". ' + f'Please confirm to continue', abort=True) click.echo("Destroying CG DevX installation...") @@ -57,15 +65,6 @@ def destroy(verbosity: str): # set envs as required by tf set_envs(tf_env_vars) - if p.has_checkpoint("vcs-tf"): - click.echo("Destroying VCS...") - - tf_wrapper = TfWrapper(LOCAL_TF_FOLDER_VCS) - tf_wrapper.init() - tf_wrapper.destroy() - - click.echo("Destroying VCS. Done!") - # ArgoCD section # wait till resources are de-provisioned by ArgoCD before destroying K8s cluster if p.has_checkpoint("k8s-delivery"): @@ -80,10 +79,14 @@ def destroy(verbosity: str): cd_man.turn_off_app_sync(registry_app_name) cd_man.turn_off_app_sync("ingress-nginx-components") cd_man.turn_off_app_sync("ingress-nginx") + cd_man.turn_off_app_sync("github-runner-components") + cd_man.turn_off_app_sync("actions-runner-controller-components") # delete app cd_man.delete_app("ingress-nginx-components") cd_man.delete_app("ingress-nginx") + cd_man.delete_app("github-runner-components") + cd_man.delete_app("actions-runner-controller-components") except Exception as e: pass @@ -113,12 +116,22 @@ def destroy(verbosity: str): click.echo("Destroying K8s cluster. Done!") + if p.has_checkpoint("vcs-tf"): + click.echo("Destroying VCS...") + + tf_wrapper = TfWrapper(LOCAL_TF_FOLDER_VCS) + tf_wrapper.init() + tf_wrapper.destroy() + + click.echo("Destroying VCS. Done!") + # unset envs as no longer needed unset_envs(tf_env_vars) # delete IaC backend storage bucket if not cloud_man.destroy_iac_state_storage(p.internals["TF_BACKEND_STORAGE_NAME"]): - click.echo(f'Failed to delete IaC state storage {p.internals["TF_BACKEND_STORAGE_NAME"]}. You should delete it manually.') + click.echo(f'Failed to delete IaC state storage {p.internals["TF_BACKEND_STORAGE_NAME"]}. You should delete ' + f'it manually.') # delete local data folder shutil.rmtree(LOCAL_FOLDER) diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index a972d1f2..41776c8e 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -144,6 +144,7 @@ def setup( p.internals["GIT_USER_NAME"] = git_user_name p.parameters[""] = git_user_name p.internals["GIT_USER_EMAIL"] = git_user_email + p.parameters[""] = git_user_email p.fragments["# "] = git_man.create_tf_module_snippet() git_subscription_plan = git_man.get_organization_plan() @@ -767,10 +768,10 @@ def setup( "code_quality_admin_password": p.internals["CODE_QUALITY_PASSWORD"] }) core_services_out = tf_wrapper.output() - p.parameters[""] = core_services_out["dockerhub_proxy_name"] - p.parameters[""] = core_services_out["gcr_proxy_name"] - p.parameters[""] = core_services_out["k8s_gcr_proxy_name"] - p.parameters[""] = core_services_out["quay_proxy_name"] + p.parameters[""] = f'{p.parameters[""]}{core_services_out["dockerhub_proxy_name"]}' + p.parameters[""] = f'{p.parameters[""]}{core_services_out["gcr_proxy_name"]}' + p.parameters[""] = f'{p.parameters[""]}{core_services_out["k8s_gcr_proxy_name"]}' + p.parameters[""] = f'{p.parameters[""]}{core_services_out["quay_proxy_name"]}' # unset envs as no longer needed unset_envs(core_services_tf_env_vars) diff --git a/tools/cli/commands/workload/README.md b/tools/cli/commands/workload/README.md index e0b704e9..e6d6b062 100644 --- a/tools/cli/commands/workload/README.md +++ b/tools/cli/commands/workload/README.md @@ -112,10 +112,13 @@ comments section. `workload delete` command deletes all the configuration generated by `workload create` [command](#create): -When executed with --destroy-resources flag it will also destroy all the resources created for a specific workload. +When executed with `--destroy-resources` flag it will also destroy all the resources created for a specific workload. Please note that workload GitOps repository name should match one for workload. +When executing with `--destroy-resources` flag enabled it **must** be executed by cluster owner. +Under the hood, it will execute tf destroy locally, and tf state storage is protected and accessible only by the owner. -**NOTE!**: this process is irreversible + +> **NOTE!**: this process is irreversible `workload delete` command could be executed using arguments, or environment variables. diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 5f4a77a3..557864bb 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -2,11 +2,8 @@ import shutil import click -from ghrepo import GHRepo -from git import Repo, GitError, Actor -from common.const.common_path import LOCAL_GITOPS_FOLDER, LOCAL_FOLDER, LOCAL_WORKLOAD_TEMP_FOLDER -from common.const.const import WL_REPOSITORY_URL, WL_GITOPS_REPOSITORY_URL +from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER from common.logging_config import configure_logging from common.state_store import StateStore from common.utils.command_utils import str_to_kebab, init_cloud_provider, check_installation_presence @@ -93,6 +90,14 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp wl_repo_params = { "": wl_name, "": wl_svc_name, + "": p.parameters[""], + "": p.parameters[""], + "": p.parameters[""], + "": p.parameters[""], + "": p.parameters[""], + "": wl_repo_name, + "": wl_gitops_repo_name, + "": p.parameters[""], } wl_man.parametrise(wl_repo_params) diff --git a/tools/cli/common/state_store.py b/tools/cli/common/state_store.py index 652b3c5a..48fa6829 100644 --- a/tools/cli/common/state_store.py +++ b/tools/cli/common/state_store.py @@ -14,61 +14,61 @@ class StateStore: - __store: dict = {} + _store: dict = {} def __init__(self, input_params: None | dict = None): if input_params is None: input_params = {} - self.__store[STATE_CHECKPOINTS] = [] - self.__store[STATE_FRAGMENTS] = {} - self.__store[STATE_PARAMS] = {} - self.__store[STATE_INTERNAL_PARAMS] = {} - self.__store[STATE_INPUT_PARAMS] = {} + self._store[STATE_CHECKPOINTS] = [] + self._store[STATE_FRAGMENTS] = {} + self._store[STATE_PARAMS] = {} + self._store[STATE_INTERNAL_PARAMS] = {} + self._store[STATE_INPUT_PARAMS] = {} if os.path.exists(LOCAL_STATE_FILE): with open(LOCAL_STATE_FILE, "r+") as infile: config = yaml.safe_load(infile) try: - self.__store[STATE_CHECKPOINTS] = config[STATE_CHECKPOINTS] - self.__store[STATE_FRAGMENTS] = config[STATE_FRAGMENTS] - self.__store[STATE_PARAMS] = config[STATE_PARAMS] - self.__store[STATE_INTERNAL_PARAMS] = config[STATE_INTERNAL_PARAMS] - self.__store[STATE_INPUT_PARAMS] = config[STATE_INPUT_PARAMS] + self._store[STATE_CHECKPOINTS] = config[STATE_CHECKPOINTS] + self._store[STATE_FRAGMENTS] = config[STATE_FRAGMENTS] + self._store[STATE_PARAMS] = config[STATE_PARAMS] + self._store[STATE_INTERNAL_PARAMS] = config[STATE_INTERNAL_PARAMS] + self._store[STATE_INPUT_PARAMS] = config[STATE_INPUT_PARAMS] except KeyError as error: # ToDo: Handle missing parameters pass - self.__store[STATE_INPUT_PARAMS].update(input_params) + self._store[STATE_INPUT_PARAMS].update(input_params) @property def cloud_provider(self) -> CloudProviders: - return self.__store[STATE_INPUT_PARAMS][CLOUD_PROVIDER] + return self._store[STATE_INPUT_PARAMS][CLOUD_PROVIDER] @property def git_provider(self) -> GitProviders: - return self.__store[STATE_INPUT_PARAMS][GIT_PROVIDER] + return self._store[STATE_INPUT_PARAMS][GIT_PROVIDER] @property def dns_registrar(self) -> DnsRegistrars: - return self.__store[STATE_INPUT_PARAMS][DNS_REGISTRAR] + return self._store[STATE_INPUT_PARAMS][DNS_REGISTRAR] @dns_registrar.setter def dns_registrar(self, value): - self.__store[STATE_INPUT_PARAMS][DNS_REGISTRAR] = value + self._store[STATE_INPUT_PARAMS][DNS_REGISTRAR] = value @classmethod def get_input_param(cls, key): - if key in cls.__store[STATE_INPUT_PARAMS]: - return cls.__store[STATE_INPUT_PARAMS].get(key) + if key in cls._store[STATE_INPUT_PARAMS]: + return cls._store[STATE_INPUT_PARAMS].get(key) else: return None @classmethod def update_input_params(cls, input_params: dict): - cls.__store[STATE_INPUT_PARAMS].update(input_params) + cls._store[STATE_INPUT_PARAMS].update(input_params) @property def input_param(self): - return self.__store[STATE_INPUT_PARAMS] + return self._store[STATE_INPUT_PARAMS] @classmethod @trace() @@ -77,33 +77,33 @@ def validate_input_params(cls, validator): @property def fragments(self): - return self.__store[STATE_FRAGMENTS] + return self._store[STATE_FRAGMENTS] @property def parameters(self): - return self.__store[STATE_PARAMS] + return self._store[STATE_PARAMS] @property def internals(self): - return self.__store[STATE_INTERNAL_PARAMS] + return self._store[STATE_INTERNAL_PARAMS] @classmethod def set_parameter(cls, key, value): - cls.__store[STATE_INTERNAL_PARAMS][key] = value + cls._store[STATE_INTERNAL_PARAMS][key] = value @classmethod def set_checkpoint(cls, name: str): - cls.__store[STATE_CHECKPOINTS].append(name) + cls._store[STATE_CHECKPOINTS].append(name) @classmethod def has_checkpoint(cls, name: str): - return name in cls.__store[STATE_CHECKPOINTS] + return name in cls._store[STATE_CHECKPOINTS] @classmethod def save_checkpoint(cls): os.makedirs(os.path.dirname(LOCAL_STATE_FILE), exist_ok=True) with open(LOCAL_STATE_FILE, "w+") as outfile: - yaml.dump(cls.__store, outfile, default_flow_style=False) + yaml.dump(cls._store, outfile, default_flow_style=False) def param_validator(paras: StateStore) -> bool: diff --git a/tools/cli/services/cloud/aws/aws_sdk.py b/tools/cli/services/cloud/aws/aws_sdk.py index cdce5539..9c34f47f 100644 --- a/tools/cli/services/cloud/aws/aws_sdk.py +++ b/tools/cli/services/cloud/aws/aws_sdk.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple -import boto3 from awscli.customizations.eks.get_token import STSClientFactory, TokenGenerator, TOKEN_EXPIRATION_MINS from botocore.exceptions import ClientError @@ -130,15 +129,27 @@ def set_bucket_policy(self, bucket_name: str, identity: str, region: str = None) "Statement": [ { "Sid": "RestrictS3Access", + "Action": ["s3:*"], + "Effect": "Allow", "Principal": { "AWS": [ self.current_user_arn(), identity ] }, - "Effect": "Allow", + "Resource": [f"arn:aws:s3:::{bucket_name}"], + }, + { + "Sid": "ExplicitlyDenyS3Actions", "Action": ["s3:*"], - "Resource": [f"arn:aws:s3:::{bucket_name}"] + "Effect": "Deny", + "NotPrincipal": { + "AWS": [ + self.current_user_arn(), + identity + ] + }, + "Resource": [f"arn:aws:s3:::{bucket_name}"], } ] } @@ -243,7 +254,7 @@ def delete_bucket(self, bucket_name: str, region: str = None): If a region is not specified, the bucket is created in the S3 default region. :param bucket_name: Bucket to create - :param region: String region to create bucket in, e.g., 'us-west-2' + :param region: Region to create bucket in, e.g., 'us-west-2' :return: True if bucket deleted, else False """ diff --git a/tools/cli/services/platform_gitops.py b/tools/cli/services/platform_gitops.py index 21d0016f..c14ab2b4 100644 --- a/tools/cli/services/platform_gitops.py +++ b/tools/cli/services/platform_gitops.py @@ -17,15 +17,17 @@ def __init__(self, git_man: GitProviderManager, key_path: str = None, author_nam self._repo = Repo(LOCAL_GITOPS_FOLDER) self._git_man = git_man self._ssh_key_path = key_path + self._ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {self._ssh_key_path}' self._author_name = author_name self._author_email = author_email @trace() def update(self): - # clean stale branches - self._repo.remotes.origin.fetch(prune=True) - self._repo.heads.main.checkout() - self._repo.remotes.origin.pull(self._repo.active_branch) + with self._repo.git.custom_environment(GIT_SSH_COMMAND=self._ssh_cmd): + # clean stale branches + self._repo.remotes.origin.fetch(prune=True) + self._repo.heads.main.checkout() + self._repo.remotes.origin.pull(self._repo.active_branch) @trace() def branch_exist(self, branch_name): @@ -38,8 +40,7 @@ def create_branch(self, branch_name): @trace() def upload_changes(self): - ssh_cmd = f'ssh -o StrictHostKeyChecking=no -i {self._ssh_key_path}' - with self._repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + with self._repo.git.custom_environment(GIT_SSH_COMMAND=self._ssh_cmd): self._repo.git.add(all=True) author = Actor(name=self._author_name, email=self._author_email) self._repo.index.commit("initial", author=author, committer=author) @@ -107,7 +108,10 @@ def rm_workload(self, wl_name: str): os.remove(wl_argo_manifest) @staticmethod - def _add_wl_vars(tf_module_path, wl_name: str, payload: dict = {}): + def _add_wl_vars(tf_module_path, wl_name: str, payload=None): + if payload is None: + payload = {} + with open(tf_module_path / "terraform.tfvars.json", "r") as file: services_tf_vars = json.load(file) From 19696b476a9327c8b5965fd02ec3906825c8c131 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Wed, 3 Jan 2024 15:44:18 +0100 Subject: [PATCH 33/49] fix: aws s3 tf state bucket policy --- tools/cli/services/cloud/aws/aws_sdk.py | 34 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tools/cli/services/cloud/aws/aws_sdk.py b/tools/cli/services/cloud/aws/aws_sdk.py index 9c34f47f..550f9ccb 100644 --- a/tools/cli/services/cloud/aws/aws_sdk.py +++ b/tools/cli/services/cloud/aws/aws_sdk.py @@ -127,29 +127,39 @@ def set_bucket_policy(self, bucket_name: str, identity: str, region: str = None) bucket_policy = { "Version": "2012-10-17", "Statement": [ + # non-restrictive allow-list { "Sid": "RestrictS3Access", "Action": ["s3:*"], "Effect": "Allow", - "Principal": { - "AWS": [ - self.current_user_arn(), - identity - ] + "Principal": "*", + "Condition": { + "ArnLike": { + "aws:PrincipalArn": [ + self.current_user_arn(), + identity, + f"arn:aws:iam::{self._account_id}:root" + ] + } }, - "Resource": [f"arn:aws:s3:::{bucket_name}"], + "Resource": [f"arn:aws:s3:::{bucket_name}", f"arn:aws:s3:::{bucket_name}/*"], }, + # an explicit deny. this one is self-sufficient { "Sid": "ExplicitlyDenyS3Actions", "Action": ["s3:*"], "Effect": "Deny", - "NotPrincipal": { - "AWS": [ - self.current_user_arn(), - identity - ] + "Principal": "*", + "Condition": { + "ArnNotLike": { + "aws:PrincipalArn": [ + self.current_user_arn(), + identity, + f"arn:aws:iam::{self._account_id}:root" + ] + } }, - "Resource": [f"arn:aws:s3:::{bucket_name}"], + "Resource": [f"arn:aws:s3:::{bucket_name}", f"arn:aws:s3:::{bucket_name}/*"], } ] } From 28a8c27a3e6b54e8845db3e6927478376326278f Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Wed, 3 Jan 2024 23:30:43 +0200 Subject: [PATCH 34/49] fix: git-clone-wft.yaml feat: promote-wft.yaml fix: version-changer-wft.yaml --- .../workflow-templates/git-clone-wft.yaml | 9 ++- .../workflow-templates/promote-wft.yaml | 62 +++++++++++++++++++ .../version-changer-wft.yaml | 10 ++- 3 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml index ae8804cb..be49405c 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/git-clone-wft.yaml @@ -24,17 +24,22 @@ spec: - name: WL_SERVICE value: "{{workflow.parameters.wl-service-name}}" - name: REGISTRY_MIRROR + value: "{{workflow.parameters.registry}}" + - name: DOCKERHUB_REGISTRY_PROXY value: "{{workflow.parameters.dockerhub-registry-proxy}}" + + command: [sh] source: | + DOCKERHUB_PROXY_PREFIX=`basename $DOCKERHUB_REGISTRY_PROXY` echo "WL_SERVICE: $WL_SERVICE" cp -rv /src/$WL_SERVICE /build/ if [[ $REGISTRY_MIRROR ]] then cd /build/$WL_SERVICE - sed -r -i.orig "s/FROM\s+(\S+(\/)\S+)/FROM $REGISTRY_MIRROR\/\1/;s/FROM\s+(\S+)/FROM $REGISTRY_MIRROR\/library\/\1/" Dockerfile + sed -r -i.orig "s/FROM\s+(\S+(\/)\S+)/FROM $DOCKERHUB_PROXY_PREFIX\/\1/;s/FROM\s+(\S+)/FROM $DOCKERHUB_PROXY_PREFIX\/library\/\1/" Dockerfile + cat Dockerfile fi - cat /src/$WL_SERVICE/?ockerfile volumeMounts: - mountPath: "/build" name: build diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml new file mode 100644 index 00000000..573df5a7 --- /dev/null +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/promote-wft.yaml @@ -0,0 +1,62 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: promote-wft + namespace: argo +spec: + entrypoint: promote + templates: + - name: promote + inputs: + artifacts: + - name: git-src + path: /src + git: + repo: "{{workflow.parameters.repo}}" + sshPrivateKeySecret: + name: ci-secrets + key: SSH_PRIVATE_KEY + depth: 1 + + script: + image: "{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: ssh-key-vol + mountPath: "/etc/ssh-key" + env: + - name: ENV_PATH + value: "{{workflow.parameters.env-path}}" + - name: SOURCE_ENV + value: "{{workflow.parameters.source-env}}" + - name: TARGET_ENV + value: "{{workflow.parameters.target-env}}" + - name: PROMOTE_SETTINGS + value: "{{workflow.parameters.promote-settings}}" + - name: PROMOTE_CONTAINER + value: "{{workflow.parameters.promote-container}}" + - name: PROMOTE_CONFIGMAPS + value: "{{workflow.parameters.promote-configmaps}}" + - name: COMMIT_MESSAGE + value: "{{workflow.parameters.commit-message}}" + command: [sh] + source: | + cd /src + [[ $PROMOTE_CONTAINER == 'true' ]] && cp -v "${ENV_PATH}/${SOURCE_ENV}/version.yaml" "${ENV_PATH}/${TARGET_ENV}" + [[ $PROMOTE_SETTINGS == 'true' ]] && cp -v "${ENV_PATH}/${SOURCE_ENV}/settings.yaml" "${ENV_PATH}/${TARGET_ENV}" + [[ $PROMOTE_CONFIGMAPS == 'true' ]] && cp -v "${ENV_PATH}/${SOURCE_ENV}/cm.yaml" "${ENV_PATH}/${TARGET_ENV}" + mkdir ~/.ssh + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + cp /etc/ssh-key/SSH_PRIVATE_KEY ~/.ssh/id_rsa + ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub + git config --global user.email "" + git config --global user.name "" + git add . && \ + git commit -m "${COMMIT_MESSAGE}" \ + && git push + if [ $? -gt 0 ] + then + echo "Please read error explanation above." + else + echo "Commit and Push successful." + fi diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml index 8176be97..0979917a 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml @@ -13,7 +13,7 @@ spec: path: /src git: repo: "{{workflow.parameters.repo}}" - #revision: "{{workflow.parameters.tag}}" + #revision: "{{workflow.parameters.tag}}" sshPrivateKeySecret: name: ci-secrets key: SSH_PRIVATE_KEY @@ -37,17 +37,15 @@ spec: - name: MIRROR_PREFIX value: "{{workflow.parameters.dockerhub-registry-proxy}}" command: [sh] - #gitops/environments/envs/dev/version.yaml source: | cd /src/${ENV_PATH}/${ENV_NAME} ls -lr if [ -e version.yaml ] then - VERSION_YAML_CHANGED='' for SVC_NAME in $WL_SERVICE_LIST do echo $SVC_NAME - sed -i -r "s/(^\s*image:.+$SVC_NAME):(.+)$/\1:$NEW_TAG\"/" version.yaml + sed -i -r "s/(^\s*image:.+$SVC_NAME).*$/\1:$NEW_TAG\"/" version.yaml done cat version.yaml mkdir ~/.ssh @@ -55,8 +53,8 @@ spec: cp /etc/ssh-key/SSH_PRIVATE_KEY ~/.ssh/id_rsa ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub ls -l ~/.ssh - git config --global user.email "" - git config --global user.name "" + git config --global user.email "cg-devx-automation@cloudgeometry.io" + git config --global user.name "cgdevx-bot" git add . && \ git commit -m "Tag updated for $WL_SERVICE_LIST to $NEW_TAG" \ && git push From 453cbdedd67c119f9d8897e16bf727aabbea9554 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Thu, 4 Jan 2024 17:27:43 +0100 Subject: [PATCH 35/49] feat: add default git runner group param --- tools/cli/commands/setup.py | 3 +++ tools/cli/commands/workload/bootstrap.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 41776c8e..31fe2dd2 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -152,6 +152,9 @@ def setup( if git_subscription_plan > 0: p.fragments["# "] = git_man.create_runner_group_snippet() p.parameters[""] = p.get_input_param(PRIMARY_CLUSTER_NAME) + else: + # match the GitHub's default runner group + p.parameters[""] = "Default" dns_provider_check(dns_man, p) click.echo("DNS provider pre-flight check. Done!") diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 557864bb..583cb242 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -98,6 +98,7 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp "": wl_repo_name, "": wl_gitops_repo_name, "": p.parameters[""], + "": p.parameters[""], } wl_man.parametrise(wl_repo_params) @@ -139,6 +140,7 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], f"workloads/{wl_name}/hosting_provider"), "# ": cloud_man.create_hosting_provider_snippet(), + "": p.parameters[""], } wl_ops_man.parametrise(wl_gitops_repo_params) From f544bb133bcf73900f7010783ef33a0a7c05f57e Mon Sep 17 00:00:00 2001 From: VADIM TSARFIN Date: Thu, 4 Jan 2024 20:11:00 +0200 Subject: [PATCH 36/49] fix: version-changer-wft w <> var for git name and email --- .../version-changer-wft.yaml | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml index 0979917a..80a31090 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: @@ -6,38 +7,37 @@ metadata: spec: entrypoint: version-changer templates: - - name: version-changer - inputs: - artifacts: - - name: git-src - path: /src - git: - repo: "{{workflow.parameters.repo}}" - #revision: "{{workflow.parameters.tag}}" - sshPrivateKeySecret: - name: ci-secrets - key: SSH_PRIVATE_KEY - depth: 1 - - script: - image: "{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git" + - name: version-changer + inputs: + artifacts: + - name: git-src + path: /src + git: + repo: '{{workflow.parameters.repo}}' + # revision: "{{workflow.parameters.tag}}" + sshPrivateKeySecret: + name: ci-secrets + key: SSH_PRIVATE_KEY + depth: 1 + script: + image: '{{workflow.parameters.dockerhub-registry-proxy}}/alpine/git' imagePullPolicy: IfNotPresent volumeMounts: - - name: ssh-key-vol - mountPath: "/etc/ssh-key" + - name: ssh-key-vol + mountPath: /etc/ssh-key env: - name: WL_SERVICE_LIST - value: "{{workflow.parameters.wl-service-list}}" + value: '{{workflow.parameters.wl-service-list}}' - name: ENV_PATH - value: "{{workflow.parameters.env-path}}" + value: '{{workflow.parameters.env-path}}' - name: ENV_NAME - value: "{{workflow.parameters.env-name}}" + value: '{{workflow.parameters.env-name}}' - name: NEW_TAG - value: "{{workflow.parameters.tag}}" + value: '{{workflow.parameters.tag}}' - name: MIRROR_PREFIX - value: "{{workflow.parameters.dockerhub-registry-proxy}}" + value: '{{workflow.parameters.dockerhub-registry-proxy}}' command: [sh] - source: | + source: |- cd /src/${ENV_PATH}/${ENV_NAME} ls -lr if [ -e version.yaml ] @@ -53,8 +53,8 @@ spec: cp /etc/ssh-key/SSH_PRIVATE_KEY ~/.ssh/id_rsa ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub ls -l ~/.ssh - git config --global user.email "cg-devx-automation@cloudgeometry.io" - git config --global user.name "cgdevx-bot" + git config --global user.email "" + git config --global user.name "" git add . && \ git commit -m "Tag updated for $WL_SERVICE_LIST to $NEW_TAG" \ && git push From ee5a289324de3395e8ffae8f7d433bae6e5e9d16 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Thu, 4 Jan 2024 20:54:36 +0100 Subject: [PATCH 37/49] - promote wft param rename - added extra params to wl-gitops bootstrap --- .../workflow-templates/version-changer-wft.yaml | 2 +- tools/cli/commands/workload/bootstrap.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml index 80a31090..d9489984 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/argo-workflows/workflow-templates/version-changer-wft.yaml @@ -13,7 +13,7 @@ spec: - name: git-src path: /src git: - repo: '{{workflow.parameters.repo}}' + repo: '{{workflow.parameters.gitops-repo}}' # revision: "{{workflow.parameters.tag}}" sshPrivateKeySecret: name: ci-secrets diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 583cb242..527a6e6e 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -4,6 +4,7 @@ import click from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER +from common.const.const import TERRAFORM_VERSION from common.logging_config import configure_logging from common.state_store import StateStore from common.utils.command_utils import str_to_kebab, init_cloud_provider, check_installation_presence @@ -140,7 +141,13 @@ def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_temp "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], f"workloads/{wl_name}/hosting_provider"), "# ": cloud_man.create_hosting_provider_snippet(), + "": p.parameters[""], + "": p.parameters[""], + "": p.parameters[""], + "": p.parameters[""], + "": p.parameters[""], "": p.parameters[""], + "": TERRAFORM_VERSION } wl_ops_man.parametrise(wl_gitops_repo_params) From 6ff667652d669562345537bec33b79cf79b738b5 Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Thu, 4 Jan 2024 21:02:31 +0100 Subject: [PATCH 38/49] fix: reload-strategy annotations for reloader --- .../components/kube-system/reloader/application.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml index a2560232..d0c44e6a 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml @@ -13,7 +13,9 @@ spec: chart: reloader helm: values: |- - ignoreSecrets: false + reloader: + ignoreSecrets: false + reload-strategy: annotations destination: server: 'https://kubernetes.default.svc' namespace: reloader From 304fb2b0745b8764a12f13bf0f7bc8736309304e Mon Sep 17 00:00:00 2001 From: Serg Shalavin Date: Fri, 5 Jan 2024 12:33:57 +0100 Subject: [PATCH 39/49] fix: reloader strategy --- .../components/kube-system/reloader/application.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml index d0c44e6a..9f5b39ea 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/kube-system/reloader/application.yaml @@ -15,7 +15,7 @@ spec: values: |- reloader: ignoreSecrets: false - reload-strategy: annotations + reloadStrategy: annotations destination: server: 'https://kubernetes.default.svc' namespace: reloader @@ -30,4 +30,4 @@ spec: backoff: duration: 5s maxDuration: 5m0s - factor: 2 \ No newline at end of file + factor: 2 From 9ea5dcde4e8ea83a45817aad80a07991d3c96124 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Fri, 5 Jan 2024 12:38:16 +0100 Subject: [PATCH 40/49] fix: proxy registry url --- tools/cli/commands/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 31fe2dd2..9fffa4a0 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -771,10 +771,10 @@ def setup( "code_quality_admin_password": p.internals["CODE_QUALITY_PASSWORD"] }) core_services_out = tf_wrapper.output() - p.parameters[""] = f'{p.parameters[""]}{core_services_out["dockerhub_proxy_name"]}' - p.parameters[""] = f'{p.parameters[""]}{core_services_out["gcr_proxy_name"]}' - p.parameters[""] = f'{p.parameters[""]}{core_services_out["k8s_gcr_proxy_name"]}' - p.parameters[""] = f'{p.parameters[""]}{core_services_out["quay_proxy_name"]}' + p.parameters[""] = f'{p.parameters[""]}/{core_services_out["dockerhub_proxy_name"]}' + p.parameters[""] = f'{p.parameters[""]}/{core_services_out["gcr_proxy_name"]}' + p.parameters[""] = f'{p.parameters[""]}/{core_services_out["k8s_gcr_proxy_name"]}' + p.parameters[""] = f'{p.parameters[""]}/{core_services_out["quay_proxy_name"]}' # unset envs as no longer needed unset_envs(core_services_tf_env_vars) From 76c24774f54011b1b510a512ef7e31f90b819fdf Mon Sep 17 00:00:00 2001 From: Turetskii Mikhail Date: Fri, 5 Jan 2024 17:24:45 +0300 Subject: [PATCH 41/49] feat: Workload manager implementation (#35) * fix: do cloud provider cli check before provider initialization * feat: Workload manager implementation * feat: Workload manager implementation --------- Co-authored-by: Alexander Ulyanov --- tools/cli/commands/setup.py | 2 - tools/cli/commands/workload/bootstrap.py | 503 +++++++++++++----- tools/cli/commands/workload/create.py | 180 +++++-- tools/cli/commands/workload/delete.py | 292 ++++++---- tools/cli/common/const/const.py | 1 + tools/cli/common/custom_excpetions.py | 32 ++ tools/cli/common/utils/command_utils.py | 101 +++- tools/cli/services/cloud/aws/aws_manager.py | 4 +- .../cli/services/cloud/azure/azure_manager.py | 4 +- .../services/cloud/cloud_provider_manager.py | 3 +- .../services/wl_gitops_template_manager.py | 100 ---- tools/cli/services/wl_template_manager.py | 328 +++++++++--- 12 files changed, 1091 insertions(+), 459 deletions(-) create mode 100644 tools/cli/common/custom_excpetions.py delete mode 100644 tools/cli/services/wl_gitops_template_manager.py diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 9fffa4a0..4cb94546 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -890,8 +890,6 @@ def prepare_parameters(p): @trace() def cloud_provider_check(manager: CloudProviderManager, p: StateStore) -> None: - if not manager.detect_cli_presence(): - raise click.ClickException("Cloud CLI is missing") if not manager.evaluate_permissions(): raise click.ClickException("Insufficient IAM permission") diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 527a6e6e..62aa6dcc 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -1,163 +1,414 @@ -import os -import shutil +import time +from typing import Dict import click -from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER -from common.const.const import TERRAFORM_VERSION -from common.logging_config import configure_logging +from common.const.const import WL_REPOSITORY_URL, WL_GITOPS_REPOSITORY_URL, WL_GITOPS_REPOSITORY_BRANCH, \ + WL_REPOSITORY_BRANCH, TERRAFORM_VERSION +from common.custom_excpetions import WorkloadManagerError +from common.logging_config import configure_logging, logger from common.state_store import StateStore -from common.utils.command_utils import str_to_kebab, init_cloud_provider, check_installation_presence -from services.wl_gitops_template_manager import WorkloadGitOpsTemplateManager -from services.wl_template_manager import WorkloadTemplateManager +from common.utils.command_utils import str_to_kebab, init_cloud_provider +from services.wl_template_manager import WorkloadManager @click.command() @click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) -@click.option('--workload-repository-name', '-wlrn', 'wl_repo_name', help='Workload repository name', type=click.STRING) -@click.option('--workload-gitops-repository-name', '-wlgrn', 'wl_gitops_repo_name', - help='Workload GitOps repository name', type=click.STRING) -@click.option('--workload-template-url', '-wltu', 'wl_template_url', help='Workload repository template', - type=click.STRING) -@click.option('--workload-template-branch', '-wltb', 'wl_template_branch', help='Workload repository template branch', - type=click.STRING) -@click.option('--workload-gitops-template-url', '-wlgu', 'wl_gitops_template_url', - help='Workload GitOps repository template', type=click.STRING) -@click.option('--workload-gitops-template-branch', '-wlgb', 'wl_gitops_template_branch', - help='Workload GitOps repository template branch', - type=click.STRING) -@click.option('--workload-service-name', '-wls', 'wl_svc_name', help='Workload service name', type=click.STRING) -@click.option('--workload-service-port', '-wlsp', 'wl_svc_port', help='Workload service port', type=click.INT) -@click.option('--verbosity', type=click.Choice( - ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - case_sensitive=False -), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') -def bootstrap(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_template_url: str, wl_template_branch: str, - wl_gitops_template_url: str, wl_gitops_template_branch: str, wl_svc_name: str, wl_svc_port: int, - verbosity: str): - """Bootstrap workload repository with template.""" +@click.option( + '--workload-repository-name', + '-wlrn', + 'wl_repo_name', + help='Workload repository name', + type=click.STRING +) +@click.option( + '--workload-gitops-repository-name', + '-wlgrn', + 'wl_gitops_repo_name', + help='Workload GitOps repository name', + type=click.STRING +) +@click.option( + '--workload-template-url', + '-wltu', + 'wl_template_url', + help='Workload repository template', + type=click.STRING, + default=WL_REPOSITORY_URL +) +@click.option( + '--workload-template-branch', + '-wltb', + 'wl_template_branch', + help='Workload repository template branch', + type=click.STRING, + default=WL_REPOSITORY_BRANCH + +) +@click.option( + '--workload-gitops-template-url', + '-wlgu', + 'wl_gitops_template_url', + help='Workload GitOps repository template', + type=click.STRING, + default=WL_GITOPS_REPOSITORY_URL +) +@click.option( + '--workload-gitops-template-branch', + '-wlgb', + 'wl_gitops_template_branch', + help='Workload GitOps repository template branch', + type=click.STRING, + default=WL_GITOPS_REPOSITORY_BRANCH +) +@click.option( + '--workload-service-name', + '-wls', + 'wl_svc_name', + help='Workload service name', + type=click.STRING +) +@click.option( + '--workload-service-port', + '-wlsp', + 'wl_svc_port', + help='Workload service port', + type=click.INT, + default=3000 +) +@click.option( + '--verbosity', + type=click.Choice( + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + case_sensitive=False + ), + default='CRITICAL', + help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)' +) +def bootstrap( + wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, wl_template_url: str, wl_template_branch: str, + wl_gitops_template_url: str, wl_gitops_template_branch: str, wl_svc_name: str, wl_svc_port: int, verbosity: str +): + """ + Bootstrap a new workload environment by setting up the necessary repositories and configurations. + + This command initializes and configures a workload repository and a GitOps repository based on provided templates. + It handles the entire setup process, including the cloning of template repositories, parameter customization, + and finalization of the workload setup. + + Args: + wl_name (str): The name of the workload. + wl_repo_name (str): The name of the workload repository. + wl_gitops_repo_name (str): The name of the GitOps repository for the workload. + wl_template_url (str): The URL of the template for the workload repository. + wl_template_branch (str): The branch of the workload repository template. + wl_gitops_template_url (str): The URL of the template for the GitOps repository. + wl_gitops_template_branch (str): The branch of the GitOps repository template. + wl_svc_name (str): The name of the service within the workload. + wl_svc_port (int): The service port number for the workload service. + verbosity (str): The logging level for the process (DEBUG, INFO, WARNING, ERROR, CRITICAL). + + Raises: + ClickException: An exception is raised if there are issues in the configuration loading, + or if the bootstrap process for either the workload or GitOps repository fails. + + Returns: + None: The function does not return anything but completes the bootstrap process. + """ click.echo("Bootstrapping workload...") - # Set up global logger + func_start_time = time.time() + state_store = StateStore() configure_logging(verbosity) - check_installation_presence() - - p: StateStore = StateStore() + try: + org_name = state_store.parameters[""] + key_path = state_store.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] + cc_cluster_fqdn = state_store.parameters[""] + registry_url = state_store.parameters[""] + tf_backend_storage_name = state_store.internals["TF_BACKEND_STORAGE_NAME"] + dockerhub_proxy = state_store.parameters[""] + gcr_proxy = state_store.parameters[""] + k8s_gcr_proxy = state_store.parameters[""] + quay_proxy = state_store.parameters[""] + git_runner_group_name = state_store.parameters[""] + git_organisation_name = state_store.parameters[""] + click.echo("1/10: Configuration loaded.") + except KeyError as e: + error_message = f'Configuration loading failed due to missing key: {e}. ' \ + 'Please verify the state store configuration and ensure all required keys are present. ' \ + 'Review the documentation for the necessary configuration keys and try again.' + logger.error(error_message) + raise click.ClickException(error_message) + + # Initialize cloud manager for GitOps parameters + cloud_man, _ = init_cloud_provider(state_store) + click.echo("2/10: Cloud manager initialized for GitOps.") + + # Initialize WorkloadManager for the workload repository + wl_name, wl_repo_name, wl_gitops_repo_name = process_workload_names( + wl_name=wl_name, + wl_repo_name=wl_repo_name, + wl_gitops_repo_name=wl_gitops_repo_name + ) + click.echo("3/10: Workload names processed.") + + # Prepare parameters for workload and GitOps repositories + wl_repo_params = _prepare_workload_params( + wl_name=wl_name, + wl_svc_name=wl_repo_name, + registry_registry_url=registry_url, + registry_dockerhub_proxy=dockerhub_proxy, + registry_gcr_proxy=gcr_proxy, + registry_k8s_gcr_proxy=k8s_gcr_proxy, + registry_quay_proxy=quay_proxy, + wl_repo_name=wl_repo_name, + wl_gitops_repo_name=wl_gitops_repo_name, + git_organization_name=git_organisation_name, + git_runner_group_name=git_runner_group_name + ) + wl_gitops_params = _prepare_gitops_params( + wl_name=wl_name, + wl_svc_name=wl_svc_name, + wl_svc_port=wl_svc_port, + cc_cluster_fqdn=cc_cluster_fqdn, + registry_registry_url=registry_url, + k8s_role_mapping=cloud_man.create_k8s_cluster_role_mapping_snippet(), + additional_labels=cloud_man.create_additional_labels(), + tf_secrets_backend=cloud_man.create_iac_backend_snippet( + location=tf_backend_storage_name, + service=f"workloads/{wl_name}/secrets" + ), + tf_hosting_backend=cloud_man.create_iac_backend_snippet( + location=tf_backend_storage_name, + service=f"workloads/{wl_name}/hosting_provider" + ), + hosting_provider_snippet=cloud_man.create_hosting_provider_snippet(), + dockerhub_proxy=dockerhub_proxy, + registry_gcr_proxy=gcr_proxy, + registry_k8s_gcr_proxy=k8s_gcr_proxy, + registry_quay_proxy=quay_proxy, + git_runner_group_name=git_runner_group_name, + terraform_version=TERRAFORM_VERSION + ) + click.echo("4/10: Parameters for workload and GitOps repositories prepared.") - wl_name = str_to_kebab(wl_name) + # Initialize WorkloadManager for the workload repository + wl_manager = WorkloadManager( + org_name=org_name, + repo_name=wl_repo_name, + key_path=key_path, + template_url=wl_template_url, + template_branch=wl_template_branch + ) + click.echo("5/10: Workload repository manager initialized.") + + try: + # Perform bootstrap steps for the workload repository + perform_bootstrap( + workload_manager=wl_manager, + params=wl_repo_params + ) + click.echo("6/10: Workload repository bootstrap process completed.") + except WorkloadManagerError as e: + raise click.ClickException(str(e)) + + # Cleanup temporary folder for workload repository + wl_manager.cleanup() + click.echo("7/10: Workload repository cleanup completed.") + + # Initialize WorkloadManager for the GitOps repository + wl_gitops_manager = WorkloadManager( + org_name=org_name, + repo_name=wl_gitops_repo_name, + key_path=key_path, + template_url=wl_gitops_template_url, + template_branch=wl_gitops_template_branch + ) + click.echo("8/10: GitOps repository manager initialized.") - if not wl_repo_name: - wl_repo_name = wl_name + try: + # Perform bootstrap steps for the GitOps repositoryR + perform_bootstrap( + workload_manager=wl_gitops_manager, + params=wl_gitops_params + ) + click.echo("9/10: GitOps repository bootstrap process completed.") + except WorkloadManagerError as e: + raise click.ClickException(str(e)) - if not wl_gitops_repo_name: - if wl_repo_name: - wl_gitops_repo_name = f"{wl_repo_name}-gitops" - else: - wl_gitops_repo_name = f"{wl_name}-gitops" + # Cleanup temporary folder for GitOps repository + wl_gitops_manager.cleanup() + click.echo("10/10: GitOps repository cleanup completed.") - if not wl_svc_name: - wl_svc_name = "wl-service" + click.echo(f"Bootstrapping workload completed in {time.time() - func_start_time:.2f} seconds.") - if not wl_svc_port: - wl_svc_port = 3000 - wl_svc_name = str_to_kebab(wl_svc_name) - wl_repo_name = str_to_kebab(wl_repo_name) - wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) +def process_workload_names(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str): + """ + Process and normalize workload names to a standard format. - if os.path.exists(LOCAL_WORKLOAD_TEMP_FOLDER): - shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) + Args: + wl_name (str): Name of the workload. + wl_repo_name (str): Name of the workload repository. + wl_gitops_repo_name (str): Name of the workload GitOps repository. - os.makedirs(LOCAL_WORKLOAD_TEMP_FOLDER) + Returns: + tuple[str, str, str]: Tuple of processed workload name, workload repository name, and GitOps repository name. + """ + processed_wl_name = str_to_kebab(wl_name) + processed_wl_repo_name = str_to_kebab(wl_repo_name or wl_name) + processed_wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name or f"{wl_repo_name}-gitops") - wl_man = WorkloadTemplateManager( - org_name=p.parameters[""], - repo_name=wl_repo_name, - key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"], - template_url=wl_template_url, - template_branch=wl_template_branch - ) + return processed_wl_name, processed_wl_repo_name, processed_wl_gitops_repo_name - # workload repo - if not (wl_man.clone_wl()): - raise click.ClickException("Failed cloning repo") - if not (wl_man.clone_template()): - raise click.ClickException("Failed cloning template repo") +def perform_bootstrap( + workload_manager: WorkloadManager, + params: Dict[str, str] +): + """ + Perform the bootstrap process using the WorkloadManager. - # make wl_svc_name multiple value param - wl_man.bootstrap([wl_svc_name]) + Args: + workload_manager (WorkloadManager): The workload manager instance. + params (Dict[str, str]): Dictionary containing parameters for bootstrap. - wl_repo_params = { - "": wl_name, - "": wl_svc_name, - "": p.parameters[""], - "": p.parameters[""], - "": p.parameters[""], - "": p.parameters[""], - "": p.parameters[""], - "": wl_repo_name, - "": wl_gitops_repo_name, - "": p.parameters[""], - "": p.parameters[""], - } + Raises: + WorkloadManagerError: If cloning, templating, or uploading process fails. + """ + if not workload_manager.clone_template(): + raise WorkloadManagerError("Failed to clone template repository") - wl_man.parametrise(wl_repo_params) + if not workload_manager.clone_wl(): + raise WorkloadManagerError("Failed to clone workload repository") - wl_man.upload(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) + # Bootstrap workload with the given service names + service_names = [params.get("", "default-service")] + workload_manager.bootstrap(service_names) - wl_man.cleanup() + workload_manager.parametrise(params) - # wl gitops repo - cloud_man, dns_man = init_cloud_provider(p) - wl_ops_man = WorkloadGitOpsTemplateManager( - org_name=p.parameters[""], - repo_name=wl_gitops_repo_name, - key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"], - template_url=wl_gitops_template_url, - template_branch=wl_gitops_template_branch + workload_manager.upload( + author_name=params.get(""), + author_email=params.get("") ) - # workload repo - if not (wl_ops_man.clone_wl()): - raise click.ClickException("Failed cloning repo") - if not (wl_ops_man.clone_template()): - raise click.ClickException("Failed cloning template repo") - - wl_ops_man.bootstrap() +def _prepare_workload_params( + wl_name: str, + wl_svc_name: str, + registry_registry_url: str, + registry_dockerhub_proxy: str, + registry_gcr_proxy: str, + registry_k8s_gcr_proxy: str, + registry_quay_proxy: str, + wl_repo_name: str, + wl_gitops_repo_name: str, + git_organization_name: str, + git_runner_group_name: str +) -> Dict[str, str]: + """ + Prepare parameters for the workload repository bootstrap process. + + Args: + wl_name (str): Workload name. + wl_svc_name (str): Workload service name. + registry_registry_url (str): URL for the registry. + registry_dockerhub_proxy (str): Proxy URL for DockerHub. + registry_gcr_proxy (str): Proxy URL for Google Container Registry. + registry_k8s_gcr_proxy (str): Proxy URL for Kubernetes GCR. + registry_quay_proxy (str): Proxy URL for Quay.io. + wl_repo_name (str): Name of the workload repository. + wl_gitops_repo_name (str): Name of the GitOps repository for the workload. + git_organization_name (str): Name of the Git organization. + git_runner_group_name (str): Name of the Git runner group. + + Returns: + Dict[str, str]: A dictionary containing prepared parameters for the workload repository bootstrap process. + """ + logger.debug(f"Preparing workload parameters for '{wl_name}' with service '{wl_svc_name}'.") + + params = { + "": wl_name, + "": wl_svc_name, + "": registry_registry_url, + "": registry_dockerhub_proxy, + "": registry_gcr_proxy, + "": registry_k8s_gcr_proxy, + "": registry_quay_proxy, + "": wl_repo_name, + "": wl_gitops_repo_name, + "": git_organization_name, + "": git_runner_group_name + } - wl_gitops_repo_params = { + logger.debug(f"Workload parameters prepared: {params}") + return params + + +def _prepare_gitops_params( + wl_name: str, + wl_svc_name: str, + wl_svc_port: str | int, + cc_cluster_fqdn: str, + registry_registry_url: str, + k8s_role_mapping: str, + additional_labels: str, + tf_secrets_backend: str, + tf_hosting_backend: str, + hosting_provider_snippet: str, + dockerhub_proxy: str, + registry_gcr_proxy: str, + registry_k8s_gcr_proxy: str, + registry_quay_proxy: str, + git_runner_group_name: str, + terraform_version: str +) -> Dict[str, str]: + """ + Prepare parameters for the GitOps repository bootstrap process. + + Args: + wl_name (str): Name of the workload. + wl_svc_name (str): Name of the workload service. + wl_svc_port (str | int): Port number for the workload service. + cc_cluster_fqdn (str): Fully qualified domain name for the cluster. + registry_registry_url (str): URL for the registry. + k8s_role_mapping (str): Kubernetes role mapping snippet. + additional_labels (str): Additional labels snippet. + tf_secrets_backend (str): Terraform secrets backend snippet. + tf_hosting_backend (str): Terraform hosting backend snippet. + hosting_provider_snippet (str): Hosting provider snippet. + dockerhub_proxy (str): Proxy URL for DockerHub. + registry_gcr_proxy (str): Proxy URL for Google Container Registry. + registry_k8s_gcr_proxy (str): Proxy URL for Kubernetes GCR. + registry_quay_proxy (str): Proxy URL for Quay.io. + git_runner_group_name (str): Name of the Git runner group. + terraform_version (str): Version of Terraform to use. + + Returns: + Dict[str, str]: A dictionary containing prepared parameters for the GitOps repository bootstrap process. + """ + logger.debug(f"Preparing GitOps parameters for workload '{wl_name}' with service '{wl_svc_name}'.") + + params = { "": wl_name, "": wl_svc_name, - "": f'{wl_svc_name}.{wl_name}.{p.parameters[""]}', - "": f'{p.parameters[""]}/{wl_name}/{wl_svc_name}', + "": f'{wl_svc_name}.{wl_name}.{cc_cluster_fqdn}', + "": f'{registry_registry_url}/{wl_name}/{wl_svc_name}', "": str(wl_svc_port), - "# ": cloud_man.create_k8s_cluster_role_mapping_snippet(), - "": "[Put your workload service role mapping]", - "# ": cloud_man.create_additional_labels(), - "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], - f"workloads/{wl_name}/secrets"), - "# ": cloud_man.create_iac_backend_snippet(p.internals["TF_BACKEND_STORAGE_NAME"], - f"workloads/{wl_name}/hosting_provider"), - "# ": cloud_man.create_hosting_provider_snippet(), - "": p.parameters[""], - "": p.parameters[""], - "": p.parameters[""], - "": p.parameters[""], - "": p.parameters[""], - "": p.parameters[""], - "": TERRAFORM_VERSION + "": k8s_role_mapping, + "": "[Placeholder for workload service role mapping]", + "": additional_labels, + "": tf_secrets_backend, + "": tf_hosting_backend, + "": hosting_provider_snippet, + "": dockerhub_proxy, + "": registry_gcr_proxy, + "": registry_k8s_gcr_proxy, + "": registry_quay_proxy, + "": git_runner_group_name, + "": terraform_version } - wl_ops_man.parametrise(wl_gitops_repo_params) - - wl_ops_man.upload(name=p.internals["GIT_USER_NAME"], email=p.internals["GIT_USER_EMAIL"]) - - wl_ops_man.cleanup() - - # remove temp folder - shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) - - click.echo("Bootstrapping workload. Done!") - return True + logger.debug(f"GitOps parameters prepared: {params}") + return params diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 4e326539..a79ff4ad 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -1,79 +1,141 @@ -import webbrowser +import time import click -from common.utils.command_utils import str_to_kebab, init_git_provider, check_installation_presence -from common.logging_config import configure_logging +from common.const.const import WL_REPOSITORY_BRANCH, WL_PR_BRANCH_NAME_PREFIX +from common.custom_excpetions import GitBranchAlreadyExists, PullRequestCreationError +from common.logging_config import configure_logging, logger from common.state_store import StateStore +from common.utils.command_utils import str_to_kebab, check_installation_presence, \ + initialize_gitops_repository, create_and_setup_branch, create_and_open_pull_request from services.platform_gitops import PlatformGitOpsRepo @click.command() @click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) -@click.option('--workload-repository-name', '-wlrn', 'wl_repo_name', help='Workload repository name', - type=click.STRING) -@click.option('--workload-gitops-repository-name', '-wlgrn', 'wl_gitops_repo_name', - help='Workload GitOps repository name', type=click.STRING) -@click.option('--verbosity', type=click.Choice( - ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - case_sensitive=False -), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') -def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: str): - """Create workload boilerplate.""" - click.echo("Creating workload GitOps code...") - - # Set up global logger - configure_logging(verbosity) +@click.option( + '--workload-repository-name', + '-wlrn', + 'wl_repo_name', + help='Workload repository name', + type=click.STRING +) +@click.option( + '--workload-gitops-repository-name', + '-wlgrn', + 'wl_gitops_repo_name', + help='Workload GitOps repository name', type=click.STRING +) +@click.option( + '--workload-gitops-main-branch-name', + '-wlgmbn', + 'main_branch', + default=WL_REPOSITORY_BRANCH, + help='Workload GitOps repository main branch name', type=click.STRING +) +@click.option( + '--verbosity', + type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False), + default='CRITICAL', + help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)' +) +def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branch: str, verbosity: str) -> None: + """ + Create workload boilerplate for GitOps. + + Parameters: + wl_name (str): Name of the workload. + wl_repo_name (str): Name of the workload repository. + wl_gitops_repo_name (str): Name of the workload GitOps repository. + main_branch (str): Main branch name of the GitOps repository. + verbosity (str): Logging level. + """ + func_start_time = time.time() + click.echo("Initializing workload GitOps code creation...") + branch_name = f"{WL_PR_BRANCH_NAME_PREFIX}{wl_name}-init" + configure_logging(verbosity) check_installation_presence() - wl_name = str_to_kebab(wl_name) + state_store = StateStore() + click.echo("1/7: State store initialized.") - if not wl_repo_name: - wl_repo_name = wl_name + wl_name, wl_repo_name, wl_gitops_repo_name = _process_workload_names( + wl_name=wl_name, + wl_repo_name=wl_repo_name, + wl_gitops_repo_name=wl_gitops_repo_name + ) + click.echo("2/7: Workload names processed.") - if not wl_gitops_repo_name: - if wl_repo_name: - wl_gitops_repo_name = f"{wl_repo_name}-gitops" - else: - wl_gitops_repo_name = f"{wl_name}-gitops" + git_man, gor = initialize_gitops_repository(state_store=state_store, logger=logger) + click.echo("3/7: GitOps repository initialized.") - wl_repo_name = str_to_kebab(wl_repo_name) - wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) - - p: StateStore = StateStore() + try: + create_and_setup_branch(gor=gor, branch_name=branch_name, logger=logger) + except GitBranchAlreadyExists as e: + raise click.ClickException(str(e)) - main_branch = "main" - git_man = init_git_provider(p) - gor = PlatformGitOpsRepo(git_man, - author_name=p.internals["GIT_USER_NAME"], - author_email=p.internals["GIT_USER_EMAIL"], - key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) - # update repo just in case - gor.update() + click.echo(f"4/7: Branch '{branch_name}' created and set up.") - # create new branch - branch_name = f"feature/{wl_name}-init" - if gor.branch_exist(branch_name): - raise click.ClickException("Branch already exist, please use different workload name. ") - gor.create_branch(branch_name) + add_workload_and_commit( + gor=gor, + wl_name=wl_name, + wl_repo_name=wl_repo_name, + wl_gitops_repo_name=wl_gitops_repo_name + ) + click.echo("5/7: Workload added and changes committed.") + try: + create_and_open_pull_request( + gor=gor, + state_store=state_store, + wl_name=wl_name, + branch_name=branch_name, + main_branch=main_branch, + logger=logger + ) + except PullRequestCreationError as e: + raise click.ClickException(str(e)) + + click.echo(f"6/7: Pull request for branch '{branch_name}' created and opened in the web browser.") + + gor.switch_to_branch(branch_name=main_branch) + click.echo(f"7/7: Switched to branch '{main_branch}'.") + click.echo(f"Workload GitOps code creation completed in {time.time() - func_start_time:.2f} seconds.") + + +def _process_workload_names(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str) -> tuple[str, str, str]: + """ + Process and normalize workload names to a standard format. + + Parameters: + wl_name (str): Name of the workload. + wl_repo_name (str): Name of the workload repository. + wl_gitops_repo_name (str): Name of the workload GitOps repository. + + Returns: + tuple[str, str, str]: Tuple of processed workload name, workload repository name, and GitOps repository name. + """ + logger.debug(f"Processing workload names: {wl_name}, {wl_repo_name}, {wl_gitops_repo_name}") + processed_wl_name = str_to_kebab(wl_name) + processed_wl_repo_name = str_to_kebab(wl_repo_name or wl_name) + processed_wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name or f"{wl_repo_name}-gitops") + logger.info(f"Processed names: {processed_wl_name}, {processed_wl_repo_name}, {processed_wl_gitops_repo_name}") + return processed_wl_name, processed_wl_repo_name, processed_wl_gitops_repo_name + + +def add_workload_and_commit( + gor: PlatformGitOpsRepo, wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str +) -> None: + """ + Add the workload to the GitOps repository and commit changes. + + Parameters: + gor: PlatformGitOpsRepo class instance. + wl_name (str): Name of the workload. + wl_repo_name (str): Name of the workload repository. + wl_gitops_repo_name (str): Name of the workload GitOps repository. + """ gor.add_workload(wl_name, wl_repo_name, wl_gitops_repo_name) - - # commit and prepare a PR gor.upload_changes() - - try: - pr_url = gor.create_pr(p.parameters[""], - branch_name, - main_branch, - f"introduce {wl_name}", - f"Add default secrets, groups and default repository structure.") - webbrowser.open(pr_url, autoraise=False) - except Exception as e: - raise click.ClickException("Could not create PR") - - gor.switch_to_branch() - - click.echo("Creating workload GitOps code. Done!") - return True + logger.info("Workload added and committed to the repository.") diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index 07d50942..3b0bd00a 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -1,131 +1,235 @@ import os import shutil -import webbrowser +import time import click -from ghrepo import GHRepo -from git import Repo, GitError -from common.utils.command_utils import str_to_kebab, prepare_cloud_provider_auth_env_vars, set_envs, \ - check_installation_presence, init_git_provider -from common.const.common_path import LOCAL_FOLDER, LOCAL_WORKLOAD_TEMP_FOLDER +from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER +from common.const.const import WL_REPOSITORY_BRANCH, WL_PR_BRANCH_NAME_PREFIX, WL_GITOPS_REPOSITORY_BRANCH, \ + WL_GITOPS_REPOSITORY_URL from common.const.parameter_names import GIT_ACCESS_TOKEN, GIT_ORGANIZATION_NAME -from common.logging_config import configure_logging +from common.custom_excpetions import GitBranchAlreadyExists, PullRequestCreationError +from common.logging_config import configure_logging, logger from common.state_store import StateStore +from common.utils.command_utils import str_to_kebab, prepare_cloud_provider_auth_env_vars, set_envs, \ + check_installation_presence, initialize_gitops_repository, create_and_setup_branch, \ + create_and_open_pull_request from services.platform_gitops import PlatformGitOpsRepo from services.tf_wrapper import TfWrapper -from services.wl_gitops_template_manager import WorkloadGitOpsTemplateManager +from services.wl_template_manager import WorkloadManager @click.command() @click.option('--workload-name', '-wl', 'wl_name', help='Workload name', type=click.STRING, prompt=True) -@click.option('--workload-gitops-repository-name', '-wlgrn', 'wl_gitops_repo_name', - help='Workload GitOps repository name', type=click.STRING) -@click.option('--destroy-resources', '-wldr', 'destroy_resources', help='Destroy workload resources', is_flag=True, - default=False) -@click.option('--verbosity', type=click.Choice( - ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - case_sensitive=False -), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') -def delete(wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, verbosity: str): - """Deletes all the workload boilerplate.""" +@click.option( + '--workload-gitops-repository-name', + '-wlgrn', + 'wl_gitops_repo_name', + help='Workload GitOps repository name', + type=click.STRING +) +@click.option( + '--destroy-resources', + '-wldr', + 'destroy_resources', + help='Destroy workload resources', + is_flag=True, + default=False +) +@click.option( + '--workload-gitops-main-branch-name', + '-wlgmbn', + 'main_branch', + default=WL_REPOSITORY_BRANCH, + help='Workload GitOps repository main branch name', type=click.STRING +) +@click.option( + '--workload-gitops-template-url', + '-wlgu', + 'wl_gitops_template_url', + help='Workload GitOps repository template', + type=click.STRING, + default=WL_GITOPS_REPOSITORY_URL +) +@click.option( + '--workload-gitops-template-branch', + '-wlgb', + 'wl_gitops_template_branch', + help='Workload GitOps repository template branch', + type=click.STRING, + default=WL_GITOPS_REPOSITORY_BRANCH +) +@click.option( + '--verbosity', + type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False), + default='CRITICAL', + help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)' +) +def delete( + wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, wl_gitops_template_url: str, + wl_gitops_template_branch: str, main_branch: str, verbosity: str +): + """ + Deletes all the workload boilerplate, including optionally destroying associated resources. + + This command performs a series of steps to remove a workload and its configurations from the GitOps repository. + It optionally destroys the resources defined in the workload's Infrastructure as Code (IaC). + + Args: + wl_name (str): Name of the workload to be deleted. + wl_gitops_repo_name (str): Name of the GitOps repository associated with the workload. + destroy_resources (bool): Flag indicating whether to destroy resources defined in the workload's IaC. + wl_gitops_template_url (str): URL of the GitOps repository template. + wl_gitops_template_branch (str): Branch of the GitOps repository template. + main_branch (str): Main branch name of the GitOps repository. + verbosity (str): Logging level. + + Raises: + click.ClickException: If any error occurs during the deletion process. + """ click.confirm(f'This will delete the workload "{wl_name}". Please confirm to continue', abort=True) - - click.echo("Deleting workload GitOps code...") + destroy_resources = destroy_resources and click.confirm( + "This will delete all the resources defined in workload IaC. Please confirm to continue" + ) + # Determine the total number of steps + logging_total_steps = 7 if destroy_resources else 6 + func_start_time = time.time() + + branch_name = f"{WL_PR_BRANCH_NAME_PREFIX}{wl_name}-destroy" + click.echo("Initializing workload deletion process...") + state_store: StateStore = StateStore() + click.echo(f"1/{logging_total_steps}: State store initialized.") # Set up global logger - configure_logging(verbosity) - + configure_logging(verbosity=verbosity) check_installation_presence() + click.echo(f"2/{logging_total_steps}: Logging and installation checked.") - wl_name = str_to_kebab(wl_name) - - if not wl_gitops_repo_name: - wl_gitops_repo_name = f"{wl_name}-gitops" - - wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name) - - p: StateStore = StateStore() - main_branch = "main" + wl_name, wl_gitops_repo_name = _process_workload_names(wl_name=wl_name, wl_gitops_repo_name=wl_gitops_repo_name) + click.echo(f"3/{logging_total_steps}: Workload names processed.") - git_man = init_git_provider(p) - gor = PlatformGitOpsRepo(git_man, - author_name=p.internals["GIT_USER_NAME"], - author_email=p.internals["GIT_USER_EMAIL"], - key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"], ) - # update repo just in case - gor.update() + git_man, gor = initialize_gitops_repository(state_store=state_store, logger=logger) + click.echo(f"4/{logging_total_steps}: GitOps repository initialized.") - # create new branch - branch_name = f"feature/{wl_name}-destroy" - if gor.branch_exist(branch_name): - raise click.ClickException("Branch already exist, please use different workload name. ") - gor.create_branch(branch_name) - - gor.rm_workload(wl_name) + try: + create_and_setup_branch(gor=gor, branch_name=branch_name, logger=logger) + except GitBranchAlreadyExists as e: + raise click.ClickException(str(e)) - # destroy resources + # Optionally destroy resources if destroy_resources: # to destroy workload resources, we need to clone workload gitops repo # and call tf destroy while pointing to remote state - tf_destroy_confirmation: bool = click.confirm( - "This will delete all the resources defined in workload IaC. Please confirm to continue") + wl_gitops_manager = WorkloadManager( + org_name=state_store.parameters[""], + repo_name=wl_gitops_repo_name, + key_path=state_store.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"], + template_url=wl_gitops_template_url, + template_branch=wl_gitops_template_branch + ) - if tf_destroy_confirmation: - wl_ops_man = WorkloadGitOpsTemplateManager( - org_name=p.parameters[""], - repo_name=wl_gitops_repo_name, - key_path=p.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"]) + wl_gitops_repo_folder = wl_gitops_manager.clone_wl() - wl_gitops_repo_folder = wl_ops_man.clone_wl() + _set_tf_env_vars_for_tf(state_store=state_store) - cloud_provider_auth_env_vars = prepare_cloud_provider_auth_env_vars(p) + destroy_all_tf_resources(repo_folder=wl_gitops_repo_folder) - tf_env_vars = { - **cloud_provider_auth_env_vars, - **{ - "GITHUB_TOKEN": p.get_input_param(GIT_ACCESS_TOKEN), - "GITHUB_OWNER": p.get_input_param(GIT_ORGANIZATION_NAME), - "VAULT_TOKEN": p.internals.get("VAULT_ROOT_TOKEN", None), - "VAULT_ADDR": f'https://{p.parameters.get("", None)}', - } - } - # set envs as required by tf - set_envs(tf_env_vars) + # remove temp folder + shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) - if os.path.exists(wl_gitops_repo_folder / "terraform"): - click.echo("Destroying WL secrets...") + click.echo(f"5/{logging_total_steps}: Workload resources destroyed.") - tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/secrets") - tf_wrapper.init() - tf_wrapper.destroy() - - click.echo("Destroying WL secrets. Done!") - - click.echo("Destroying WL cloud resources...") + _remove_workload_and_commit(wl_name=wl_name, gor=gor) + click.echo(f"{6 if destroy_resources else 5}/{logging_total_steps}: Workload removed and changes committed.") + try: + create_and_open_pull_request( + gor=gor, + state_store=state_store, + wl_name=wl_name, + branch_name=branch_name, + main_branch=main_branch, + logger=logger + ) + except PullRequestCreationError as e: + raise click.ClickException(str(e)) + + click.echo(f"{7 if destroy_resources else 6}/{logging_total_steps}: Pull request created and opened.") - tf_wrapper = TfWrapper(wl_gitops_repo_folder / "terraform/infrastructure") - tf_wrapper.init() - tf_wrapper.destroy() + gor.switch_to_branch() - click.echo("Destroying WL cloud resources. Done!") + click.echo(f"Deleting workload GitOps code completed in {time.time() - func_start_time:.2f} seconds.") - # remove temp folder - shutil.rmtree(LOCAL_WORKLOAD_TEMP_FOLDER) - # commit and prepare a PR - gor.upload_changes() +def _process_workload_names(wl_name: str, wl_gitops_repo_name: str) -> tuple[str, str]: + """Normalize and process workload and GitOps repository names.""" + logger.debug(f"Processing workload names: {wl_name}, {wl_gitops_repo_name}") + wl_name = str_to_kebab(wl_name) + wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name or f"{wl_name}-gitops") + logger.info(f"Processed names: {wl_name}, {wl_gitops_repo_name}") + return wl_name, wl_gitops_repo_name - try: - pr_url = gor.create_pr(p.parameters[""], - branch_name, main_branch, - f"remove {wl_name}", - f"Remove default secrets, groups and repository structure.") - webbrowser.open(pr_url, autoraise=False) - except Exception as e: - raise click.ClickException("Could not create PR") - gor.switch_to_branch() +def _remove_workload_and_commit(gor: PlatformGitOpsRepo, wl_name: str) -> None: + """ + Remove the workload to the GitOps repository and commit changes. - click.echo("Deleting workload GitOps code. Done!") - return True + Parameters: + gor: PlatformGitOpsRepo class instance. + wl_name (str): Name of the workload. + """ + gor.rm_workload(wl_name=wl_name) + gor.upload_changes() + logger.info("Workload added and committed to the repository.") + + +def _set_tf_env_vars_for_tf(state_store: StateStore) -> None: + """ + Set environment variables required for Terraform operations. + + Args: + state_store (StateStore): State store instance for accessing configuration. + """ + cloud_provider_auth_env_vars = prepare_cloud_provider_auth_env_vars(state_store) + tf_env_vars = { + **cloud_provider_auth_env_vars, + **{ + "GITHUB_TOKEN": state_store.get_input_param(GIT_ACCESS_TOKEN), + "GITHUB_OWNER": state_store.get_input_param(GIT_ORGANIZATION_NAME), + "VAULT_TOKEN": state_store.internals.get("VAULT_ROOT_TOKEN", None), + "VAULT_ADDR": f'https://{state_store.parameters.get("", None)}', + } + } + set_envs(tf_env_vars) + + +def _destroy_tf_resources(tf_directory: str, resource_description: str) -> None: + """ + Destroy Terraform resources in the specified directory. + + Args: + tf_directory (str): Path to the Terraform directory containing configurations to be destroyed. + resource_description (str): Description of the resources being destroyed (e.g., 'WL secrets', 'WL cloud resources'). + """ + if os.path.exists(tf_directory): + logger.debug(f"Destroying {resource_description}...") + tf_wrapper = TfWrapper(tf_directory) + tf_wrapper.init() + tf_wrapper.destroy() + logger.info(f"Destroying {resource_description}. Done!") + + +def destroy_all_tf_resources(repo_folder: str) -> None: + """ + Destroy all Terraform resources related to the workload in the specified repository folder. + + Args: + repo_folder (str): Path to the repository folder containing Terraform configurations. + """ + _destroy_tf_resources( + tf_directory=os.path.join(repo_folder, "terraform", "secrets"), + resource_description="WL secrets" + ) + _destroy_tf_resources( + tf_directory=os.path.join(repo_folder, "terraform", "infrastructure"), + resource_description="WL cloud resources" + ) diff --git a/tools/cli/common/const/const.py b/tools/cli/common/const/const.py index dba39262..c0e1740d 100644 --- a/tools/cli/common/const/const.py +++ b/tools/cli/common/const/const.py @@ -14,5 +14,6 @@ TERRAFORM_VERSION = "1.6.4" WL_REPOSITORY_URL = "git@github.com:CloudGeometry/cg-devx-wl-template.git" WL_REPOSITORY_BRANCH = "main" +WL_PR_BRANCH_NAME_PREFIX = "feature/" WL_GITOPS_REPOSITORY_URL = "git@github.com:CloudGeometry/cg-devx-wl-gitops-template.git" WL_GITOPS_REPOSITORY_BRANCH = "main" diff --git a/tools/cli/common/custom_excpetions.py b/tools/cli/common/custom_excpetions.py new file mode 100644 index 00000000..c3e6e7e0 --- /dev/null +++ b/tools/cli/common/custom_excpetions.py @@ -0,0 +1,32 @@ +class GitBranchAlreadyExists(Exception): + """Exception raised when a branch already exists in the GitOps repository.""" + + def __init__(self, branch_name: str, message: str = "Branch already exists"): + self.branch_name = branch_name + self.message = f"{message} '{branch_name}'" + super().__init__(self.message) + + def __str__(self): + return f"{self.message} in the repository." + + +class PullRequestCreationError(Exception): + """Exception raised when creating a pull request fails.""" + + def __init__(self, message: str): + self.message = message + super().__init__(self.message) + + +class RepositoryNotInitializedError(Exception): + """Exception raised when trying to access an uninitialized repository.""" + + def __init__(self, message: str): + super().__init__(message) + + +class WorkloadManagerError(Exception): + """Custom exception for errors in the WorkloadManager.""" + + def __init__(self, message: str): + super().__init__(message) diff --git a/tools/cli/common/utils/command_utils.py b/tools/cli/common/utils/command_utils.py index 1f6c00b4..160f3e5d 100644 --- a/tools/cli/common/utils/command_utils.py +++ b/tools/cli/common/utils/command_utils.py @@ -1,6 +1,8 @@ import os import os import time +import webbrowser +from logging import Logger from re import sub import click @@ -11,6 +13,7 @@ from common.const.parameter_names import CLOUD_REGION, CLOUD_PROFILE, CLOUD_ACCOUNT_ACCESS_KEY, \ CLOUD_ACCOUNT_ACCESS_SECRET, DNS_REGISTRAR_ACCESS_KEY, DNS_REGISTRAR_ACCESS_SECRET, GIT_ACCESS_TOKEN, \ GIT_ORGANIZATION_NAME +from common.custom_excpetions import GitBranchAlreadyExists, PullRequestCreationError from common.enums.cloud_providers import CloudProviders from common.enums.dns_registrars import DnsRegistrars from common.enums.git_providers import GitProviders @@ -22,6 +25,7 @@ from services.dns.azure_dns.azure_dns import AzureDNSManager from services.dns.dns_provider_manager import DNSManager from services.dns.route53.route53 import Route53Manager +from services.platform_gitops import PlatformGitOpsRepo from services.vcs.git_provider_manager import GitProviderManager from services.vcs.github.github_manager import GitHubProviderManager from services.vcs.gitlab.gitlab_manager import GitLabProviderManager @@ -30,9 +34,13 @@ def init_cloud_provider(state: StateStore) -> tuple[CloudProviderManager, DNSManager]: cloud_manager: CloudProviderManager = None domain_manager: DNSManager = None - # init proper cloud provider + # init proper cloud provider if state.cloud_provider == CloudProviders.AWS: + # need to check CLI dependencies before initializing cloud providers as they depend on cli tools + if not AWSManager.detect_cli_presence(): + raise click.ClickException("Cloud CLI is missing") + cloud_manager: AWSManager = AWSManager(state.get_input_param(CLOUD_REGION), state.get_input_param(CLOUD_PROFILE), state.get_input_param(CLOUD_ACCOUNT_ACCESS_KEY), @@ -54,6 +62,10 @@ def init_cloud_provider(state: StateStore) -> tuple[CloudProviderManager, DNSMan secret=state.get_input_param(DNS_REGISTRAR_ACCESS_SECRET)) elif state.cloud_provider == CloudProviders.Azure: + # need to check CLI dependencies before initializing cloud providers as they depend on cli tools + if not AzureManager.detect_cli_presence(): + raise click.ClickException("Cloud CLI is missing") + cloud_manager: AzureManager = AzureManager( state.get_input_param(CLOUD_PROFILE), state.get_input_param(CLOUD_REGION) ) @@ -144,3 +156,90 @@ def check_installation_presence(): if not os.path.exists(LOCAL_GITOPS_FOLDER): raise click.ClickException("GitOps repo does not exist") + + +def initialize_gitops_repository(state_store: StateStore, logger: Logger) -> tuple: + """ + Initialize and return the GitOps repository manager. + + This function sets up the GitOps repository manager using the provided state store configuration. + It initializes a GitOps repository object with necessary authentication and configuration details. + + Parameters: + state_store (StateStore): An instance of StateStore containing configuration and state information. + logger (Logger): A logger instance for logging the process. + + Returns: + tuple: A tuple containing the Git manager and GitOps repository instance. + The Git manager is used for Git operations, while the GitOps repository instance + is a specific repository related to GitOps operations. + + The function updates the GitOps repository to ensure it is synchronized with its remote version and logs + the initialization process. + """ + git_man = init_git_provider(state_store) + gor = PlatformGitOpsRepo( + git_man=git_man, + author_name=state_store.internals["GIT_USER_NAME"], + author_email=state_store.internals["GIT_USER_EMAIL"], + key_path=state_store.internals["DEFAULT_SSH_PRIVATE_KEY_PATH"] + ) + gor.update() + logger.info("GitOps repository initialized.") + return git_man, gor + + +def create_and_setup_branch(gor: PlatformGitOpsRepo, branch_name: str, logger: Logger) -> None: + """ + Create and set up a new branch for the workload in the GitOps repository. + + This function tries to create a new branch with the given name in the GitOps repository. + If the branch creation fails due to the branch already existing, it raises a + BranchAlreadyExistsException. + + Parameters: + gor (PlatformGitOpsRepo): Instance of the GitOps repository. + branch_name (str): Name of the branch to be created. + logger (Logger): A logger instance for logging the process. + + Raises: + BranchAlreadyExistsException: If the branch already exists in the repository. + """ + try: + logger.debug(f"Attempting to create branch '{branch_name}' in the GitOps repository.") + gor.create_branch(branch_name) + logger.info(f"Branch '{branch_name}' created successfully.") + except OSError as e: + logger.error(f"Error occurred while creating branch '{branch_name}': {e}") + raise GitBranchAlreadyExists(branch_name) + + +def create_and_open_pull_request( + gor: PlatformGitOpsRepo, + state_store: StateStore, + wl_name: str, + branch_name: str, + main_branch: str, + logger: Logger +) -> None: + """ + Create a pull request for the workload and open it in a web browser. + + Parameters: + gor: PlatformGitOpsRepo class instance. + state_store (StateStore): State store instance for accessing configuration. + wl_name (str): Name of the workload. + branch_name (str): The branch for which the pull request is created. + main_branch (str): Main branch of the repository. + logger (Logger): A logger instance for logging the process. + """ + try: + pr_url = gor.create_pr( + state_store.parameters[""], branch_name, main_branch, + f"Introduce {wl_name}", "Add default secrets, groups and default repository structure." + ) + webbrowser.open(pr_url, autoraise=False) + logger.info(f"Pull request created: {pr_url}") + except Exception as e: + logger.error(f"Error in creating pull request: {e}") + raise PullRequestCreationError(f"Could not create PR due to: {e}") diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index 64c2c68e..cb1f4a61 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -21,8 +21,8 @@ def __init__(self, region, profile, key, secret): def region(self): return self.__aws_sdk.region - @trace() - def detect_cli_presence(self) -> bool: + @classmethod + def detect_cli_presence(cls) -> bool: """Check whether `name` is on PATH and marked as executable.""" return detect_command_presence(CLI) diff --git a/tools/cli/services/cloud/azure/azure_manager.py b/tools/cli/services/cloud/azure/azure_manager.py index 02548e34..89eeb073 100644 --- a/tools/cli/services/cloud/azure/azure_manager.py +++ b/tools/cli/services/cloud/azure/azure_manager.py @@ -109,8 +109,8 @@ def get_k8s_auth_command(self) -> tuple[str, [str]]: def get_k8s_token(self, cluster_name: str) -> str: raise NotImplementedError() - @trace() - def detect_cli_presence(self) -> bool: + @classmethod + def detect_cli_presence(cls) -> bool: """Check whether dependencies are on PATH and marked as executable.""" return detect_command_presence(CLI) & detect_command_presence(K8s) diff --git a/tools/cli/services/cloud/cloud_provider_manager.py b/tools/cli/services/cloud/cloud_provider_manager.py index 88706e56..5402ec71 100644 --- a/tools/cli/services/cloud/cloud_provider_manager.py +++ b/tools/cli/services/cloud/cloud_provider_manager.py @@ -8,8 +8,9 @@ class CloudProviderManager(ABC): def region(self) -> str: pass + @classmethod @abstractmethod - def detect_cli_presence(self) -> bool: + def detect_cli_presence(cls) -> bool: """ Check if cloud provider CLI tools are installed :return: True or False diff --git a/tools/cli/services/wl_gitops_template_manager.py b/tools/cli/services/wl_gitops_template_manager.py deleted file mode 100644 index e244aadf..00000000 --- a/tools/cli/services/wl_gitops_template_manager.py +++ /dev/null @@ -1,100 +0,0 @@ -import os -import shutil - -from ghrepo import GHRepo -from git import Repo, GitError, Actor - -from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER -from common.const.const import WL_GITOPS_REPOSITORY_URL, WL_GITOPS_REPOSITORY_BRANCH -from common.tracing_decorator import trace - - -class WorkloadGitOpsTemplateManager: - """CG DevX Workload GitOps templates manager.""" - - def __init__(self, org_name: str, repo_name: str, key_path: str, template_url: str = None, - template_branch: str = None): - if template_url is None: - self._url = WL_GITOPS_REPOSITORY_URL - else: - self._url = template_url - - if template_branch is None: - self._branch = WL_GITOPS_REPOSITORY_BRANCH - else: - self._branch = template_branch - - self._git_org_name = org_name - self._repo_name = repo_name - self._key_path = key_path - self._wl_template_repo = None - self._wl_repo = None - - self._wl_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / self._repo_name - self._wl_template_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / GHRepo.parse(self._url).name - - @trace() - def clone_template(self): - if os.path.exists(self._wl_template_repo_folder): - shutil.rmtree(self._wl_template_repo_folder) - - os.makedirs(self._wl_template_repo_folder) - - try: - self._wl_template_repo = Repo.clone_from(self._url, - self._wl_template_repo_folder, - branch=self._branch, - env={ - "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) - return self._wl_template_repo_folder - except GitError as e: - return None - - @trace() - def clone_wl(self): - if os.path.exists(self._wl_repo_folder): - shutil.rmtree(self._wl_repo_folder) - os.makedirs(self._wl_repo_folder) - - try: - self._wl_repo = Repo.clone_from(GHRepo(self._git_org_name, self._repo_name).ssh_url, - self._wl_repo_folder, - env={ - "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) - return self._wl_repo_folder - except GitError as e: - return None - - @trace() - def bootstrap(self): - if os.path.exists(self._wl_template_repo_folder / ".git"): - shutil.rmtree(self._wl_template_repo_folder / ".git") - shutil.copytree(self._wl_template_repo_folder, self._wl_repo_folder, dirs_exist_ok=True) - - @trace() - def parametrise(self, params: dict = {}): - for root, dirs, files in os.walk(self._wl_repo_folder): - for name in files: - if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): - file_path = os.path.join(root, name) - with open(file_path, "r") as file: - data = file.read() - for k, v in params.items(): - data = data.replace(k, v) - with open(file_path, "w") as file: - file.write(data) - - @trace() - def upload(self, name: str = None, email: str = None): - self._wl_repo.git.add(all=True) - author = Actor(name=name, email=email) - self._wl_repo.index.commit("initial", author=author, committer=author) - - self._wl_repo.remotes.origin.push(self._wl_repo.active_branch.name) - - @trace() - def cleanup(self): - if os.path.exists(self._wl_template_repo_folder): - shutil.rmtree(self._wl_template_repo_folder) - if os.path.exists(self._wl_repo_folder): - shutil.rmtree(self._wl_repo_folder) diff --git a/tools/cli/services/wl_template_manager.py b/tools/cli/services/wl_template_manager.py index aa395618..6db13e2c 100644 --- a/tools/cli/services/wl_template_manager.py +++ b/tools/cli/services/wl_template_manager.py @@ -1,104 +1,288 @@ import os import shutil +from pathlib import Path +from typing import Optional, Union, List, Dict from ghrepo import GHRepo from git import Repo, GitError, Actor from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER -from common.const.const import WL_REPOSITORY_URL, WL_REPOSITORY_BRANCH +from common.custom_excpetions import RepositoryNotInitializedError +from common.logging_config import logger from common.tracing_decorator import trace -class WorkloadTemplateManager: +class WorkloadManager: """CG DevX Workload templates manager.""" - def __init__(self, org_name: str, repo_name: str, key_path: str, template_url: str = None, - template_branch: str = None): - if template_url is None: - self._url = WL_REPOSITORY_URL - else: - self._url = template_url - - if template_branch is None: - self._branch = WL_REPOSITORY_BRANCH - else: - self._branch = template_branch - + def __init__( + self, + org_name: str, + repo_name: str, + key_path: str, + template_url: Optional[str] = None, + template_branch: Optional[str] = None + ): + self._url = template_url + self._branch = template_branch self._git_org_name = org_name - self._repo_name = repo_name - self._key_path = key_path - self._wl_template_repo = None - self._wl_repo = None + self.repo_name = repo_name + self.key_path = key_path + self.wl_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / repo_name + self.template_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / GHRepo.parse(self._url).name + self.wl_repo = None + self.template_repo = None - self._wl_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / self._repo_name - self._wl_template_repo_folder = LOCAL_WORKLOAD_TEMP_FOLDER / GHRepo.parse(self._url).name + @trace() + def clone_template(self) -> str: + """ + Clone the Git repository template. + """ + self._prepare_clone_folder(folder=self.template_repo_folder) + self._clone_repository(url=self._url, branch=self._branch, folder=self.template_repo_folder) + return self.template_repo_folder @trace() - def clone_template(self): - if os.path.exists(self._wl_template_repo_folder): - shutil.rmtree(self._wl_template_repo_folder) + def clone_wl(self) -> str: + """ + Clone the workload Git repository. + """ + wl_repo_url = GHRepo(owner=self._git_org_name, name=self.repo_name).ssh_url + self._prepare_clone_folder(folder=self.wl_repo_folder) + self.wl_repo = self._clone_repository(url=wl_repo_url, folder=self.wl_repo_folder) + return self.wl_repo_folder - os.makedirs(self._wl_template_repo_folder) + @trace() + def bootstrap(self, services: Optional[List[str]] = None) -> None: + """ + Sets up the workload repository using the cloned template. + Args: + services (Optional[List[str]]): A list of service names to be set up in the workload repository. + """ + logger.info("Starting bootstrap process for the workload repository.") try: - self._wl_template_repo = Repo.clone_from(self._url, - self._wl_template_repo_folder, - branch=self._branch, - env={ - "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) - return self._wl_template_repo_folder + self._remove_git_directory() + self._copy_template_to_workload_repo() + self._setup_services(services) + logger.info("Bootstrap completed successfully.") except GitError as e: - return None + logger.error(f"Git error during bootstrap process: {e}") + raise + except shutil.Error as e: + logger.error(f"Shutil error during file operations: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error during bootstrap process: {e}") + raise @trace() - def clone_wl(self): - if os.path.exists(self._wl_repo_folder): - shutil.rmtree(self._wl_repo_folder) - os.makedirs(self._wl_repo_folder) + def parametrise(self, params: Optional[Dict[str, str]] = None) -> None: + """ + Update file contents within the workload repository, replacing placeholders with actual values. + Args: + params (Optional[Dict[str, str]]): Key-value pairs where the key is a placeholder and the value is the actual value. + """ + if not params: + logger.info("No parameters provided for parameterization. Skipping process.") + return + + logger.info(f"Parameterizing workload repository '{self.repo_name}' with provided parameters.") try: - self._wl_repo = Repo.clone_from(GHRepo(self._git_org_name, self._repo_name).ssh_url, - self._wl_repo_folder, - env={ - "GIT_SSH_COMMAND": f'ssh -o StrictHostKeyChecking=no -i {self._key_path}'}) - return self._wl_repo_folder - except GitError as e: - return None + # Processing each file in the repository and replacing placeholders + self._replace_placeholders_in_folder(folder=self.wl_repo_folder, params=params) + logger.info(f"Parameterization of repository '{self.repo_name}' completed successfully.") + except GitError as git_err: + logger.error(f"Git error during parameterization: {git_err}") + raise + except shutil.Error as shutil_err: + logger.error(f"Error during file copying operations: {shutil_err}") + raise + except IOError as io_err: + logger.error(f"I/O error during file processing: {io_err}") + raise @trace() - def bootstrap(self, services: [str]): - if os.path.exists(self._wl_template_repo_folder / ".git"): - shutil.rmtree(self._wl_template_repo_folder / ".git") - shutil.copytree(self._wl_template_repo_folder, self._wl_repo_folder, dirs_exist_ok=True) - for wl_svc_name in services: - shutil.copytree(self._wl_repo_folder / "wl-service-name", self._wl_repo_folder / wl_svc_name, - dirs_exist_ok=True) - shutil.rmtree(self._wl_repo_folder / "wl-service-name") + def upload(self, author_name: str, author_email: str) -> None: + """ + Commit and push changes to the workload repository. + + Args: + author_name (str): The name of the author for the commit. + author_email (str): The email of the author for the commit. + """ + if not self.wl_repo: + logger.error("Workload repository is not initialized. Please clone it first.") + raise RepositoryNotInitializedError("Workload repository not initialized") + + logger.info(f"Uploading changes to the repository '{self.repo_name}'.") + + try: + self.wl_repo.git.add(all=True) + author = Actor(name=author_name, email=author_email) + self.wl_repo.index.commit("initial commit", author=author, committer=author) + self.wl_repo.remotes.origin.push(self.wl_repo.active_branch.name) + logger.info(f"Changes by {author_name} <{author_email}> uploaded successfully.") + except GitError as e: + logger.error(f"GitError during upload: {e}") + raise @trace() - def parametrise(self, params: dict = {}): - for root, dirs, files in os.walk(self._wl_repo_folder): + def cleanup(self) -> None: + """ + Clean up the cloned template and workload repositories by removing their folders. + """ + try: + self._remove_folder(self.template_repo_folder) + self._remove_folder(self.wl_repo_folder) + logger.info("Cleanup completed successfully for both template and workload repositories.") + except shutil.Error as e: + logger.error(f"Error during cleanup process: {e}") + raise + + def _replace_placeholders_in_folder(self, folder: Union[str, Path], params: Dict[str, str]) -> None: + """ + Replace placeholders in all eligible files within the specified folder. + + Args: + folder (Union[str, Path]): Directory containing files to process. + params (Dict[str, str]): Key-value pairs for placeholder replacement. + """ + logger.debug(f"Scanning folder '{folder}' for files to parameterize.") + for root, dirs, files in os.walk(folder): for name in files: - if name.endswith(".tf") or name.endswith(".yaml") or name.endswith(".yml") or name.endswith(".md"): - file_path = os.path.join(root, name) - with open(file_path, "r") as file: - data = file.read() - for k, v in params.items(): - data = data.replace(k, v) - with open(file_path, "w") as file: - file.write(data) + if self._is_parametrizable_file(name): + file_path = Path(root) / name + # Process individual file + self._replace_placeholder_in_file(file_path, params) - @trace() - def upload(self, name: str = None, email: str = None): - self._wl_repo.git.add(all=True) - author = Actor(name=name, email=email) - self._wl_repo.index.commit("initial", author=author, committer=author) + @staticmethod + def _is_parametrizable_file(file_name: str) -> bool: + """ + Check if a file is eligible for parameterization based on its extension. - self._wl_repo.remotes.origin.push(self._wl_repo.active_branch.name) + Args: + file_name (str): Name of the file to check. - @trace() - def cleanup(self): - if os.path.exists(self._wl_template_repo_folder): - shutil.rmtree(self._wl_template_repo_folder) - if os.path.exists(self._wl_repo_folder): - shutil.rmtree(self._wl_repo_folder) + Returns: + bool: True if the file is eligible for parameterization, False otherwise. + """ + is_parametrizable = file_name.endswith((".tf", ".yaml", ".yml", ".md")) + if is_parametrizable: + logger.debug(f"File '{file_name}' is eligible for parameterization.") + return is_parametrizable + + @staticmethod + def _replace_placeholder_in_file(file_path: Path, params: Dict[str, str]) -> None: + """ + Process a single file, replacing placeholders with actual values. + + Args: + file_path (Path): Path of the file to process. + params (Dict[str, str]): Key-value pairs for placeholder replacement. + """ + with open(file_path, "r") as file: + data = file.read() + + # Replace each placeholder in the file + for placeholder, value in params.items(): + data = data.replace(placeholder, value) + + with open(file_path, "w") as file: + file.write(data) + + logger.debug(f"File '{file_path}' parameterized successfully.") + + def _remove_git_directory(self) -> None: + """Removes the .git directory from the template repository folder if it exists.""" + git_dir = self.template_repo_folder / ".git" + self._remove_folder(folder=git_dir) + + def _copy_template_to_workload_repo(self) -> None: + """Copies the entire contents of the template repository to the workload repository.""" + logger.debug(f"Copying template from {self.template_repo_folder} to {self.wl_repo_folder}.") + shutil.copytree(self.template_repo_folder, self.wl_repo_folder, dirs_exist_ok=True) + + def _setup_services(self, services: Optional[List[str]]) -> None: + """ + Sets up individual services in the workload repository based on the provided list. + + Args: + services (Optional[List[str]]): A list of service names to set up. + """ + if not services: + logger.debug("No additional services to set up.") + return + + service_src = self.wl_repo_folder / "wl-service-name" + if not os.path.exists(service_src): + logger.warning(f"Service template directory '{service_src}' does not exist.") + return + + for wl_svc_name in services: + service_dst = self.wl_repo_folder / wl_svc_name + logger.debug(f"Setting up service: {wl_svc_name}") + shutil.copytree(src=service_src, dst=service_dst, dirs_exist_ok=True) + + logger.debug(f"Cleaning up template service directory: {service_src}.") + self._remove_folder(folder=service_src) + + def _prepare_clone_folder(self, folder: Union[str, Path]) -> None: + """ + Prepares the folder for cloning by removing it if it exists and then creating it. + + Args: + folder (Union[str, Path]): Local directory path to prepare for cloning. + """ + self._remove_folder(folder=folder) + logger.debug(f"Creating folder: {folder}") + os.makedirs(name=folder, exist_ok=True) + logger.info(f"Folder '{folder}' prepared for cloning.") + + def _clone_repository(self, url: str, folder: Union[str, Path], branch: Optional[str] = None) -> Repo: + """ + Clones the repository into the specified folder. If the branch is not specified, + clones the default branch. + + Args: + url (str): URL of the repository to clone. + folder (Union[str, Path]): Local directory path to clone the repository into. + branch (Optional[str]): Branch name to clone. If None, the default branch is cloned. + + Returns: + Repo: The cloned Git repository object. + """ + logger.info(f"Cloning repository from {url} into {folder}") + + clone_kwargs = { + "url": url, + "to_path": folder, + "env": {"GIT_SSH_COMMAND": f"ssh -o StrictHostKeyChecking=no -i {self.key_path}"} + } + + if branch: + clone_kwargs["branch"] = branch + + try: + repo = Repo.clone_from(**clone_kwargs) + logger.info("Repository cloned successfully.") + return repo + except GitError as e: + logger.error(f"Error while cloning repository: {e}") + raise + + @staticmethod + def _remove_folder(folder: Union[str, Path]) -> None: + """ + Remove the specified folder if it exists. + + Args: + folder (Union[str, Path]): The path to the folder to be removed. + """ + if os.path.exists(path=folder): + logger.debug(f"Removing folder: {folder}") + shutil.rmtree(path=folder) + logger.info(f"Folder {folder} removed successfully.") + else: + logger.debug(f"Folder {folder} does not exist. No action needed.") From 509333e2927dcea74e99a2fb843e6a7db650a10a Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Fri, 5 Jan 2024 21:11:11 +0100 Subject: [PATCH 42/49] post wl refactor changes --- platform/terraform/hosting_provider/output.tf | 5 + .../terraform/modules/cloud_aws/outputs.tf | 5 +- .../terraform/modules/cloud_azure/outputs.tf | 6 + tools/cli/commands/setup.py | 1 + tools/cli/commands/workload/bootstrap.py | 320 +++++++----------- tools/cli/commands/workload/create.py | 32 +- tools/cli/commands/workload/delete.py | 24 +- tools/cli/common/const/const.py | 2 + tools/cli/common/utils/command_utils.py | 37 +- tools/cli/services/wl_template_manager.py | 23 ++ 10 files changed, 215 insertions(+), 240 deletions(-) diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index 5010689d..f35b8d9c 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -38,6 +38,11 @@ output "cluster_certificate_authority_data" { sensitive = true } +output "cluster_oidc_provider" { + value = module.hosting-provider.cluster_oidc_provider + description = "Cluster OIDC provider" +} + # secret manager output "secret_manager_seal_key" { value = module.hosting-provider.secret_manager_unseal_key diff --git a/platform/terraform/modules/cloud_aws/outputs.tf b/platform/terraform/modules/cloud_aws/outputs.tf index deaf47cd..fef4b175 100644 --- a/platform/terraform/modules/cloud_aws/outputs.tf +++ b/platform/terraform/modules/cloud_aws/outputs.tf @@ -153,13 +153,12 @@ output "node_security_group_id" { ################################################################################ # IRSA ################################################################################ - -output "cluster_oidc_provider" { +output "cluster_oidc_provider_url" { description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" value = module.eks.oidc_provider } -output "cluster_oidc_provider_arn" { +output "cluster_oidc_provider" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = module.eks.oidc_provider_arn } diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index f4a9f16a..03ad8681 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -56,3 +56,9 @@ output "artifacts_storage" { value = azurerm_storage_container.artifacts_repository.name description = "Continuous Integration Artifact Repository storage backend" } + +output "cluster_oidc_provider" { + value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + sensitive = true + description = "The OpenID Connect identity provider." +} \ No newline at end of file diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 4cb94546..da1298c4 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -346,6 +346,7 @@ def setup( p.internals["CC_CLUSTER_ENDPOINT"] = hp_out["cluster_endpoint"] p.internals["CC_CLUSTER_CA_CERT_DATA"] = hp_out["cluster_certificate_authority_data"] p.internals["CC_CLUSTER_CA_CERT_PATH"] = write_ca_cert(hp_out["cluster_certificate_authority_data"]) + p.parameters[""] = hp_out["cluster_oidc_provider"] # artifact storage p.parameters[""] = hp_out["artifact_storage"] # kms keys diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 62aa6dcc..7600a9df 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -1,14 +1,19 @@ +import shutil import time +import webbrowser from typing import Dict import click +from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER from common.const.const import WL_REPOSITORY_URL, WL_GITOPS_REPOSITORY_URL, WL_GITOPS_REPOSITORY_BRANCH, \ - WL_REPOSITORY_BRANCH, TERRAFORM_VERSION -from common.custom_excpetions import WorkloadManagerError + TERRAFORM_VERSION, WL_PR_BRANCH_NAME_PREFIX, WL_REPOSITORY_BRANCH, WL_SERVICE_NAME +from common.custom_excpetions import WorkloadManagerError, GitBranchAlreadyExists +from common.enums.cloud_providers import CloudProviders from common.logging_config import configure_logging, logger from common.state_store import StateStore -from common.utils.command_utils import str_to_kebab, init_cloud_provider +from common.utils.command_utils import init_cloud_provider, preprocess_workload_names, \ + init_git_provider from services.wl_template_manager import WorkloadManager @@ -43,7 +48,6 @@ help='Workload repository template branch', type=click.STRING, default=WL_REPOSITORY_BRANCH - ) @click.option( '--workload-gitops-template-url', @@ -66,7 +70,8 @@ '-wls', 'wl_svc_name', help='Workload service name', - type=click.STRING + type=click.STRING, + default=WL_SERVICE_NAME ) @click.option( '--workload-service-port', @@ -132,7 +137,10 @@ def bootstrap( quay_proxy = state_store.parameters[""] git_runner_group_name = state_store.parameters[""] git_organisation_name = state_store.parameters[""] - click.echo("1/10: Configuration loaded.") + cluster_oidc_issuer_url = state_store.parameters[""] + cluster_name = state_store.parameters[""] + + click.echo("1/11: Configuration loaded.") except KeyError as e: error_message = f'Configuration loading failed due to missing key: {e}. ' \ 'Please verify the state store configuration and ensure all required keys are present. ' \ @@ -142,55 +150,60 @@ def bootstrap( # Initialize cloud manager for GitOps parameters cloud_man, _ = init_cloud_provider(state_store) - click.echo("2/10: Cloud manager initialized for GitOps.") + click.echo("2/11: Cloud manager initialized for GitOps.") # Initialize WorkloadManager for the workload repository - wl_name, wl_repo_name, wl_gitops_repo_name = process_workload_names( + wl_name, wl_repo_name, wl_gitops_repo_name = preprocess_workload_names( + logger=logger, wl_name=wl_name, wl_repo_name=wl_repo_name, wl_gitops_repo_name=wl_gitops_repo_name ) - click.echo("3/10: Workload names processed.") + click.echo("3/11: Workload names processed.") # Prepare parameters for workload and GitOps repositories - wl_repo_params = _prepare_workload_params( - wl_name=wl_name, - wl_svc_name=wl_repo_name, - registry_registry_url=registry_url, - registry_dockerhub_proxy=dockerhub_proxy, - registry_gcr_proxy=gcr_proxy, - registry_k8s_gcr_proxy=k8s_gcr_proxy, - registry_quay_proxy=quay_proxy, - wl_repo_name=wl_repo_name, - wl_gitops_repo_name=wl_gitops_repo_name, - git_organization_name=git_organisation_name, - git_runner_group_name=git_runner_group_name - ) - wl_gitops_params = _prepare_gitops_params( - wl_name=wl_name, - wl_svc_name=wl_svc_name, - wl_svc_port=wl_svc_port, - cc_cluster_fqdn=cc_cluster_fqdn, - registry_registry_url=registry_url, - k8s_role_mapping=cloud_man.create_k8s_cluster_role_mapping_snippet(), - additional_labels=cloud_man.create_additional_labels(), - tf_secrets_backend=cloud_man.create_iac_backend_snippet( + wl_repo_params = { + "": wl_name, + "": wl_svc_name, + "": registry_url, + "": dockerhub_proxy, + "": gcr_proxy, + "": k8s_gcr_proxy, + "": quay_proxy, + "": wl_repo_name, + "": wl_gitops_repo_name, + "": git_organisation_name, + "": git_runner_group_name, + } + + wl_gitops_params = { + "": wl_name, + "": wl_svc_name, + "": f'{wl_svc_name}.{wl_name}.{cc_cluster_fqdn}', + "": f'{registry_url}/{wl_name}/{wl_svc_name}', + "": str(wl_svc_port), + "# ": cloud_man.create_k8s_cluster_role_mapping_snippet(), + "": f"{cluster_name}-{wl_name}-{wl_svc_name}-role", + "# ": cloud_man.create_additional_labels(), + "# ": cloud_man.create_iac_backend_snippet( location=tf_backend_storage_name, service=f"workloads/{wl_name}/secrets" ), - tf_hosting_backend=cloud_man.create_iac_backend_snippet( + "# ": cloud_man.create_iac_backend_snippet( location=tf_backend_storage_name, service=f"workloads/{wl_name}/hosting_provider" ), - hosting_provider_snippet=cloud_man.create_hosting_provider_snippet(), - dockerhub_proxy=dockerhub_proxy, - registry_gcr_proxy=gcr_proxy, - registry_k8s_gcr_proxy=k8s_gcr_proxy, - registry_quay_proxy=quay_proxy, - git_runner_group_name=git_runner_group_name, - terraform_version=TERRAFORM_VERSION - ) - click.echo("4/10: Parameters for workload and GitOps repositories prepared.") + "# ": cloud_man.create_hosting_provider_snippet(), + "": dockerhub_proxy, + "": gcr_proxy, + "": k8s_gcr_proxy, + "": quay_proxy, + "": git_runner_group_name, + "": TERRAFORM_VERSION, + "": cluster_oidc_issuer_url, + "": cluster_name, + } + click.echo("4/11: Parameters for workload and GitOps repositories prepared.") # Initialize WorkloadManager for the workload repository wl_manager = WorkloadManager( @@ -200,21 +213,20 @@ def bootstrap( template_url=wl_template_url, template_branch=wl_template_branch ) - click.echo("5/10: Workload repository manager initialized.") + click.echo("5/11: Workload repository manager initialized.") try: # Perform bootstrap steps for the workload repository - perform_bootstrap( - workload_manager=wl_manager, - params=wl_repo_params - ) - click.echo("6/10: Workload repository bootstrap process completed.") + perform_bootstrap(workload_manager=wl_manager, params=wl_repo_params, + author_name=state_store.internals["GIT_USER_NAME"], + author_email=state_store.internals["GIT_USER_EMAIL"]) + click.echo("6/11: Workload repository bootstrap process completed.") except WorkloadManagerError as e: raise click.ClickException(str(e)) # Cleanup temporary folder for workload repository wl_manager.cleanup() - click.echo("7/10: Workload repository cleanup completed.") + click.echo("7/11: Workload repository cleanup completed.") # Initialize WorkloadManager for the GitOps repository wl_gitops_manager = WorkloadManager( @@ -224,55 +236,95 @@ def bootstrap( template_url=wl_gitops_template_url, template_branch=wl_gitops_template_branch ) - click.echo("8/10: GitOps repository manager initialized.") + click.echo("8/11: GitOps repository manager initialized.") try: - # Perform bootstrap steps for the GitOps repositoryR + # Perform bootstrap steps for the GitOps repository perform_bootstrap( workload_manager=wl_gitops_manager, - params=wl_gitops_params + params=wl_gitops_params, + author_name=state_store.internals["GIT_USER_NAME"], + author_email=state_store.internals["GIT_USER_EMAIL"], ) - click.echo("9/10: GitOps repository bootstrap process completed.") + click.echo("9/11: GitOps repository bootstrap process completed.") + except WorkloadManagerError as e: + raise click.ClickException(str(e)) + + try: + # Create a PR in GitOps repository to trigger IaC execution + branch_name = f"{WL_PR_BRANCH_NAME_PREFIX}{wl_name}-iac-init" + git_man = init_git_provider(state_store) + wl_gitops_manager.update() + wl_gitops_manager.create_branch(branch_name) + + perform_gitops_iac_bootstrap(state_store, wl_gitops_repo_name, wl_gitops_params) + + wl_gitops_manager.upload(author_name=state_store.internals["GIT_USER_NAME"], + author_email=state_store.internals["GIT_USER_EMAIL"]) + wl_gitops_manager.switch_to_branch() + + pr_url = git_man.create_pr(wl_gitops_repo_name, branch_name, "main", + f"Execute IaC changes for {wl_name}", "Setup default secrets.") + webbrowser.open(pr_url, autoraise=False) + + click.echo("10/11: Created PR for GitOps repository.") + except WorkloadManagerError as e: raise click.ClickException(str(e)) + except GitBranchAlreadyExists as e: + raise click.ClickException(str(e)) # Cleanup temporary folder for GitOps repository wl_gitops_manager.cleanup() - click.echo("10/10: GitOps repository cleanup completed.") + click.echo("11/11: GitOps repository cleanup completed.") click.echo(f"Bootstrapping workload completed in {time.time() - func_start_time:.2f} seconds.") -def process_workload_names(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str): +def perform_gitops_iac_bootstrap(state_store, wl_gitops_repo_name, wl_gitops_params): """ - Process and normalize workload names to a standard format. + Perform the GitOps repo IaC bootstrap process. Args: - wl_name (str): Name of the workload. - wl_repo_name (str): Name of the workload repository. + state_store (StateStore): An instance of StateStore containing configuration and state information. wl_gitops_repo_name (str): Name of the workload GitOps repository. + wl_gitops_params (Dict[str, str]): Dictionary containing parameters for bootstrap. - Returns: - tuple[str, str, str]: Tuple of processed workload name, workload repository name, and GitOps repository name. """ - processed_wl_name = str_to_kebab(wl_name) - processed_wl_repo_name = str_to_kebab(wl_repo_name or wl_name) - processed_wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name or f"{wl_repo_name}-gitops") - - return processed_wl_name, processed_wl_repo_name, processed_wl_gitops_repo_name - - -def perform_bootstrap( - workload_manager: WorkloadManager, - params: Dict[str, str] -): + cloud_provider = state_store.parameters[""] + identity_template_file = LOCAL_WORKLOAD_TEMP_FOLDER / f"{wl_gitops_repo_name}/terraform/infrastructure/samples/{cloud_provider}/" + if cloud_provider == CloudProviders.AWS: + identity_template_file = identity_template_file / "irsa_role.tf" + elif cloud_provider == CloudProviders.Azure: + identity_template_file = identity_template_file / "managed_identity.tf" + else: + raise click.ClickException("Unknown cloud provider") + + identity_file = LOCAL_WORKLOAD_TEMP_FOLDER / f"{wl_gitops_repo_name}/terraform/infrastructure/wl_identities.tf" + # with open(identity_template_file, "r") as file: + # data = file.read() + # for k, v in wl_gitops_params.items(): + # data = data.replace(k, v) + # + # with open(identity_file, "w") as file: + # file.write(data) + + shutil.copy(identity_template_file, identity_file) + + # modify secrets.tf to trigger atlantis + with open(LOCAL_WORKLOAD_TEMP_FOLDER / f"{wl_gitops_repo_name}/terraform/secrets/secrets.tf", "a") as file: + file.write("\n") + + +def perform_bootstrap(workload_manager: WorkloadManager, params: Dict[str, str], author_name: str, author_email: str): """ Perform the bootstrap process using the WorkloadManager. Args: workload_manager (WorkloadManager): The workload manager instance. params (Dict[str, str]): Dictionary containing parameters for bootstrap. - + author_name (str): Git commit author name. + author_email (str): Git commit author email. Raises: WorkloadManagerError: If cloning, templating, or uploading process fails. """ @@ -283,132 +335,12 @@ def perform_bootstrap( raise WorkloadManagerError("Failed to clone workload repository") # Bootstrap workload with the given service names - service_names = [params.get("", "default-service")] + service_names = [params.get("")] workload_manager.bootstrap(service_names) workload_manager.parametrise(params) workload_manager.upload( - author_name=params.get(""), - author_email=params.get("") + author_name=author_name, + author_email=author_email ) - - -def _prepare_workload_params( - wl_name: str, - wl_svc_name: str, - registry_registry_url: str, - registry_dockerhub_proxy: str, - registry_gcr_proxy: str, - registry_k8s_gcr_proxy: str, - registry_quay_proxy: str, - wl_repo_name: str, - wl_gitops_repo_name: str, - git_organization_name: str, - git_runner_group_name: str -) -> Dict[str, str]: - """ - Prepare parameters for the workload repository bootstrap process. - - Args: - wl_name (str): Workload name. - wl_svc_name (str): Workload service name. - registry_registry_url (str): URL for the registry. - registry_dockerhub_proxy (str): Proxy URL for DockerHub. - registry_gcr_proxy (str): Proxy URL for Google Container Registry. - registry_k8s_gcr_proxy (str): Proxy URL for Kubernetes GCR. - registry_quay_proxy (str): Proxy URL for Quay.io. - wl_repo_name (str): Name of the workload repository. - wl_gitops_repo_name (str): Name of the GitOps repository for the workload. - git_organization_name (str): Name of the Git organization. - git_runner_group_name (str): Name of the Git runner group. - - Returns: - Dict[str, str]: A dictionary containing prepared parameters for the workload repository bootstrap process. - """ - logger.debug(f"Preparing workload parameters for '{wl_name}' with service '{wl_svc_name}'.") - - params = { - "": wl_name, - "": wl_svc_name, - "": registry_registry_url, - "": registry_dockerhub_proxy, - "": registry_gcr_proxy, - "": registry_k8s_gcr_proxy, - "": registry_quay_proxy, - "": wl_repo_name, - "": wl_gitops_repo_name, - "": git_organization_name, - "": git_runner_group_name - } - - logger.debug(f"Workload parameters prepared: {params}") - return params - - -def _prepare_gitops_params( - wl_name: str, - wl_svc_name: str, - wl_svc_port: str | int, - cc_cluster_fqdn: str, - registry_registry_url: str, - k8s_role_mapping: str, - additional_labels: str, - tf_secrets_backend: str, - tf_hosting_backend: str, - hosting_provider_snippet: str, - dockerhub_proxy: str, - registry_gcr_proxy: str, - registry_k8s_gcr_proxy: str, - registry_quay_proxy: str, - git_runner_group_name: str, - terraform_version: str -) -> Dict[str, str]: - """ - Prepare parameters for the GitOps repository bootstrap process. - - Args: - wl_name (str): Name of the workload. - wl_svc_name (str): Name of the workload service. - wl_svc_port (str | int): Port number for the workload service. - cc_cluster_fqdn (str): Fully qualified domain name for the cluster. - registry_registry_url (str): URL for the registry. - k8s_role_mapping (str): Kubernetes role mapping snippet. - additional_labels (str): Additional labels snippet. - tf_secrets_backend (str): Terraform secrets backend snippet. - tf_hosting_backend (str): Terraform hosting backend snippet. - hosting_provider_snippet (str): Hosting provider snippet. - dockerhub_proxy (str): Proxy URL for DockerHub. - registry_gcr_proxy (str): Proxy URL for Google Container Registry. - registry_k8s_gcr_proxy (str): Proxy URL for Kubernetes GCR. - registry_quay_proxy (str): Proxy URL for Quay.io. - git_runner_group_name (str): Name of the Git runner group. - terraform_version (str): Version of Terraform to use. - - Returns: - Dict[str, str]: A dictionary containing prepared parameters for the GitOps repository bootstrap process. - """ - logger.debug(f"Preparing GitOps parameters for workload '{wl_name}' with service '{wl_svc_name}'.") - - params = { - "": wl_name, - "": wl_svc_name, - "": f'{wl_svc_name}.{wl_name}.{cc_cluster_fqdn}', - "": f'{registry_registry_url}/{wl_name}/{wl_svc_name}', - "": str(wl_svc_port), - "": k8s_role_mapping, - "": "[Placeholder for workload service role mapping]", - "": additional_labels, - "": tf_secrets_backend, - "": tf_hosting_backend, - "": hosting_provider_snippet, - "": dockerhub_proxy, - "": registry_gcr_proxy, - "": registry_k8s_gcr_proxy, - "": registry_quay_proxy, - "": git_runner_group_name, - "": terraform_version - } - - logger.debug(f"GitOps parameters prepared: {params}") - return params diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index a79ff4ad..8d15f0b8 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -2,12 +2,12 @@ import click -from common.const.const import WL_REPOSITORY_BRANCH, WL_PR_BRANCH_NAME_PREFIX +from common.const.const import GITOPS_REPOSITORY_MAIN_BRANCH, WL_PR_BRANCH_NAME_PREFIX from common.custom_excpetions import GitBranchAlreadyExists, PullRequestCreationError from common.logging_config import configure_logging, logger from common.state_store import StateStore from common.utils.command_utils import str_to_kebab, check_installation_presence, \ - initialize_gitops_repository, create_and_setup_branch, create_and_open_pull_request + initialize_gitops_repository, create_and_setup_branch, create_and_open_pull_request, preprocess_workload_names from services.platform_gitops import PlatformGitOpsRepo @@ -30,7 +30,7 @@ '--workload-gitops-main-branch-name', '-wlgmbn', 'main_branch', - default=WL_REPOSITORY_BRANCH, + default=GITOPS_REPOSITORY_MAIN_BRANCH, help='Workload GitOps repository main branch name', type=click.STRING ) @click.option( @@ -60,7 +60,8 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branc state_store = StateStore() click.echo("1/7: State store initialized.") - wl_name, wl_repo_name, wl_gitops_repo_name = _process_workload_names( + wl_name, wl_repo_name, wl_gitops_repo_name = preprocess_workload_names( + logger=logger, wl_name=wl_name, wl_repo_name=wl_repo_name, wl_gitops_repo_name=wl_gitops_repo_name @@ -89,7 +90,8 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branc create_and_open_pull_request( gor=gor, state_store=state_store, - wl_name=wl_name, + title=f"Introduce {wl_name}", + body="Add default secrets, groups and default repository structure.", branch_name=branch_name, main_branch=main_branch, logger=logger @@ -104,26 +106,6 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branc click.echo(f"Workload GitOps code creation completed in {time.time() - func_start_time:.2f} seconds.") -def _process_workload_names(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str) -> tuple[str, str, str]: - """ - Process and normalize workload names to a standard format. - - Parameters: - wl_name (str): Name of the workload. - wl_repo_name (str): Name of the workload repository. - wl_gitops_repo_name (str): Name of the workload GitOps repository. - - Returns: - tuple[str, str, str]: Tuple of processed workload name, workload repository name, and GitOps repository name. - """ - logger.debug(f"Processing workload names: {wl_name}, {wl_repo_name}, {wl_gitops_repo_name}") - processed_wl_name = str_to_kebab(wl_name) - processed_wl_repo_name = str_to_kebab(wl_repo_name or wl_name) - processed_wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name or f"{wl_repo_name}-gitops") - logger.info(f"Processed names: {processed_wl_name}, {processed_wl_repo_name}, {processed_wl_gitops_repo_name}") - return processed_wl_name, processed_wl_repo_name, processed_wl_gitops_repo_name - - def add_workload_and_commit( gor: PlatformGitOpsRepo, wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str ) -> None: diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index 3b0bd00a..979c9f4c 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -5,7 +5,7 @@ import click from common.const.common_path import LOCAL_WORKLOAD_TEMP_FOLDER -from common.const.const import WL_REPOSITORY_BRANCH, WL_PR_BRANCH_NAME_PREFIX, WL_GITOPS_REPOSITORY_BRANCH, \ +from common.const.const import GITOPS_REPOSITORY_MAIN_BRANCH, WL_PR_BRANCH_NAME_PREFIX, WL_GITOPS_REPOSITORY_BRANCH, \ WL_GITOPS_REPOSITORY_URL from common.const.parameter_names import GIT_ACCESS_TOKEN, GIT_ORGANIZATION_NAME from common.custom_excpetions import GitBranchAlreadyExists, PullRequestCreationError @@ -13,7 +13,7 @@ from common.state_store import StateStore from common.utils.command_utils import str_to_kebab, prepare_cloud_provider_auth_env_vars, set_envs, \ check_installation_presence, initialize_gitops_repository, create_and_setup_branch, \ - create_and_open_pull_request + create_and_open_pull_request, preprocess_workload_names from services.platform_gitops import PlatformGitOpsRepo from services.tf_wrapper import TfWrapper from services.wl_template_manager import WorkloadManager @@ -40,7 +40,7 @@ '--workload-gitops-main-branch-name', '-wlgmbn', 'main_branch', - default=WL_REPOSITORY_BRANCH, + default=GITOPS_REPOSITORY_MAIN_BRANCH, help='Workload GitOps repository main branch name', type=click.STRING ) @click.option( @@ -105,7 +105,11 @@ def delete( check_installation_presence() click.echo(f"2/{logging_total_steps}: Logging and installation checked.") - wl_name, wl_gitops_repo_name = _process_workload_names(wl_name=wl_name, wl_gitops_repo_name=wl_gitops_repo_name) + wl_name, wl_repo_name, wl_gitops_repo_name = preprocess_workload_names( + logger=logger, + wl_name=wl_name, + wl_gitops_repo_name=wl_gitops_repo_name, + ) click.echo(f"3/{logging_total_steps}: Workload names processed.") git_man, gor = initialize_gitops_repository(state_store=state_store, logger=logger) @@ -145,7 +149,8 @@ def delete( create_and_open_pull_request( gor=gor, state_store=state_store, - wl_name=wl_name, + title=f"Remove {wl_name}", + body="Remove default secrets, groups and repository structure.", branch_name=branch_name, main_branch=main_branch, logger=logger @@ -160,15 +165,6 @@ def delete( click.echo(f"Deleting workload GitOps code completed in {time.time() - func_start_time:.2f} seconds.") -def _process_workload_names(wl_name: str, wl_gitops_repo_name: str) -> tuple[str, str]: - """Normalize and process workload and GitOps repository names.""" - logger.debug(f"Processing workload names: {wl_name}, {wl_gitops_repo_name}") - wl_name = str_to_kebab(wl_name) - wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name or f"{wl_name}-gitops") - logger.info(f"Processed names: {wl_name}, {wl_gitops_repo_name}") - return wl_name, wl_gitops_repo_name - - def _remove_workload_and_commit(gor: PlatformGitOpsRepo, wl_name: str) -> None: """ Remove the workload to the GitOps repository and commit changes. diff --git a/tools/cli/common/const/const.py b/tools/cli/common/const/const.py index c0e1740d..07211f39 100644 --- a/tools/cli/common/const/const.py +++ b/tools/cli/common/const/const.py @@ -12,8 +12,10 @@ ARGOCD_REGISTRY_APP_PATH = "gitops-pipelines/delivery/clusters/cc-cluster/core-services" KUBECTL_VERSION = "1.27.4" TERRAFORM_VERSION = "1.6.4" +GITOPS_REPOSITORY_MAIN_BRANCH = "main" WL_REPOSITORY_URL = "git@github.com:CloudGeometry/cg-devx-wl-template.git" WL_REPOSITORY_BRANCH = "main" WL_PR_BRANCH_NAME_PREFIX = "feature/" WL_GITOPS_REPOSITORY_URL = "git@github.com:CloudGeometry/cg-devx-wl-gitops-template.git" WL_GITOPS_REPOSITORY_BRANCH = "main" +WL_SERVICE_NAME = "default-service" diff --git a/tools/cli/common/utils/command_utils.py b/tools/cli/common/utils/command_utils.py index 160f3e5d..3c251a61 100644 --- a/tools/cli/common/utils/command_utils.py +++ b/tools/cli/common/utils/command_utils.py @@ -4,6 +4,7 @@ import webbrowser from logging import Logger from re import sub +from typing import Optional import click import requests @@ -158,7 +159,9 @@ def check_installation_presence(): raise click.ClickException("GitOps repo does not exist") -def initialize_gitops_repository(state_store: StateStore, logger: Logger) -> tuple: +def initialize_gitops_repository( + state_store: StateStore, logger: Logger +) -> tuple[GitProviderManager, PlatformGitOpsRepo]: """ Initialize and return the GitOps repository manager. @@ -217,7 +220,8 @@ def create_and_setup_branch(gor: PlatformGitOpsRepo, branch_name: str, logger: L def create_and_open_pull_request( gor: PlatformGitOpsRepo, state_store: StateStore, - wl_name: str, + title: str, + body: str, branch_name: str, main_branch: str, logger: Logger @@ -228,7 +232,8 @@ def create_and_open_pull_request( Parameters: gor: PlatformGitOpsRepo class instance. state_store (StateStore): State store instance for accessing configuration. - wl_name (str): Name of the workload. + title (str): PR title. + body (str): PR body. branch_name (str): The branch for which the pull request is created. main_branch (str): Main branch of the repository. logger (Logger): A logger instance for logging the process. @@ -236,10 +241,34 @@ def create_and_open_pull_request( try: pr_url = gor.create_pr( state_store.parameters[""], branch_name, main_branch, - f"Introduce {wl_name}", "Add default secrets, groups and default repository structure." + title, body ) webbrowser.open(pr_url, autoraise=False) logger.info(f"Pull request created: {pr_url}") except Exception as e: logger.error(f"Error in creating pull request: {e}") raise PullRequestCreationError(f"Could not create PR due to: {e}") + + +def preprocess_workload_names( + logger: Logger, + wl_name: str, wl_repo_name: Optional[str] = None, wl_gitops_repo_name: Optional[str] = None + ) -> tuple[str, str, str]: + """ + Process and normalize workload names to a standard format. + + Parameters: + wl_name (str): Name of the workload. + wl_repo_name (str): Name of the workload repository. + wl_gitops_repo_name (str): Name of the workload GitOps repository. + logger (Logger): A logger instance for logging the process. + + Returns: + tuple[str, str, str]: Tuple of processed workload name, workload repository name, and GitOps repository name. + """ + logger.debug(f"Processing workload names: {wl_name}, {wl_repo_name}, {wl_gitops_repo_name}") + wl_name = str_to_kebab(wl_name) + wl_repo_name = str_to_kebab(wl_repo_name or wl_name) + wl_gitops_repo_name = str_to_kebab(wl_gitops_repo_name or f"{wl_repo_name}-gitops") + logger.info(f"Processed names: {wl_name}, {wl_repo_name}, {wl_gitops_repo_name}") + return wl_name, wl_repo_name, wl_gitops_repo_name diff --git a/tools/cli/services/wl_template_manager.py b/tools/cli/services/wl_template_manager.py index 6db13e2c..14869e2d 100644 --- a/tools/cli/services/wl_template_manager.py +++ b/tools/cli/services/wl_template_manager.py @@ -141,6 +141,29 @@ def cleanup(self) -> None: logger.error(f"Error during cleanup process: {e}") raise + @trace() + def update(self): + if not self.wl_repo: + self.wl_repo = Repo(self.wl_repo_folder) + with self.wl_repo.git.custom_environment(GIT_SSH_COMMAND=f"ssh -o StrictHostKeyChecking=no -i {self.key_path}"): + # clean stale branches + self.wl_repo.remotes.origin.fetch(prune=True) + self.wl_repo.heads.main.checkout() + self.wl_repo.remotes.origin.pull(self.wl_repo.active_branch) + + @trace() + def branch_exist(self, branch_name): + return branch_name in self.wl_repo.branches + + @trace() + def create_branch(self, branch_name): + current = self.wl_repo.create_head(branch_name) + current.checkout() + + @trace() + def switch_to_branch(self, branch_name: str = "main"): + self.wl_repo.heads[branch_name].checkout() + def _replace_placeholders_in_folder(self, folder: Union[str, Path], params: Dict[str, str]) -> None: """ Replace placeholders in all eligible files within the specified folder. From 45db0e8d943560db76baea85282ab683bca996ff Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Mon, 8 Jan 2024 13:52:27 +0100 Subject: [PATCH 43/49] fix: mark oidc provider output sensitive --- platform/terraform/hosting_provider/output.tf | 1 + platform/terraform/modules/cloud_azure/outputs.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index f35b8d9c..970a4d5b 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -41,6 +41,7 @@ output "cluster_certificate_authority_data" { output "cluster_oidc_provider" { value = module.hosting-provider.cluster_oidc_provider description = "Cluster OIDC provider" + sensitive = true } # secret manager diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index 03ad8681..f3940385 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -59,6 +59,6 @@ output "artifacts_storage" { output "cluster_oidc_provider" { value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - sensitive = true description = "The OpenID Connect identity provider." + sensitive = true } \ No newline at end of file From 826bbae5b95ab64f9ee702009c439ef51b0711ef Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Mon, 8 Jan 2024 15:25:53 +0100 Subject: [PATCH 44/49] oidc arn refactor --- platform/terraform/hosting_provider/output.tf | 13 +++++++------ platform/terraform/modules/cloud_aws/outputs.tf | 2 +- platform/terraform/modules/cloud_azure/outputs.tf | 6 ------ tools/cli/commands/setup.py | 2 +- tools/cli/commands/workload/bootstrap.py | 12 ++++++++++-- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index 970a4d5b..e1ae6318 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -38,12 +38,6 @@ output "cluster_certificate_authority_data" { sensitive = true } -output "cluster_oidc_provider" { - value = module.hosting-provider.cluster_oidc_provider - description = "Cluster OIDC provider" - sensitive = true -} - # secret manager output "secret_manager_seal_key" { value = module.hosting-provider.secret_manager_unseal_key @@ -63,3 +57,10 @@ output "kube_config_raw" { sensitive = true description = "Contains the Kubernetes config to be used by kubectl and other compatible tools." } + +# Cluster OIDC provider ARN for AWS only: +output "cluster_oidc_provider_arn" { + value = module.hosting-provider.cluster_oidc_provider_arn + description = "Cluster OIDC provider" + sensitive = true +} \ No newline at end of file diff --git a/platform/terraform/modules/cloud_aws/outputs.tf b/platform/terraform/modules/cloud_aws/outputs.tf index fef4b175..25ca65db 100644 --- a/platform/terraform/modules/cloud_aws/outputs.tf +++ b/platform/terraform/modules/cloud_aws/outputs.tf @@ -158,7 +158,7 @@ output "cluster_oidc_provider_url" { value = module.eks.oidc_provider } -output "cluster_oidc_provider" { +output "cluster_oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = module.eks.oidc_provider_arn } diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index f3940385..f4a9f16a 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -56,9 +56,3 @@ output "artifacts_storage" { value = azurerm_storage_container.artifacts_repository.name description = "Continuous Integration Artifact Repository storage backend" } - -output "cluster_oidc_provider" { - value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - description = "The OpenID Connect identity provider." - sensitive = true -} \ No newline at end of file diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index da1298c4..c6d72f72 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -346,7 +346,6 @@ def setup( p.internals["CC_CLUSTER_ENDPOINT"] = hp_out["cluster_endpoint"] p.internals["CC_CLUSTER_CA_CERT_DATA"] = hp_out["cluster_certificate_authority_data"] p.internals["CC_CLUSTER_CA_CERT_PATH"] = write_ca_cert(hp_out["cluster_certificate_authority_data"]) - p.parameters[""] = hp_out["cluster_oidc_provider"] # artifact storage p.parameters[""] = hp_out["artifact_storage"] # kms keys @@ -373,6 +372,7 @@ def setup( "": p.parameters[""] } kctl_config_path = create_k8s_config(command, command_args, cloud_provider_auth_env_vars, kubeconfig_params) + p.parameters[""] = hp_out["cluster_oidc_provider_arn"] elif p.cloud_provider == CloudProviders.Azure: # user could get kubeconfig by running command # `az aks get-credentials --name my-cluster --resource-group my-rg --admin` diff --git a/tools/cli/commands/workload/bootstrap.py b/tools/cli/commands/workload/bootstrap.py index 7600a9df..e29767e0 100644 --- a/tools/cli/commands/workload/bootstrap.py +++ b/tools/cli/commands/workload/bootstrap.py @@ -137,7 +137,6 @@ def bootstrap( quay_proxy = state_store.parameters[""] git_runner_group_name = state_store.parameters[""] git_organisation_name = state_store.parameters[""] - cluster_oidc_issuer_url = state_store.parameters[""] cluster_name = state_store.parameters[""] click.echo("1/11: Configuration loaded.") @@ -200,9 +199,18 @@ def bootstrap( "": quay_proxy, "": git_runner_group_name, "": TERRAFORM_VERSION, - "": cluster_oidc_issuer_url, "": cluster_name, } + + # set cloud provider specific params + cloud_provider = state_store.parameters[""] + if cloud_provider == CloudProviders.AWS: + wl_gitops_params[""] = state_store.parameters[""] + elif cloud_provider == CloudProviders.Azure: + pass + else: + raise click.ClickException("Unknown cloud provider") + click.echo("4/11: Parameters for workload and GitOps repositories prepared.") # Initialize WorkloadManager for the workload repository From 985b590566c1bc657670af8babcfde80c87cf4eb Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Mon, 8 Jan 2024 20:34:49 +0100 Subject: [PATCH 45/49] feat: remove gitops repo branch name from wl commands input params --- tools/cli/commands/workload/create.py | 16 ++++------------ tools/cli/commands/workload/delete.py | 11 ++--------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/tools/cli/commands/workload/create.py b/tools/cli/commands/workload/create.py index 8d15f0b8..e7e8eb4d 100644 --- a/tools/cli/commands/workload/create.py +++ b/tools/cli/commands/workload/create.py @@ -26,20 +26,13 @@ 'wl_gitops_repo_name', help='Workload GitOps repository name', type=click.STRING ) -@click.option( - '--workload-gitops-main-branch-name', - '-wlgmbn', - 'main_branch', - default=GITOPS_REPOSITORY_MAIN_BRANCH, - help='Workload GitOps repository main branch name', type=click.STRING -) @click.option( '--verbosity', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], case_sensitive=False), default='CRITICAL', help='Set the verbosity level (DEBUG, INFO, WARNING, ERROR, CRITICAL)' ) -def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branch: str, verbosity: str) -> None: +def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, verbosity: str) -> None: """ Create workload boilerplate for GitOps. @@ -47,7 +40,6 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branc wl_name (str): Name of the workload. wl_repo_name (str): Name of the workload repository. wl_gitops_repo_name (str): Name of the workload GitOps repository. - main_branch (str): Main branch name of the GitOps repository. verbosity (str): Logging level. """ func_start_time = time.time() @@ -93,7 +85,7 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branc title=f"Introduce {wl_name}", body="Add default secrets, groups and default repository structure.", branch_name=branch_name, - main_branch=main_branch, + main_branch=GITOPS_REPOSITORY_MAIN_BRANCH, logger=logger ) except PullRequestCreationError as e: @@ -101,8 +93,8 @@ def create(wl_name: str, wl_repo_name: str, wl_gitops_repo_name: str, main_branc click.echo(f"6/7: Pull request for branch '{branch_name}' created and opened in the web browser.") - gor.switch_to_branch(branch_name=main_branch) - click.echo(f"7/7: Switched to branch '{main_branch}'.") + gor.switch_to_branch(branch_name=GITOPS_REPOSITORY_MAIN_BRANCH) + click.echo(f"7/7: Switched to branch '{GITOPS_REPOSITORY_MAIN_BRANCH}'.") click.echo(f"Workload GitOps code creation completed in {time.time() - func_start_time:.2f} seconds.") diff --git a/tools/cli/commands/workload/delete.py b/tools/cli/commands/workload/delete.py index 979c9f4c..43a282c1 100644 --- a/tools/cli/commands/workload/delete.py +++ b/tools/cli/commands/workload/delete.py @@ -36,13 +36,6 @@ is_flag=True, default=False ) -@click.option( - '--workload-gitops-main-branch-name', - '-wlgmbn', - 'main_branch', - default=GITOPS_REPOSITORY_MAIN_BRANCH, - help='Workload GitOps repository main branch name', type=click.STRING -) @click.option( '--workload-gitops-template-url', '-wlgu', @@ -67,7 +60,7 @@ ) def delete( wl_name: str, wl_gitops_repo_name: str, destroy_resources: bool, wl_gitops_template_url: str, - wl_gitops_template_branch: str, main_branch: str, verbosity: str + wl_gitops_template_branch: str, verbosity: str ): """ Deletes all the workload boilerplate, including optionally destroying associated resources. @@ -152,7 +145,7 @@ def delete( title=f"Remove {wl_name}", body="Remove default secrets, groups and repository structure.", branch_name=branch_name, - main_branch=main_branch, + main_branch=GITOPS_REPOSITORY_MAIN_BRANCH, logger=logger ) except PullRequestCreationError as e: From 31e80d70c96037e6264ca263291229c900dd9920 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Tue, 9 Jan 2024 12:48:01 +0100 Subject: [PATCH 46/49] feat: set allow empty for workloads-appset --- .../clusters/cc-cluster/core-services/200-wl-argocd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml index b8f37bdf..5a89e2a2 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/200-wl-argocd.yaml @@ -20,6 +20,7 @@ spec: automated: prune: true selfHeal: true + allowEmpty: true retry: limit: 5 backoff: From e0b14be155d4af9bcdea4f35f889b28b1c078fd2 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Tue, 9 Jan 2024 19:35:36 +0100 Subject: [PATCH 47/49] docs: add workloads quickstart --- QUICKSTART.md | 20 ++++++++---- WORKLOADS.md | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 WORKLOADS.md diff --git a/QUICKSTART.md b/QUICKSTART.md index 980c2d09..3399a320 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -21,7 +21,8 @@ - Azure - Supported - GCP - Will be added to a future release -\* Experimental functions are provided for you to try, but are not documented or supported, and are likely to be buggy, +> * Experimental functions are provided for you to try, but are not documented or supported, and are likely to +be buggy, or to change after release. ## Prerequisites @@ -36,7 +37,7 @@ You should have: make commits, manage users, and perform other tasks, such as executing Terraform scripts. For Github, you can create one following [this guide](https://docs.github.com/en/get-started/signing-up-for-github/signing-up-for-a-new-github-account). -3. A Github Organization. Organizations are used to group repositories, and CGDevX will create a new repo within a +3. A GitHub Organization. Organizations are used to group repositories, and CGDevX will create a new repo within a specific organization so that it's easy to find and manage later should you decide to stop using CGDevX. you don't have one, please @@ -118,17 +119,24 @@ Before deploying to Azure, ensure that you have: To set this up, you can follow [this guide](https://learn.microsoft.com/en-us/azure/dns/dns-delegate-domain-azure-dns). 4. A user account with `Owner` access. - You can use [this guide](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal-subscription-admin) + You can + use [this guide](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal-subscription-admin) to set it up, or [this guide](https://learn.microsoft.com/en-us/azure/role-based-access-control/quickstart-assign-role-user-portal) to grant permissions to an existing user. 5. The Azure CLI (**az**) and **[kubelogin](https://aka.ms/aks/kubelogin)** installed and configured to use this user. You can - use [this](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) and [this](https://azure.github.io/kubelogin/install.html) guides + use [this](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) + and [this](https://azure.github.io/kubelogin/install.html) guides to install the CLI. ## Installation process Once you have the prerequisites installed and configured, you are ready to install the CGDevX CLI. -Follow the instructions in the [CLI tool readme](tools/README.md). Once installed, you can find the -CLI commands [here](tools/cli/commands/README.md). +Follow the instructions in the [CLI tool readme](tools/README.md). + +Once installed the CLI, +you should be able to provision a cluster using [`setup`](tools/cli/commands/README.md#setup) command. +You can find the full list of the CLI commands [here](tools/cli/commands/README.md). + +Once you have a running cluster, you could create your first workload using this [guide](WORKLOADS.md). \ No newline at end of file diff --git a/WORKLOADS.md b/WORKLOADS.md new file mode 100644 index 00000000..1dcea663 --- /dev/null +++ b/WORKLOADS.md @@ -0,0 +1,84 @@ +# Workloads Quickstart Guide + +**Workload** is a high-level abstraction describing application(s) and/or service(s) that provides business value. +Workload is self-contained. + +Workloads are configured under CG DevX installation GitOps repository, and then managed by the team owning the Workload. + +CG DevX goal is +to provide isolation on Workload level through all the core services provided by CG DevX platform*. + +> * Experimental functions are provided for you to try, but are not documented or supported, and are likely +> to +> be buggy, or to change after release. + +## Prerequisites + +- Up and running CG DevX cluster. + Workloads are defined in CG DevX platform GitOps repository. + All the operations with workloads should be done from the same machine used CG DevX cluster provisioning, + as there is a hard dependency on the output of CG DevX setup flow. + +- There should be no other ongoing changes (active PRs) in CG DevX platform GitOps repository. This is required to avoid + inconsistency in infrastructure state. + +## Workloads management + +### Create + +Workload configuration is generated by `workload create` [command](tools/cli/commands/workload/README.md#create). +This will create a new feature branch in CG DevX platform GitOps repository, +push all the required configurations, and open a pull request (PR). + +You as a user should review a PR and apply the changes via PR automation solution +([Atlantis](https://www.runatlantis.io/)). +You could get more details on Atlantis commands [here](https://www.runatlantis.io/docs/using-atlantis.html). + +This will create new repositories for your Workload under your Git organizations. +By default, CG DevX will create two repositories, +one for Workload application(s)/service(s) source code, +and another for manifests + environment definitions, and Infrastructure as Code (IaC). +This is done to provide better isolation and access control, +and could be changed by updating Workload configuration for VCS module. +For instance, you may want to use one repo per application(s)/service(s) +instead of using monorepo that is CG DevX default behavior. + +You could delete Workload by running `workload delete` [command](tools/cli/commands/workload/README.md#delete). +This will reverse the changes done by `workload create` command, and open a PR to apply them. + +> **Note!**: You must create and delete workloads on by one to avoid conflicts. + +### Bootstrap + +When PR opened by `workload create` command is merged and closed, +you could bootstrap Workload repositories +with `workload bootstrap` [command](tools/cli/commands/workload/README.md#bootstrap) +This will provide the following features based on templates created by CG DevX team: + +- pre-defined folder structure +- environments definition (dev, sta, prod environments) +- IaC templates +- CI/CD process for Workload application(s)/service(s) +- IaC PR automation configuration + +> **Note!**: Reference implementation of delivery pipelines, repository structure, +> manifest and environments definitions are given as example of platform capabilities. +> They should and must be adjusted for your specific use case before production use. + +The following templates are used by default: + +- Workload template [repository](https://github.com/CloudGeometry/cg-devx-wl-template) +- Workload GitOps template [repository](https://github.com/CloudGeometry/cg-devx-wl-gitops-template) + +After uploading parametrized repo templates, `workload bootstrap` process will create a PR under your Workload GitOps +repository to initialize secrets, and create IAM role for the Workload service. + +Your new workload will have a pre-built CI process triggered by a tag applied to workload source code repository. +[Semantic versioning](https://semver.org/) is used. +CI will build an image (images when you have more than one service in your monorepo) +and upload them to the CG DevX image registry ([Harbor](https://goharbor.io/)). +After that, +it will update the image version of Workload service in K8s deployment definitions in Workload GitOps repository +to trigger CD process. +At this point you could promote changes from `dev` environment to other environments +by running promotion action under Workload GitOps repository. From a49fa5145351d070bd97d63cb29a4238dcd56b38 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Thu, 11 Jan 2024 18:55:58 +0100 Subject: [PATCH 48/49] fix: azure permissions for atlantis --- .../components/atlantis/application.yaml | 4 + .../components/vault/application.yaml | 6 +- platform/terraform/hosting_provider/main.tf | 14 ++-- platform/terraform/hosting_provider/output.tf | 2 +- .../terraform/hosting_provider/variables.tf | 2 +- platform/terraform/modules/cloud_aws/eks.tf | 2 +- .../terraform/modules/cloud_aws/variables.tf | 2 +- platform/terraform/modules/cloud_azure/aks.tf | 3 +- .../modules/cloud_azure/key_vault.tf | 11 ++- .../terraform/modules/cloud_azure/main.tf | 4 +- .../cloud_azure/modules/aks_rbac/main.tf | 36 ++++---- .../cloud_azure/modules/aks_rbac/outputs.tf | 4 +- .../cloud_azure/modules/aks_rbac/variables.tf | 5 ++ .../terraform/modules/cloud_azure/outputs.tf | 7 ++ .../modules/cloud_azure/service_accounts.tf | 64 ++++++++------- .../terraform/modules/cloud_azure/storage.tf | 5 +- .../modules/cloud_azure/variables.tf | 2 +- .../modules/secrets_vault/secrets.tf | 82 ++++++++++--------- .../modules/secrets_vault/variables.tf | 12 +++ platform/terraform/secrets/main.tf | 2 + platform/terraform/secrets/variable.tf | 12 +++ tools/cli/commands/destroy.py | 2 +- tools/cli/commands/setup.py | 24 ++++-- tools/cli/common/utils/command_utils.py | 4 +- tools/cli/services/cloud/aws/aws_manager.py | 5 ++ .../cli/services/cloud/azure/azure_manager.py | 41 ++++++---- tools/cli/services/cloud/azure/azure_sdk.py | 5 ++ .../services/cloud/cloud_provider_manager.py | 8 ++ 28 files changed, 233 insertions(+), 137 deletions(-) diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/atlantis/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/atlantis/application.yaml index b5ca3893..1b613951 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/atlantis/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/atlantis/application.yaml @@ -24,6 +24,10 @@ spec: mount: true annotations: : "" + podTemplate: + labels: { + # + } resources: limits: cpu: 400m diff --git a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/vault/application.yaml b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/vault/application.yaml index a1fd3ccb..db282f27 100644 --- a/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/vault/application.yaml +++ b/platform/gitops-pipelines/delivery/clusters/cc-cluster/core-services/components/vault/application.yaml @@ -18,10 +18,12 @@ spec: value: vault. values: |- server: - annotations: + annotations: { # - extraLabels: + } + extraLabels: { # + } serviceAccount: create: true name: vault diff --git a/platform/terraform/hosting_provider/main.tf b/platform/terraform/hosting_provider/main.tf index 48e197b7..3106228a 100644 --- a/platform/terraform/hosting_provider/main.tf +++ b/platform/terraform/hosting_provider/main.tf @@ -21,11 +21,11 @@ locals { module "hosting-provider" { - source = "../modules/cloud_" - cluster_name = local.cluster_name - region = local.region - alert_emails = local.email - ssh_public_key = var.ssh_public_key - tags = local.tags - cluster_node_labels = local.labels + source = "../modules/cloud_" + cluster_name = local.cluster_name + region = local.region + alert_emails = local.email + cluster_ssh_public_key = var.cluster_ssh_public_key + tags = local.tags + cluster_node_labels = local.labels } diff --git a/platform/terraform/hosting_provider/output.tf b/platform/terraform/hosting_provider/output.tf index e1ae6318..de01fed8 100644 --- a/platform/terraform/hosting_provider/output.tf +++ b/platform/terraform/hosting_provider/output.tf @@ -54,8 +54,8 @@ output "artifact_storage" { # Output part for Azure module only: output "kube_config_raw" { value = module.hosting-provider.kube_config_raw - sensitive = true description = "Contains the Kubernetes config to be used by kubectl and other compatible tools." + sensitive = true } # Cluster OIDC provider ARN for AWS only: diff --git a/platform/terraform/hosting_provider/variables.tf b/platform/terraform/hosting_provider/variables.tf index a6abedec..0bb379ed 100644 --- a/platform/terraform/hosting_provider/variables.tf +++ b/platform/terraform/hosting_provider/variables.tf @@ -1,4 +1,4 @@ -variable "ssh_public_key" { +variable "cluster_ssh_public_key" { description = "(Optional) SSH public key to access worker nodes." type = string default = "" diff --git a/platform/terraform/modules/cloud_aws/eks.tf b/platform/terraform/modules/cloud_aws/eks.tf index 4af81309..1e615df3 100644 --- a/platform/terraform/modules/cloud_aws/eks.tf +++ b/platform/terraform/modules/cloud_aws/eks.tf @@ -92,7 +92,7 @@ module "eks" { # key-pair for custom launch template #resource "aws_key_pair" "eks_nodes" { # key_name = "${local.name}-eks-nodes" -# public_key = var.ssh_public_key +# public_key = var.cluster_ssh_public_key #} # #resource "aws_launch_template" "eks_node_with_keypair" { diff --git a/platform/terraform/modules/cloud_aws/variables.tf b/platform/terraform/modules/cloud_aws/variables.tf index b370622d..eac77708 100644 --- a/platform/terraform/modules/cloud_aws/variables.tf +++ b/platform/terraform/modules/cloud_aws/variables.tf @@ -93,7 +93,7 @@ variable "alert_emails" { default = [] } -variable "ssh_public_key" { +variable "cluster_ssh_public_key" { description = "(Optional) SSH public key to access worker nodes." type = string default = "" diff --git a/platform/terraform/modules/cloud_azure/aks.tf b/platform/terraform/modules/cloud_azure/aks.tf index 35e5f695..4f620409 100644 --- a/platform/terraform/modules/cloud_azure/aks.tf +++ b/platform/terraform/modules/cloud_azure/aks.tf @@ -30,7 +30,6 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { name = local.default_node_group.name vm_size = local.default_node_group.instance_types[0] vnet_subnet_id = azurerm_subnet.private_subnet.id - # pod_subnet_id = [] ?? do we need it zones = local.azs node_labels = var.cluster_node_labels enable_auto_scaling = true @@ -46,7 +45,7 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { linux_profile { admin_username = local.node_admin_username ssh_key { - key_data = var.ssh_public_key + key_data = var.cluster_ssh_public_key } } diff --git a/platform/terraform/modules/cloud_azure/key_vault.tf b/platform/terraform/modules/cloud_azure/key_vault.tf index e27d78b1..b10ae486 100644 --- a/platform/terraform/modules/cloud_azure/key_vault.tf +++ b/platform/terraform/modules/cloud_azure/key_vault.tf @@ -21,12 +21,13 @@ resource "azurerm_key_vault" "key_vault" { default_action = "Deny" # still affected by issue https://github.com/hashicorp/terraform-provider-azurerm/issues/14783 virtual_network_subnet_ids = [azurerm_subnet.private_subnet.id] - ip_rules = [chomp(data.http.runner_ip_address.body)] + ip_rules = [chomp(data.http.runner_ip_address.response_body)] } lifecycle { ignore_changes = [ - tags + tags, + network_acls.0.ip_rules ] } @@ -46,6 +47,12 @@ resource "azurerm_role_assignment" "rbac_keyvault_administrator" { scope = azurerm_key_vault.key_vault.id role_definition_name = "Key Vault Administrator" principal_id = data.azurerm_client_config.client_identity.object_id + + lifecycle { + ignore_changes = [ + principal_id + ] + } } resource "random_string" "key_random_suffix" { diff --git a/platform/terraform/modules/cloud_azure/main.tf b/platform/terraform/modules/cloud_azure/main.tf index c952c4a5..ba48e1aa 100644 --- a/platform/terraform/modules/cloud_azure/main.tf +++ b/platform/terraform/modules/cloud_azure/main.tf @@ -2,11 +2,11 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.75" + version = "~>3.86" } random = { source = "hashicorp/random" - version = "3.5.1" + version = "~>3.5.1" } } } diff --git a/platform/terraform/modules/cloud_azure/modules/aks_rbac/main.tf b/platform/terraform/modules/cloud_azure/modules/aks_rbac/main.tf index e9d7271c..43d93991 100644 --- a/platform/terraform/modules/cloud_azure/modules/aks_rbac/main.tf +++ b/platform/terraform/modules/cloud_azure/modules/aks_rbac/main.tf @@ -6,37 +6,29 @@ terraform { } } -data "azurerm_client_config" "current_subscription" {} - -## Azure AD application that represents the app -resource "azuread_application" "appname" { - display_name = var.name -} - -resource "azuread_service_principal" "sp_name" { - application_id = azuread_application.appname.application_id +resource "azurerm_user_assigned_identity" "aks_workload_identity" { + location = var.resource_group_location + name = "${var.name}-aks-workload-identity" + resource_group_name = var.resource_group_name } -resource "azuread_service_principal_password" "asp_pass" { - service_principal_id = azuread_service_principal.sp_name.id -} +resource "azurerm_federated_identity_credential" "workload_identity_credentials" { + audience = ["api://AzureADTokenExchange"] + issuer = var.oidc_issuer_url + name = "${var.name}-workload-identity-credentials" + parent_id = azurerm_user_assigned_identity.aks_workload_identity.id + resource_group_name = var.resource_group_name + subject = "system:serviceaccount:${var.namespace}:${var.service_account_name}" -## Azure AD federated identity used to federate kubernetes with Azure AD -resource "azuread_application_federated_identity_credential" "aks-app-id" { - application_object_id = azuread_application.appname.object_id - display_name = "fed-identity-aks-id-${var.service_account_name}" - description = "The federated identity used to federate ${var.service_account_name} with Azure AD with the app service running in k8s in ${var.resource_group_name}" - audiences = ["api://AzureADTokenExchange"] - issuer = var.oidc_issuer_url - subject = "system:serviceaccount:${var.namespace}:${var.service_account_name}" + depends_on = [azurerm_user_assigned_identity.aks_workload_identity] } +data "azurerm_client_config" "current_subscription" {} -## Role assignment to the application resource "azurerm_role_assignment" "aks_rbac" { count = length(var.role_definitions) scope = "/subscriptions/${data.azurerm_client_config.current_subscription.subscription_id}${var.role_definitions[count.index].scope}" role_definition_name = var.role_definitions[count.index].name - principal_id = azuread_service_principal.sp_name.id + principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id } diff --git a/platform/terraform/modules/cloud_azure/modules/aks_rbac/outputs.tf b/platform/terraform/modules/cloud_azure/modules/aks_rbac/outputs.tf index df9015b4..4fa4bfb7 100644 --- a/platform/terraform/modules/cloud_azure/modules/aks_rbac/outputs.tf +++ b/platform/terraform/modules/cloud_azure/modules/aks_rbac/outputs.tf @@ -1,4 +1,4 @@ output "app_client_id" { - value = azuread_application.appname.application_id - description = "ID of current app client" + value = azurerm_user_assigned_identity.aks_workload_identity.client_id + description = "ID of client" } diff --git a/platform/terraform/modules/cloud_azure/modules/aks_rbac/variables.tf b/platform/terraform/modules/cloud_azure/modules/aks_rbac/variables.tf index 6d29fddb..ea86a5af 100644 --- a/platform/terraform/modules/cloud_azure/modules/aks_rbac/variables.tf +++ b/platform/terraform/modules/cloud_azure/modules/aks_rbac/variables.tf @@ -3,6 +3,11 @@ variable "resource_group_name" { type = string } +variable "resource_group_location" { + description = "(Required) Specifies the location of the resource group." + type = string +} + variable "name" { description = "(Required) Specifies the name of the resource group." type = string diff --git a/platform/terraform/modules/cloud_azure/outputs.tf b/platform/terraform/modules/cloud_azure/outputs.tf index f4a9f16a..6e0995b2 100644 --- a/platform/terraform/modules/cloud_azure/outputs.tf +++ b/platform/terraform/modules/cloud_azure/outputs.tf @@ -56,3 +56,10 @@ output "artifacts_storage" { value = azurerm_storage_container.artifacts_repository.name description = "Continuous Integration Artifact Repository storage backend" } + +# stub value for module compatibility +output "cluster_oidc_provider_arn" { + value = "" + sensitive = true + description = "Cluster OIDC provider stub." +} diff --git a/platform/terraform/modules/cloud_azure/service_accounts.tf b/platform/terraform/modules/cloud_azure/service_accounts.tf index 2038e51d..a8c451b4 100644 --- a/platform/terraform/modules/cloud_azure/service_accounts.tf +++ b/platform/terraform/modules/cloud_azure/service_accounts.tf @@ -1,12 +1,14 @@ module "iac_pr_automation_sa" { source = "./modules/aks_rbac" - oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - resource_group_name = azurerm_resource_group.rg.name - name = "atlantis" - service_account_name = "atlantis" - role_definitions = [ + oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + resource_group_name = azurerm_resource_group.rg.name + resource_group_location = azurerm_resource_group.rg.location + name = "atlantis" + service_account_name = "atlantis" + role_definitions = [ { "name" = "Contributor", "scope" = "" }, + { "name" = "User Access Administrator", "scope" = "" }, { "name" = "Key Vault Administrator", "scope" = "" } ] namespace = "atlantis" @@ -17,12 +19,13 @@ module "iac_pr_automation_sa" { module "ci_sa" { source = "./modules/aks_rbac" - oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - resource_group_name = azurerm_resource_group.rg.name - name = "argo-workflow" - service_account_name = "argo-server" - role_definitions = [{ "name" = "Contributor", "scope" = "" }] - namespace = "argo" + oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + resource_group_name = azurerm_resource_group.rg.name + resource_group_location = azurerm_resource_group.rg.location + name = "argo-workflow" + service_account_name = "argo-server" + role_definitions = [{ "name" = "Contributor", "scope" = "" }] + namespace = "argo" depends_on = [azurerm_kubernetes_cluster.aks_cluster] } @@ -30,12 +33,13 @@ module "ci_sa" { module "cert_manager_sa" { source = "./modules/aks_rbac" - oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - resource_group_name = azurerm_resource_group.rg.name - name = "cert-manager" - service_account_name = "cert-manager" - role_definitions = [{ "name" = "Contributor", "scope" = "" }] - namespace = "cert-manager" + oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + resource_group_name = azurerm_resource_group.rg.name + resource_group_location = azurerm_resource_group.rg.location + name = "cert-manager" + service_account_name = "cert-manager" + role_definitions = [{ "name" = "Contributor", "scope" = "" }] + namespace = "cert-manager" depends_on = [azurerm_kubernetes_cluster.aks_cluster] } @@ -43,12 +47,13 @@ module "cert_manager_sa" { module "external_dns_sa" { source = "./modules/aks_rbac" - oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - resource_group_name = azurerm_resource_group.rg.name - name = "external-dns" - service_account_name = "external-dns" - role_definitions = [{ "name" = "Contributor", "scope" = "" }] - namespace = "external-dns" + oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + resource_group_name = azurerm_resource_group.rg.name + resource_group_location = azurerm_resource_group.rg.location + name = "external-dns" + service_account_name = "external-dns" + role_definitions = [{ "name" = "Contributor", "scope" = "" }] + namespace = "external-dns" depends_on = [azurerm_kubernetes_cluster.aks_cluster] } @@ -56,12 +61,13 @@ module "external_dns_sa" { module "secret_manager_sa" { source = "./modules/aks_rbac" - oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - resource_group_name = azurerm_resource_group.rg.name - name = "vault" - service_account_name = "vault" - role_definitions = [{ "name" = "Key Vault Administrator", "scope" = "" }] - namespace = "vault" + oidc_issuer_url = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + resource_group_name = azurerm_resource_group.rg.name + resource_group_location = azurerm_resource_group.rg.location + name = "vault" + service_account_name = "vault" + role_definitions = [{ "name" = "Key Vault Administrator", "scope" = "" }] + namespace = "vault" depends_on = [azurerm_kubernetes_cluster.aks_cluster] } \ No newline at end of file diff --git a/platform/terraform/modules/cloud_azure/storage.tf b/platform/terraform/modules/cloud_azure/storage.tf index 2567b1cb..605bb0c9 100644 --- a/platform/terraform/modules/cloud_azure/storage.tf +++ b/platform/terraform/modules/cloud_azure/storage.tf @@ -20,7 +20,7 @@ resource "azurerm_storage_account" "storage_account" { bypass = ["AzureServices"] default_action = "Deny" virtual_network_subnet_ids = [azurerm_subnet.private_subnet.id] - ip_rules = [chomp(data.http.runner_ip_address.body)] + ip_rules = [chomp(data.http.runner_ip_address.response_body)] } identity { @@ -29,7 +29,8 @@ resource "azurerm_storage_account" "storage_account" { lifecycle { ignore_changes = [ - tags + tags, + network_rules.0.ip_rules ] } } diff --git a/platform/terraform/modules/cloud_azure/variables.tf b/platform/terraform/modules/cloud_azure/variables.tf index 4e20c230..f9a6fe36 100644 --- a/platform/terraform/modules/cloud_azure/variables.tf +++ b/platform/terraform/modules/cloud_azure/variables.tf @@ -80,7 +80,7 @@ variable "alert_emails" { default = [] } -variable "ssh_public_key" { +variable "cluster_ssh_public_key" { description = "(Required) Specifies the SSH public key for AKS worker nodes." type = string default = "" diff --git a/platform/terraform/modules/secrets_vault/secrets.tf b/platform/terraform/modules/secrets_vault/secrets.tf index a2054bad..9b92747a 100644 --- a/platform/terraform/modules/secrets_vault/secrets.tf +++ b/platform/terraform/modules/secrets_vault/secrets.tf @@ -51,30 +51,32 @@ resource "vault_generic_secret" "atlantis_secrets" { data_json = jsonencode( { - ARGO_SERVER_URL = "argo.argo.svc.cluster.local:2746", + ARGO_SERVER_URL = "argo.argo.svc.cluster.local:2746", # github specific section - ATLANTIS_GH_HOSTNAME = "github.com", - ATLANTIS_GH_TOKEN = var.vcs_token, - ATLANTIS_GH_USER = "", - ATLANTIS_GH_WEBHOOK_SECRET = var.atlantis_repo_webhook_secret, - GITHUB_OWNER = "", - GITHUB_TOKEN = var.vcs_token, + ATLANTIS_GH_HOSTNAME = "github.com", + ATLANTIS_GH_TOKEN = var.vcs_token, + ATLANTIS_GH_USER = "", + ATLANTIS_GH_WEBHOOK_SECRET = var.atlantis_repo_webhook_secret, + GITHUB_OWNER = "", + GITHUB_TOKEN = var.vcs_token, # ---- - TF_VAR_atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret, - TF_VAR_atlantis_repo_webhook_url = var.atlantis_repo_webhook_url, - TF_VAR_vcs_token = var.vcs_token, - TF_VAR_cluster_endpoint = var.cluster_endpoint - # ---- - TF_VAR_hosted_zone_name = "", - TF_VAR_vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key, - TF_VAR_vcs_bot_ssh_private_key = var.vcs_bot_ssh_private_key, + TF_VAR_atlantis_repo_webhook_secret = var.atlantis_repo_webhook_secret, + TF_VAR_atlantis_repo_webhook_url = var.atlantis_repo_webhook_url, + TF_VAR_vcs_token = var.vcs_token, + TF_VAR_cluster_endpoint = var.cluster_endpoint, + TF_VAR_tf_backend_storage_access_key = var.tf_backend_storage_access_key, + TF_VAR_cluster_ssh_public_key = var.cluster_ssh_public_key, + # + TF_VAR_hosted_zone_name = "", + TF_VAR_vcs_bot_ssh_public_key = var.vcs_bot_ssh_public_key, + TF_VAR_vcs_bot_ssh_private_key = var.vcs_bot_ssh_private_key, # harbor specific section - TF_VAR_registry_oidc_client_id = module.harbor.vault_oidc_client_id, - TF_VAR_registry_oidc_client_secret = module.harbor.vault_oidc_client_secret, - TF_VAR_registry_main_robot_password = random_password.harbor_main_robot_password.result, - HARBOR_URL = "https://", - HARBOR_USERNAME = local.harbor_admin_user, - HARBOR_PASSWORD = random_password.harbor_password.result, + TF_VAR_registry_oidc_client_id = module.harbor.vault_oidc_client_id, + TF_VAR_registry_oidc_client_secret = module.harbor.vault_oidc_client_secret, + TF_VAR_registry_main_robot_password = random_password.harbor_main_robot_password.result, + HARBOR_URL = "https://", + HARBOR_USERNAME = local.harbor_admin_user, + HARBOR_PASSWORD = random_password.harbor_password.result, # ---- # vault specific section @@ -96,9 +98,9 @@ resource "random_password" "grafana_password" { length = 22 special = true override_special = "!#$" - min_lower = 1 - min_upper = 1 - min_numeric = 1 + min_lower = 1 + min_upper = 1 + min_numeric = 1 } resource "vault_generic_secret" "grafana_secrets" { @@ -119,9 +121,9 @@ resource "random_password" "atlantis_password" { length = 22 special = true override_special = "!#$" - min_lower = 1 - min_upper = 1 - min_numeric = 1 + min_lower = 1 + min_upper = 1 + min_numeric = 1 } resource "vault_generic_secret" "atlantis_auth_secrets" { @@ -139,10 +141,10 @@ resource "vault_generic_secret" "atlantis_auth_secrets" { # harbor web ui admin auth credentials resource "random_password" "harbor_password" { - length = 22 - special = false - min_lower = 1 - min_upper = 1 + length = 22 + special = false + min_lower = 1 + min_upper = 1 min_numeric = 1 } @@ -160,10 +162,10 @@ resource "vault_generic_secret" "harbor_admin_secret" { } resource "random_password" "harbor_main_robot_password" { - length = 22 - special = false - min_lower = 1 - min_upper = 1 + length = 22 + special = false + min_lower = 1 + min_upper = 1 min_numeric = 1 } @@ -182,10 +184,10 @@ resource "vault_generic_secret" "harbor_main_robot_secret" { } resource "random_password" "sonarqube_password" { - length = 22 - special = false - min_lower = 1 - min_upper = 1 + length = 22 + special = false + min_lower = 1 + min_upper = 1 min_numeric = 1 } @@ -213,7 +215,7 @@ resource "vault_generic_secret" "oauth2_cookie_secret" { data_json = jsonencode( { - backstage_cookie_secret = random_password.oauth2_backstage_cookie_password.result, + backstage_cookie_secret = random_password.oauth2_backstage_cookie_password.result, } ) diff --git a/platform/terraform/modules/secrets_vault/variables.tf b/platform/terraform/modules/secrets_vault/variables.tf index 7bc42167..879b6768 100644 --- a/platform/terraform/modules/secrets_vault/variables.tf +++ b/platform/terraform/modules/secrets_vault/variables.tf @@ -57,3 +57,15 @@ variable "cluster_endpoint" { description = "(Required) K8s cluster endpoint" type = string } + +variable "cluster_ssh_public_key" { + description = "Specifies the SSH public key for K8s worker nodes. Only applicable to AKS." + type = string + default = null +} + +variable "tf_backend_storage_access_key" { + description = "Specifies the access key for tf backend storage. Only applicable to AKS." + type = string + default = "" +} diff --git a/platform/terraform/secrets/main.tf b/platform/terraform/secrets/main.tf index 51da261f..290481b5 100644 --- a/platform/terraform/secrets/main.tf +++ b/platform/terraform/secrets/main.tf @@ -31,4 +31,6 @@ module "secrets" { atlantis_repo_webhook_url = var.atlantis_repo_webhook_url vault_token = var.vault_token cluster_endpoint = var.cluster_endpoint + cluster_ssh_public_key = var.cluster_ssh_public_key + tf_backend_storage_access_key = var.tf_backend_storage_access_key } diff --git a/platform/terraform/secrets/variable.tf b/platform/terraform/secrets/variable.tf index d798ff4e..9f9fc56c 100644 --- a/platform/terraform/secrets/variable.tf +++ b/platform/terraform/secrets/variable.tf @@ -50,3 +50,15 @@ variable "workloads" { })) default = {} } + +variable "cluster_ssh_public_key" { + description = "Specifies the SSH public key for K8s worker nodes. Only applicable to AKS." + type = string + default = "" +} + +variable "tf_backend_storage_access_key" { + description = "Specifies the access key for tf backend storage. Only applicable to AKS." + type = string + default = "" +} diff --git a/tools/cli/commands/destroy.py b/tools/cli/commands/destroy.py index cf0fc039..e0ac37f1 100644 --- a/tools/cli/commands/destroy.py +++ b/tools/cli/commands/destroy.py @@ -112,7 +112,7 @@ def destroy(verbosity: str): tf_wrapper = TfWrapper(LOCAL_TF_FOLDER_HOSTING_PROVIDER) tf_wrapper.init() - tf_wrapper.destroy({"ssh_public_key": p.parameters.get("", "")}) + tf_wrapper.destroy({"cluster_ssh_public_key": p.parameters.get("", "")}) click.echo("Destroying K8s cluster. Done!") diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index c6d72f72..243835ab 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -226,7 +226,8 @@ def setup( # create terraform storage backend click.echo("Creating tf backend storage...") - tf_backend_storage = cloud_man.create_iac_state_storage(p.get_input_param(GITOPS_REPOSITORY_NAME)) + tf_backend_storage, key = cloud_man.create_iac_state_storage(p.get_input_param(GITOPS_REPOSITORY_NAME)) + p.internals["TF_BACKEND_STORAGE_ACCESS_KEY"] = key p.internals["TF_BACKEND_STORAGE_NAME"] = tf_backend_storage p.fragments["# "] = cloud_man.create_iac_backend_snippet(tf_backend_storage, @@ -242,12 +243,16 @@ def setup( p.fragments["# "] = cloud_man.create_hosting_provider_snippet() + p.fragments["# "] = cloud_man.create_iac_pr_automation_config_snippet() + p.parameters[""] = cloud_man.create_k8s_cluster_role_mapping_snippet() p.fragments["# "] = cloud_man.create_additional_labels() + p.fragments["# "] = cloud_man.create_additional_labels() p.fragments["# "] = cloud_man.create_ingress_annotations() p.fragments["# "] = cloud_man.create_sidecar_annotation() + # dns zone info for external dns dns_zone_name, is_dns_zone_private = dns_man.get_domain_zone(p.parameters[""]) p.internals["DNS_ZONE_NAME"] = dns_zone_name @@ -330,7 +335,7 @@ def setup( tf_wrapper = TfWrapper(LOCAL_TF_FOLDER_HOSTING_PROVIDER) tf_wrapper.init() - tf_wrapper.apply({"ssh_public_key": p.parameters.get("", "")}) + tf_wrapper.apply({"cluster_ssh_public_key": p.parameters.get("", "")}) hp_out = tf_wrapper.output() # store out params @@ -649,15 +654,24 @@ def setup( tf_wrapper = TfWrapper(LOCAL_TF_FOLDER_SECRETS_MANAGER) tf_wrapper.init() - tf_wrapper.apply({ + + sec_man_tf_params = { "vcs_bot_ssh_public_key": p.internals["DEFAULT_SSH_PUBLIC_KEY"], "vcs_bot_ssh_private_key": p.internals["DEFAULT_SSH_PRIVATE_KEY"], "vcs_token": p.internals["GIT_ACCESS_TOKEN"], "atlantis_repo_webhook_secret": p.parameters[""], "atlantis_repo_webhook_url": p.parameters[""], "vault_token": p.internals["VAULT_ROOT_TOKEN"], - "cluster_endpoint": p.internals["CC_CLUSTER_ENDPOINT"] - }) + "cluster_endpoint": p.internals["CC_CLUSTER_ENDPOINT"], + } + if "" in p.parameters: + sec_man_tf_params["cluster_ssh_public_key"] = p.parameters[""] + + if "TF_BACKEND_STORAGE_ACCESS_KEY" in p.internals: + sec_man_tf_params["tf_backend_storage_access_key"] = p.internals["TF_BACKEND_STORAGE_ACCESS_KEY"] + + tf_wrapper.apply(sec_man_tf_params) + sec_man_out = tf_wrapper.output() p.internals["REGISTRY_OIDC_CLIENT_ID"] = sec_man_out["registry_oidc_client_id"] p.internals["REGISTRY_OIDC_CLIENT_SECRET"] = sec_man_out["registry_oidc_client_secret"] diff --git a/tools/cli/common/utils/command_utils.py b/tools/cli/common/utils/command_utils.py index 3c251a61..72885415 100644 --- a/tools/cli/common/utils/command_utils.py +++ b/tools/cli/common/utils/command_utils.py @@ -67,10 +67,12 @@ def init_cloud_provider(state: StateStore) -> tuple[CloudProviderManager, DNSMan if not AzureManager.detect_cli_presence(): raise click.ClickException("Cloud CLI is missing") + subscription_id = state.get_input_param(CLOUD_PROFILE) cloud_manager: AzureManager = AzureManager( - state.get_input_param(CLOUD_PROFILE), state.get_input_param(CLOUD_REGION) + subscription_id, state.get_input_param(CLOUD_REGION) ) domain_manager: DNSManager = AzureDNSManager(state.get_input_param(CLOUD_PROFILE)) + state.parameters[""] = subscription_id return cloud_manager, domain_manager diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index cb1f4a61..e330d388 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -157,3 +157,8 @@ def create_sidecar_annotation(self) -> str: @trace() def create_external_secrets_config(self, **kwargs) -> str: return "" + + @trace() + def create_iac_pr_automation_config_snippet(self): + return '''# aws specific section + # ----''' diff --git a/tools/cli/services/cloud/azure/azure_manager.py b/tools/cli/services/cloud/azure/azure_manager.py index 89eeb073..a94224f4 100644 --- a/tools/cli/services/cloud/azure/azure_manager.py +++ b/tools/cli/services/cloud/azure/azure_manager.py @@ -18,11 +18,11 @@ class AzureManager(CloudProviderManager): def __init__(self, subscription_id: str, location: Optional[str] = None): self.iac_backend_storage_container_name: Optional[str] = None - self.__azure_sdk = AzureSdk(subscription_id, location) + self._azure_sdk = AzureSdk(subscription_id, location) @property def region(self): - return self.__azure_sdk.location + return self._azure_sdk.location @trace() def protect_iac_state_storage(self, name: str, identity: str): @@ -32,7 +32,7 @@ def protect_iac_state_storage(self, name: str, identity: str): self.iac_backend_storage_container_name = name resource_group_name = self._generate_resource_group_name() storage_account_name = self._generate_storage_account_name() - self.__azure_sdk.set_storage_access(identity, storage_account_name, resource_group_name) + self._azure_sdk.set_storage_access(identity, storage_account_name, resource_group_name) @trace() def destroy_iac_state_storage(self, bucket: str) -> bool: @@ -51,7 +51,7 @@ def destroy_iac_state_storage(self, bucket: str) -> bool: bool: True if the resource group was successfully destroyed, False otherwise. """ self.iac_backend_storage_container_name = bucket - return self.__azure_sdk.destroy_resource_group(self._generate_resource_group_name()) + return self._azure_sdk.destroy_resource_group(self._generate_resource_group_name()) @trace() def create_iac_backend_snippet(self, location: str, service: str, **kwargs) -> str: @@ -124,12 +124,13 @@ def create_iac_state_storage(self, name: str, **kwargs: dict) -> str: resource_group_name = self._generate_resource_group_name() storage_account_name = self._generate_storage_account_name() - storage = self.__azure_sdk.create_storage(self.iac_backend_storage_container_name, - storage_account_name, - resource_group_name) - self.__azure_sdk.set_storage_account_versioning(storage_account_name, resource_group_name) + storage = self._azure_sdk.create_storage(self.iac_backend_storage_container_name, + storage_account_name, + resource_group_name) + keys = self._azure_sdk.get_storage_account_keys(resource_group_name, storage_account_name) + self._azure_sdk.set_storage_account_versioning(storage_account_name, resource_group_name) - return storage + return storage, keys[0].value @trace() def evaluate_permissions(self) -> bool: @@ -138,10 +139,10 @@ def evaluate_permissions(self) -> bool: :return: True or False """ missing_permissions = [] - missing_permissions.extend(self.__azure_sdk.blocked(aks_permissions)) - missing_permissions.extend(self.__azure_sdk.blocked(blob_permissions)) - missing_permissions.extend(self.__azure_sdk.blocked(vnet_permissions)) - missing_permissions.extend(self.__azure_sdk.blocked(rbac_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(aks_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(blob_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(vnet_permissions)) + missing_permissions.extend(self._azure_sdk.blocked(rbac_permissions)) return len(missing_permissions) == 0 @staticmethod @@ -185,7 +186,7 @@ def create_ingress_annotations(self) -> str: @trace() def create_additional_labels(self) -> str: - return 'azure.workload.identity/use: "true"' + return "" @trace() def create_sidecar_annotation(self) -> str: @@ -209,4 +210,14 @@ def create_external_secrets_config(self, **kwargs) -> str: "subscriptionId": "{subscription_id}", "resourceGroup": "{resource_group}", "useWorkloadIdentityExtension": true - }}'''.format(subscription_id=self.__azure_sdk.subscription_id, resource_group=location) + }}'''.format(subscription_id=self._azure_sdk.subscription_id, resource_group=location) + + @trace() + def create_iac_pr_automation_config_snippet(self): + return '''# azure specific section + ARM_USE_AKS_WORKLOAD_IDENTITY = true, + ARM_USE_CLI = false, + ARM_SUBSCRIPTION_ID = "", + ARM_CLIENT_ID = "", + ARM_ACCESS_KEY = var.tf_backend_storage_access_key, + # ----''' diff --git a/tools/cli/services/cloud/azure/azure_sdk.py b/tools/cli/services/cloud/azure/azure_sdk.py index 7825ffdf..443f0fc6 100644 --- a/tools/cli/services/cloud/azure/azure_sdk.py +++ b/tools/cli/services/cloud/azure/azure_sdk.py @@ -377,6 +377,10 @@ def create_storage_account(self, resource_group_name: str, storage_account_name: except ResourceExistsError: logger.warning(f"Storage name {storage_account_name} is already in use. Try another name.") + def get_storage_account_keys(self, resource_group_name: str, storage_account_name: str): + r = self.storage_mgmt_client.storage_accounts.list_keys(resource_group_name, storage_account_name) + return r.keys + def set_storage_account_versioning(self, storage_account_name: str, resource_group_name: str) -> None: """ Set a storage account data protection options. @@ -443,6 +447,7 @@ def create_storage(self, container_name: str, storage_account_name: str, resourc self.create_resource_group(resource_group_name) self.create_storage_account(resource_group_name, storage_account_name) self.create_blob_container(storage_account_name, container_name) + return container_name def _validate_location(self, location: Optional[str]) -> str: diff --git a/tools/cli/services/cloud/cloud_provider_manager.py b/tools/cli/services/cloud/cloud_provider_manager.py index 5402ec71..e5eb9a25 100644 --- a/tools/cli/services/cloud/cloud_provider_manager.py +++ b/tools/cli/services/cloud/cloud_provider_manager.py @@ -126,3 +126,11 @@ def create_external_secrets_config(self, **kwargs) -> str: :return: External secrets operator configuration """ pass + + @abstractmethod + def create_iac_pr_automation_config_snippet(self): + """ + Creates Cloud Provider specific configuration section for Atlantis + :return: Atlantis configuration section + """ + pass From 774f002868d25bec07794c48c230bfdc27e51d24 Mon Sep 17 00:00:00 2001 From: Alex Ulyanov Date: Fri, 12 Jan 2024 12:02:52 +0100 Subject: [PATCH 49/49] fix: aws compatibility for tf iac store --- tools/cli/commands/setup.py | 1 - tools/cli/services/cloud/aws/aws_manager.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/cli/commands/setup.py b/tools/cli/commands/setup.py index 243835ab..b490c6af 100644 --- a/tools/cli/commands/setup.py +++ b/tools/cli/commands/setup.py @@ -252,7 +252,6 @@ def setup( p.fragments["# "] = cloud_man.create_ingress_annotations() p.fragments["# "] = cloud_man.create_sidecar_annotation() - # dns zone info for external dns dns_zone_name, is_dns_zone_private = dns_man.get_domain_zone(p.parameters[""]) p.internals["DNS_ZONE_NAME"] = dns_zone_name diff --git a/tools/cli/services/cloud/aws/aws_manager.py b/tools/cli/services/cloud/aws/aws_manager.py index e330d388..a3e237b6 100644 --- a/tools/cli/services/cloud/aws/aws_manager.py +++ b/tools/cli/services/cloud/aws/aws_manager.py @@ -40,7 +40,7 @@ def create_iac_state_storage(self, name: str, **kwargs: dict) -> str: self.__aws_sdk.create_bucket(tf_backend_storage_name, region) self.__aws_sdk.enable_bucket_versioning(tf_backend_storage_name, region) - return tf_backend_storage_name + return tf_backend_storage_name, "" @trace() def protect_iac_state_storage(self, name: str, identity: str, **kwargs: dict):