From a985870ef3f0ccb25491ddc88765b91628a394c2 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 17:57:00 -0500 Subject: [PATCH 01/40] docs: Get the project started --- pulumi/github/repos/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 pulumi/github/repos/README.md diff --git a/pulumi/github/repos/README.md b/pulumi/github/repos/README.md new file mode 100644 index 0000000..e31936a --- /dev/null +++ b/pulumi/github/repos/README.md @@ -0,0 +1,18 @@ +# Repos + +Goal is to replace https://oldsite.nf-co.re/pipeline_health + +## Initial Roll-out + +The new pipelines that are broken: + +- denovotranscript +- meerpipe +- pairgenomealign +- phaseimpute +- reportho + +Maybe: + +- scdownstream +- scnanoseq From c39226a041b6006ff9a86e4bc491f216946485e0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 17:59:46 -0500 Subject: [PATCH 02/40] chore: Copy over code from teams --- pulumi/github/repos/.gitignore | 3 +++ pulumi/github/repos/Pulumi.dev.yaml | 4 ++++ pulumi/github/repos/Pulumi.yaml | 10 ++++++++++ pulumi/github/repos/__main__.py | 6 ++++++ pulumi/github/repos/requirements.txt | 3 +++ pulumi/github/teams/.gitignore | 5 ----- 6 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 pulumi/github/repos/.gitignore create mode 100644 pulumi/github/repos/Pulumi.dev.yaml create mode 100644 pulumi/github/repos/Pulumi.yaml create mode 100644 pulumi/github/repos/__main__.py create mode 100644 pulumi/github/repos/requirements.txt diff --git a/pulumi/github/repos/.gitignore b/pulumi/github/repos/.gitignore new file mode 100644 index 0000000..18fa2a3 --- /dev/null +++ b/pulumi/github/repos/.gitignore @@ -0,0 +1,3 @@ +*.pyc +venv/ +__pycache__/ diff --git a/pulumi/github/repos/Pulumi.dev.yaml b/pulumi/github/repos/Pulumi.dev.yaml new file mode 100644 index 0000000..f388f6e --- /dev/null +++ b/pulumi/github/repos/Pulumi.dev.yaml @@ -0,0 +1,4 @@ +config: + github:owner: nf-core-tf + github:token: + secure: AAABADQ5983Zkr3Cb5e3Ql44AV0OkR66r4aU1seWGmEhzkgBnSlL3WfSk+qcXrFPdelbLM05rnd0thzVjSWbaR5B0Kor/GUFgvWxoUDlXomH/mFpizDV9QsqBgSNRrKlYgAxt9n4SwQ1j0aH8MpDPLyetyhLHH/cJrI33BA= diff --git a/pulumi/github/repos/Pulumi.yaml b/pulumi/github/repos/Pulumi.yaml new file mode 100644 index 0000000..d492c4a --- /dev/null +++ b/pulumi/github/repos/Pulumi.yaml @@ -0,0 +1,10 @@ +name: github-repos +runtime: + name: python + options: + virtualenv: venv +description: Managing GitHub repos +config: + pulumi:tags: + value: + pulumi:template: https://www.pulumi.com/ai/api/project/3cb51e5f-2548-4d7b-9d9d-1ea680ac96ee.zip diff --git a/pulumi/github/repos/__main__.py b/pulumi/github/repos/__main__.py new file mode 100644 index 0000000..bd11f66 --- /dev/null +++ b/pulumi/github/repos/__main__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import yaml + +import pulumi +import pulumi_github as github diff --git a/pulumi/github/repos/requirements.txt b/pulumi/github/repos/requirements.txt new file mode 100644 index 0000000..d265c6c --- /dev/null +++ b/pulumi/github/repos/requirements.txt @@ -0,0 +1,3 @@ +pulumi>=3 +pulumi_github>=5.20.0 +ruff>=0.3.7 diff --git a/pulumi/github/teams/.gitignore b/pulumi/github/teams/.gitignore index 75384ec..18fa2a3 100644 --- a/pulumi/github/teams/.gitignore +++ b/pulumi/github/teams/.gitignore @@ -1,8 +1,3 @@ *.pyc venv/ __pycache__/ - -# sensitive data -Pulumi*yaml -*.txt -!requirements.txt From 5a60259718b7cb85931282f3f548967dcca073db Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 18:07:48 -0500 Subject: [PATCH 03/40] chore: re-encrypt GitHub token --- pulumi/github/repos/Pulumi.dev.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulumi/github/repos/Pulumi.dev.yaml b/pulumi/github/repos/Pulumi.dev.yaml index f388f6e..b222191 100644 --- a/pulumi/github/repos/Pulumi.dev.yaml +++ b/pulumi/github/repos/Pulumi.dev.yaml @@ -1,4 +1,5 @@ config: github:owner: nf-core-tf + # https://start.1password.com/open/i?a=O5GICFDKPNABLLVGMKBL5JWDWA&v=rdfcz6oy6qxxrc4clu467a7dmm&i=4ajrv44kc5lcbboa37fr5oydla&h=nf-core.1password.eu github:token: - secure: AAABADQ5983Zkr3Cb5e3Ql44AV0OkR66r4aU1seWGmEhzkgBnSlL3WfSk+qcXrFPdelbLM05rnd0thzVjSWbaR5B0Kor/GUFgvWxoUDlXomH/mFpizDV9QsqBgSNRrKlYgAxt9n4SwQ1j0aH8MpDPLyetyhLHH/cJrI33BA= + secure: AAABAFMgBNyCNuYsps6YVPV2L7Ji5qBJj0omEQQa9HrdhT2iHo3ex0e9NsDER3Q04itGiY698X/ZQCnTM2zu9op3tcjmzfITdHxGy0FGATuUFamYsSiztHrNAKiIEJ9E0M4Al8/yJeB6X4BXvkLEgik/I+GPvZIXK3tE65Q= From 77afb52bd386a336608e3edbd9ea7d9949b6b9b4 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 18:09:21 -0500 Subject: [PATCH 04/40] chore: pulumi import github:index/repository:Repository nf-core-tf modules --- pulumi/github/repos/__main__.py | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pulumi/github/repos/__main__.py b/pulumi/github/repos/__main__.py index bd11f66..103aade 100644 --- a/pulumi/github/repos/__main__.py +++ b/pulumi/github/repos/__main__.py @@ -4,3 +4,41 @@ import pulumi import pulumi_github as github + + +nf_core_tf = github.Repository( + "nf-core-tf", + allow_merge_commit=False, + allow_rebase_merge=False, + allow_squash_merge=False, + default_branch="master", + description="Repository to host tool-specific module files for the Nextflow DSL2 community!", + has_downloads=True, + has_issues=True, + has_projects=True, + homepage_url="https://nf-co.re", + merge_commit_message="", + merge_commit_title="", + name="modules", + security_and_analysis=github.RepositorySecurityAndAnalysisArgs( + secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( + status="disabled", + ), + secret_scanning_push_protection=github.RepositorySecurityAndAnalysisSecretScanningPushProtectionArgs( + status="disabled", + ), + ), + squash_merge_commit_message="", + squash_merge_commit_title="", + topics=[ + "nextflow", + "pipelines", + "nf-test", + "modules", + "nf-core", + "dsl2", + "workflows", + ], + visibility="public", + opts=pulumi.ResourceOptions(protect=True), +) From d43aa80fc121bb4c52bffb488e9abeb242164837 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 18:47:48 -0500 Subject: [PATCH 05/40] docs: Write up some plans --- pulumi/github/repos/README.md | 16 ++++++++++++++++ pulumi/github/repos/core_repos.yml | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 pulumi/github/repos/core_repos.yml diff --git a/pulumi/github/repos/README.md b/pulumi/github/repos/README.md index e31936a..6466569 100644 --- a/pulumi/github/repos/README.md +++ b/pulumi/github/repos/README.md @@ -2,6 +2,8 @@ Goal is to replace https://oldsite.nf-co.re/pipeline_health +This repo will be the "Actions" section at the bottom. We can then create a reporting page if we really need to see all the green checks + ## Initial Roll-out The new pipelines that are broken: @@ -16,3 +18,17 @@ Maybe: - scdownstream - scnanoseq + +### Plan + +#### Short-term + +1. [ ] Import a pipeline that has all the right settings +2. [ ] Fix the 5 pipelines above with the correct settings from the "model" repo +3. [ ] Keep importing new pipelines until we gain confidence in it. + +#### Long-term + +1. Wrangle in `core_repos` +2. Roll out to all pipelines +3. Switch all repos to main diff --git a/pulumi/github/repos/core_repos.yml b/pulumi/github/repos/core_repos.yml new file mode 100644 index 0000000..1fdaf6b --- /dev/null +++ b/pulumi/github/repos/core_repos.yml @@ -0,0 +1,14 @@ +- .github +- basic_training +- configs +- logos +- modules +- ops +- prettier-plugin-nextflow +- references +- setup-nextflow +- sublime +- test-datasets +- tools +- vscode-extensionpack +- website From abff7e731dafa866a1782e8bbbd21043bf1f8cd3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 18:49:04 -0500 Subject: [PATCH 06/40] Add missing repos --- pulumi/github/repos/core_repos.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulumi/github/repos/core_repos.yml b/pulumi/github/repos/core_repos.yml index 1fdaf6b..d353e21 100644 --- a/pulumi/github/repos/core_repos.yml +++ b/pulumi/github/repos/core_repos.yml @@ -12,3 +12,5 @@ - tools - vscode-extensionpack - website +- vale +- setup-nf-test From 20514de1db312c08e6b2814e2c035707049a8034 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 18:49:22 -0500 Subject: [PATCH 07/40] chore: Sort lines --- pulumi/github/repos/core_repos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulumi/github/repos/core_repos.yml b/pulumi/github/repos/core_repos.yml index d353e21..14ebcf5 100644 --- a/pulumi/github/repos/core_repos.yml +++ b/pulumi/github/repos/core_repos.yml @@ -7,10 +7,10 @@ - prettier-plugin-nextflow - references - setup-nextflow +- setup-nf-test - sublime - test-datasets - tools +- vale - vscode-extensionpack - website -- vale -- setup-nf-test From 33c8a50ff070fa8b9b583e29e9aa14ed74655358 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 18:52:35 -0500 Subject: [PATCH 08/40] chore: Add pipelines --- pulumi/github/repos/pipelines.yml | 99 +++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 pulumi/github/repos/pipelines.yml diff --git a/pulumi/github/repos/pipelines.yml b/pulumi/github/repos/pipelines.yml new file mode 100644 index 0000000..f73ac11 --- /dev/null +++ b/pulumi/github/repos/pipelines.yml @@ -0,0 +1,99 @@ +- airrflow +- ampliseq +- atacseq +- bacass +- bactmap +- bamtofastq +- cageseq +- callingcards +- chipseq +- circdna +- circrna +- clipseq +- coproid +- createpanelrefs +- createtaxdb +- crisprseq +- cutandrun +- datasync +- demo +- demultiplex +- denovotranscript +- detaxizer +- diaproteomics +- differentialabundance +- dualrnaseq +- eager +- epitopeprediction +- fastquorum +- fetchngs +- funcscan +- genomeannotator +- genomeassembler +- genomeskim +- gwas +- hgtseq +- hic +- hicar +- hlatyping +- imcyto +- isoseq +- lncpipe +- mag +- magmap +- marsseq +- mcmicro +- meerpipe +- metaboigniter +- metapep +- metatdenovo +- methylseq +- mhcquant +- mnaseseq +- molkart +- multiplesequencealign +- nanoseq +- nanostring +- nascent +- omicsgenetraitassociation +- oncoanalyser +- pairgenomealign +- pangenome +- pathogensurveillance +- pgdb +- phageannotator +- phaseimpute +- phyloplace +- pixelator +- proteinfold +- proteomicslfq +- radseq +- rangeland +- raredisease +- readsimulator +- reportho +- riboseq +- rnadnavar +- rnafusion +- rnaseq +- rnasplice +- rnavar +- sammyseq +- sarek +- scdownstream +- scnanoseq +- scrnaseq +- seqinspector +- setup-nf-test +- slamseq +- smrnaseq +- spatialvi +- spinningjenny +- taxprofiler +- tbanalyzer +- tfactivity +- vale +- variantbenchmarking +- variantcatalogue +- viralintegration +- viralrecon From bd199e96afa986914a102822a919ff4bb4041ff8 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 20 Jul 2024 19:21:26 -0500 Subject: [PATCH 09/40] chore: Add two different ways of tackling this 1. Importing them all by hand, some code duplication and effort, but probably the least likely to blow up 2. Looping through them all We can also start with 1, and then move to 2 once everything is captured in the Pulumi state with 1(which seems like the sane option) --- pulumi/github/repos/core/modules.py | 42 ++++++++++++++++++++ pulumi/github/repos/import_by_hand.py | 20 ++++++++++ pulumi/github/repos/loop_example.py | 55 +++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 pulumi/github/repos/core/modules.py create mode 100644 pulumi/github/repos/import_by_hand.py create mode 100644 pulumi/github/repos/loop_example.py diff --git a/pulumi/github/repos/core/modules.py b/pulumi/github/repos/core/modules.py new file mode 100644 index 0000000..0f9bf16 --- /dev/null +++ b/pulumi/github/repos/core/modules.py @@ -0,0 +1,42 @@ +import yaml + +import pulumi +import pulumi_github as github + + +nf_core_tf = github.Repository( + "nf-core-tf", + allow_merge_commit=False, + allow_rebase_merge=False, + allow_squash_merge=False, + default_branch="master", + description="Repository to host tool-specific module files for the Nextflow DSL2 community!", + has_downloads=True, + has_issues=True, + has_projects=True, + homepage_url="https://nf-co.re", + merge_commit_message="", + merge_commit_title="", + name="modules", + security_and_analysis=github.RepositorySecurityAndAnalysisArgs( + secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( + status="disabled", + ), + secret_scanning_push_protection=github.RepositorySecurityAndAnalysisSecretScanningPushProtectionArgs( + status="disabled", + ), + ), + squash_merge_commit_message="", + squash_merge_commit_title="", + topics=[ + "nextflow", + "pipelines", + "nf-test", + "modules", + "nf-core", + "dsl2", + "workflows", + ], + visibility="public", + opts=pulumi.ResourceOptions(protect=True), +) diff --git a/pulumi/github/repos/import_by_hand.py b/pulumi/github/repos/import_by_hand.py new file mode 100644 index 0000000..b28f9c2 --- /dev/null +++ b/pulumi/github/repos/import_by_hand.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +import yaml + +import pulumi +import pulumi_github as github + +import pipelines.denovotranscript +import pipelines.meerpipe +import pipelines.pairgenomealign +import pipelines.phaseimpute +import pipelines.reportho + +# ... + +import core.github +import core.modules + +# ... +import core.website diff --git a/pulumi/github/repos/loop_example.py b/pulumi/github/repos/loop_example.py new file mode 100644 index 0000000..eef61d7 --- /dev/null +++ b/pulumi/github/repos/loop_example.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +import yaml + +import pulumi +import pulumi_github as github + +TOPICS = [ + "nextflow", + "pipelines", + "nf-test", + "modules", + "nf-core", + "dsl2", + "workflows", +] + +alpha_test_pipeline_repos = [ + "denovotranscript", + "meerpipe", + "pairgenomealign", + "phaseimpute", + "reportho", +] + +for pipeline in alpha_test_pipeline_repos: + github.Repository( + "nf-core-tf", + allow_merge_commit=True, + allow_rebase_merge=True, + allow_squash_merge=True, + default_branch="master", + description="Alpha test repository for nf-core", + has_downloads=True, + has_issues=True, + has_projects=True, + homepage_url=f"https://nf-co.re/{pipeline}", + merge_commit_message="", + merge_commit_title="", + name=pipeline, + security_and_analysis=github.RepositorySecurityAndAnalysisArgs( + secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( + status="disabled", + ), + secret_scanning_push_protection=github.RepositorySecurityAndAnalysisSecretScanningPushProtectionArgs( + status="disabled", + ), + ), + squash_merge_commit_message="", + squash_merge_commit_title="", + topics=TOPICS, + visibility="public", + # NOTE Idk if this will work + opts=pulumi.ResourceOptions(protect=True), + ) From 005acec685ce2a2840304774277f6d475b7efd73 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 11:23:44 -0500 Subject: [PATCH 10/40] Add demo and testpipeline --- pulumi/github/repos/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pulumi/github/repos/README.md b/pulumi/github/repos/README.md index 6466569..8a753d2 100644 --- a/pulumi/github/repos/README.md +++ b/pulumi/github/repos/README.md @@ -8,6 +8,9 @@ This repo will be the "Actions" section at the bottom. We can then create a repo The new pipelines that are broken: +- demo +- testpipeline + - denovotranscript - meerpipe - pairgenomealign From 157829356fcf327175296bca5111cc5880bb08ff Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 11:31:26 -0500 Subject: [PATCH 11/40] Add Old Pipeline Health PHP code --- pulumi/github/repos/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulumi/github/repos/README.md b/pulumi/github/repos/README.md index 8a753d2..cd803ab 100644 --- a/pulumi/github/repos/README.md +++ b/pulumi/github/repos/README.md @@ -4,6 +4,10 @@ Goal is to replace https://oldsite.nf-co.re/pipeline_health This repo will be the "Actions" section at the bottom. We can then create a reporting page if we really need to see all the green checks +[Old Pipeline Health PHP code](https://github.com/nf-core/website/blob/old-site/public_html/pipeline_health.php) + +[New Pipeline Health page](https://github.com/nf-core/website/blob/main/sites/pipelines/src/pages/pipeline_health.astro) + ## Initial Roll-out The new pipelines that are broken: From 4b25a4a129ae80748b342bdbd54e8a61bd1394c2 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 11:49:11 -0500 Subject: [PATCH 12/40] Add docs on using 1password to automatically switch contexts --- docs/1password.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/1password.md diff --git a/docs/1password.md b/docs/1password.md new file mode 100644 index 0000000..757d666 --- /dev/null +++ b/docs/1password.md @@ -0,0 +1,23 @@ +# Pulumi + +[Pulumi Shell Plugin](https://developer.1password.com/docs/cli/shell-plugins/pulumi/) + +[How to use 1Password with different accounts automatically](https://developer.1password.com/docs/cli/shell-plugins/multiple-accounts/) + +```console +$ cd ~/src/nf-core + +~/src/nf-core $ op signin +# Select nf-core + +~/src/nf-core $ op plugin init pulumi + +Pulumi CLI +Authenticate with Pulumi Personal Access Token. + +? Locate your Pulumi Personal Access Token: Search in 1Password... + +? Locate your Pulumi Personal Access Token: Pulumi Personal Access Token (Private) + +? Configure when the chosen credential(s) will be used to authenticate: Use automatically when in this directory or subdirectories +``` From f7598557265406c5f74de5f5ee9eb9fe8218d7a7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 12:07:04 -0500 Subject: [PATCH 13/40] feat: Add prod esc environment --- pulumi/github/repos/Pulumi.prod.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pulumi/github/repos/Pulumi.prod.yaml diff --git a/pulumi/github/repos/Pulumi.prod.yaml b/pulumi/github/repos/Pulumi.prod.yaml new file mode 100644 index 0000000..37afc74 --- /dev/null +++ b/pulumi/github/repos/Pulumi.prod.yaml @@ -0,0 +1,4 @@ +config: + github:owner: nf-core +environment: + - github-prod From feaf7d4bb662d0aad7b65c631e1120d804a11f48 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 12:12:21 -0500 Subject: [PATCH 14/40] chore: Remove template --- pulumi/github/repos/Pulumi.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulumi/github/repos/Pulumi.yaml b/pulumi/github/repos/Pulumi.yaml index d492c4a..6154903 100644 --- a/pulumi/github/repos/Pulumi.yaml +++ b/pulumi/github/repos/Pulumi.yaml @@ -4,7 +4,3 @@ runtime: options: virtualenv: venv description: Managing GitHub repos -config: - pulumi:tags: - value: - pulumi:template: https://www.pulumi.com/ai/api/project/3cb51e5f-2548-4d7b-9d9d-1ea680ac96ee.zip From 4141c81aeffdcde1a7f5e3235c095e38a749e853 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 12:21:26 -0500 Subject: [PATCH 15/40] chore: Import nf-core/testpipeline pulumi env run nf-core/github-prod -i pulumi import github:index/repository:Repository nf-core testpipeline --- pulumi/github/repos/__main__.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/pulumi/github/repos/__main__.py b/pulumi/github/repos/__main__.py index 103aade..1838794 100644 --- a/pulumi/github/repos/__main__.py +++ b/pulumi/github/repos/__main__.py @@ -5,21 +5,15 @@ import pulumi import pulumi_github as github - -nf_core_tf = github.Repository( - "nf-core-tf", - allow_merge_commit=False, - allow_rebase_merge=False, - allow_squash_merge=False, +nf_core = github.Repository( + "nf-core", default_branch="master", - description="Repository to host tool-specific module files for the Nextflow DSL2 community!", + description="A small example pipeline used to test new nf-core infrastructure and common code.", has_downloads=True, has_issues=True, has_projects=True, - homepage_url="https://nf-co.re", - merge_commit_message="", - merge_commit_title="", - name="modules", + has_wiki=True, + name="testpipeline", security_and_analysis=github.RepositorySecurityAndAnalysisArgs( secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( status="disabled", @@ -28,17 +22,6 @@ status="disabled", ), ), - squash_merge_commit_message="", - squash_merge_commit_title="", - topics=[ - "nextflow", - "pipelines", - "nf-test", - "modules", - "nf-core", - "dsl2", - "workflows", - ], visibility="public", opts=pulumi.ResourceOptions(protect=True), ) From 5f20c05bf1283f8f6a9788b11982e51184f2bcbd Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 12:35:16 -0500 Subject: [PATCH 16/40] refactor: Move testpipeline into it's own file --- pulumi/github/repos/README.md | 10 ++++++ pulumi/github/repos/__main__.py | 26 +-------------- pulumi/github/repos/loop_example.py | 2 +- pulumi/github/repos/pipelines/__init__.py | 0 pulumi/github/repos/pipelines/testpipeline.py | 33 +++++++++++++++++++ 5 files changed, 45 insertions(+), 26 deletions(-) create mode 100644 pulumi/github/repos/pipelines/__init__.py create mode 100644 pulumi/github/repos/pipelines/testpipeline.py diff --git a/pulumi/github/repos/README.md b/pulumi/github/repos/README.md index cd803ab..933c12b 100644 --- a/pulumi/github/repos/README.md +++ b/pulumi/github/repos/README.md @@ -39,3 +39,13 @@ Maybe: 1. Wrangle in `core_repos` 2. Roll out to all pipelines 3. Switch all repos to main + +## Docs + +https://www.pulumi.com/registry/packages/github/api-docs/repository/ + +### Importing Repos + +```sh +pulumi env run nf-core/github-prod -i pulumi import github:index/repository:Repository testpipeline testpipeline +``` diff --git a/pulumi/github/repos/__main__.py b/pulumi/github/repos/__main__.py index 1838794..a8bcc2d 100644 --- a/pulumi/github/repos/__main__.py +++ b/pulumi/github/repos/__main__.py @@ -1,27 +1,3 @@ #!/usr/bin/env python -import yaml - -import pulumi -import pulumi_github as github - -nf_core = github.Repository( - "nf-core", - default_branch="master", - description="A small example pipeline used to test new nf-core infrastructure and common code.", - has_downloads=True, - has_issues=True, - has_projects=True, - has_wiki=True, - name="testpipeline", - security_and_analysis=github.RepositorySecurityAndAnalysisArgs( - secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( - status="disabled", - ), - secret_scanning_push_protection=github.RepositorySecurityAndAnalysisSecretScanningPushProtectionArgs( - status="disabled", - ), - ), - visibility="public", - opts=pulumi.ResourceOptions(protect=True), -) +import pipelines.testpipeline diff --git a/pulumi/github/repos/loop_example.py b/pulumi/github/repos/loop_example.py index eef61d7..268cec4 100644 --- a/pulumi/github/repos/loop_example.py +++ b/pulumi/github/repos/loop_example.py @@ -25,7 +25,7 @@ for pipeline in alpha_test_pipeline_repos: github.Repository( - "nf-core-tf", + pipeline, allow_merge_commit=True, allow_rebase_merge=True, allow_squash_merge=True, diff --git a/pulumi/github/repos/pipelines/__init__.py b/pulumi/github/repos/pipelines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py new file mode 100644 index 0000000..76bd2ce --- /dev/null +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -0,0 +1,33 @@ +import pulumi +import pulumi_github as github + +TOPICS = [ + "nextflow", + "pipelines", + "nf-test", + "modules", + "nf-core", + "dsl2", + "workflows", +] + +nfcore_testpipeline = github.Repository( + "testpipeline", + default_branch="master", + description="A small example pipeline used to test new nf-core infrastructure and common code.", + has_downloads=True, + has_issues=True, + has_projects=True, + has_wiki=True, + name="testpipeline", + security_and_analysis=github.RepositorySecurityAndAnalysisArgs( + secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( + status="disabled", + ), + secret_scanning_push_protection=github.RepositorySecurityAndAnalysisSecretScanningPushProtectionArgs( + status="disabled", + ), + ), + visibility="public", + opts=pulumi.ResourceOptions(protect=True), +) From e5f3540590e313f3ce96bb6c3be9cbcffc3cd8cc Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 12:37:12 -0500 Subject: [PATCH 17/40] Take a stab at some settings --- pulumi/github/repos/pipelines/testpipeline.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 76bd2ce..73b4af0 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -1,6 +1,8 @@ import pulumi import pulumi_github as github +NAME = "testpipeline" + TOPICS = [ "nextflow", "pipelines", @@ -12,14 +14,15 @@ ] nfcore_testpipeline = github.Repository( - "testpipeline", + NAME, default_branch="master", description="A small example pipeline used to test new nf-core infrastructure and common code.", has_downloads=True, has_issues=True, has_projects=True, - has_wiki=True, - name="testpipeline", + has_wiki=False, + homepage_url=f"https://nf-co.re/{NAME}", + name=NAME, security_and_analysis=github.RepositorySecurityAndAnalysisArgs( secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( status="disabled", @@ -29,5 +32,6 @@ ), ), visibility="public", + topics=TOPICS, opts=pulumi.ResourceOptions(protect=True), ) From 56ac14d71a0f9273386fe3f391f47e63d8e120d8 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 13:49:44 -0500 Subject: [PATCH 18/40] Add Specs from php code This is the best I'm gonna do. We can iterate in a readable way here. --- pulumi/github/repos/pipelines/testpipeline.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 73b4af0..4ffc354 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -21,6 +21,10 @@ has_issues=True, has_projects=True, has_wiki=False, + allow_merge_commit=True, + allow_rebase_merge=True, + allow_squash_merge=False, + delete_branch_on_merge=True, homepage_url=f"https://nf-co.re/{NAME}", name=NAME, security_and_analysis=github.RepositorySecurityAndAnalysisArgs( @@ -35,3 +39,41 @@ topics=TOPICS, opts=pulumi.ResourceOptions(protect=True), ) +# TODO Names of required CI checks. These are added to whatever already exists. +# public $required_status_check_contexts = [ +# 'pre-commit', +# 'nf-core', + +# TODO Make branches foreach (['master', 'dev', 'TEMPLATE'] as $branch) { +# TODO Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 +# TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 +# https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 +# TODO Set contributors to push +# TODO Set core to admin +# TODO 'repo_wikis' => 'Disable wikis', +# TODO 'repo_issues' => 'Enable issues', +# TODO 'repo_merge_commits' => 'Allow merge commits', +# TODO 'repo_merge_rebase' => 'Allow rebase merging', +# TODO 'repo_merge_squash' => 'Do not allow squash merges', +# TODO 'repo_default_branch' => 'default branch master (released) or dev (no releases)', +# TODO 'repo_keywords' => 'Minimum keywords set', +# TODO 'repo_description' => 'Description must be set', +# TODO 'repo_url' => 'URL should be set to https://nf-co.re', +# TODO 'team_contributors' => 'Write access for nf-core/contributors', +# TODO 'team_core' => 'Admin access for nf-core/core', +# TODO 'branch_master_exists' => 'master branch: branch must exist', +# TODO 'branch_dev_exists' => 'dev branch: branch must exist', +# TODO 'branch_template_exists' => 'TEMPLATE branch: branch must exist', +# TODO 'branch_master_strict_updates' => 'master branch: do not require branch to be up to date before merging', +# TODO 'branch_master_required_ci' => 'master branch: minimum set of CI tests must pass', +# TODO 'branch_master_stale_reviews' => 'master branch: reviews not marked stale after new commits', +# TODO 'branch_master_code_owner_reviews' => 'master branch: code owner reviews not required', +# TODO 'branch_master_required_num_reviews' => 'master branch: 2 reviews required', +# TODO 'branch_master_enforce_admins' => 'master branch: do not enforce rules for admins', +# TODO 'branch_dev_strict_updates' => 'dev branch: do not require branch to be up to date before merging', +# TODO 'branch_dev_required_ci' => 'dev branch: minimum set of CI tests must pass', +# TODO 'branch_dev_stale_reviews' => 'dev branch: reviews not marked stale after new commits', +# TODO 'branch_dev_code_owner_reviews' => 'dev branch: code owner reviews not required', +# TODO 'branch_dev_required_num_reviews' => 'dev branch: 1 review required', +# TODO 'branch_dev_enforce_admins' => 'dev branch: do not enforce rules for admins', +# TODO 'branch_template_restrict_push' => 'Restrict push to TEMPLATE to @nf-core-bot', From df0656d8ad574b99f96ffa8e1f1cd9909198d47a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 13:52:08 -0500 Subject: [PATCH 19/40] chore: Import default branch pulumi import github:index/branchDefault:BranchDefault branch_default_testpipeline testpipeline --- pulumi/github/repos/pipelines/testpipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 4ffc354..72cc9ae 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -15,7 +15,6 @@ nfcore_testpipeline = github.Repository( NAME, - default_branch="master", description="A small example pipeline used to test new nf-core infrastructure and common code.", has_downloads=True, has_issues=True, @@ -39,12 +38,19 @@ topics=TOPICS, opts=pulumi.ResourceOptions(protect=True), ) + # TODO Names of required CI checks. These are added to whatever already exists. # public $required_status_check_contexts = [ # 'pre-commit', # 'nf-core', # TODO Make branches foreach (['master', 'dev', 'TEMPLATE'] as $branch) { +branch_default_testpipeline = github.BranchDefault( + f"branch_default_{NAME}", + branch="master", + repository={NAME}, + opts=pulumi.ResourceOptions(protect=True), +) # TODO Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 # TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 # https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 From eb4b899cc86f4104cd853f743145bca4cb624c18 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:01:08 -0500 Subject: [PATCH 20/40] chore: Import testpipeline template branch --- pulumi/github/repos/pipelines/testpipeline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 72cc9ae..0f4a1ec 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -51,6 +51,12 @@ repository={NAME}, opts=pulumi.ResourceOptions(protect=True), ) +branch_template_testpipeline = github.Branch( + "branch_template_testpipeline", + branch="TEMPLATE", + repository="testpipeline", + opts=pulumi.ResourceOptions(protect=True), +) # TODO Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 # TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 # https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 From f9aa2618c42465499858593c1b69e934ca984135 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:03:06 -0500 Subject: [PATCH 21/40] chore: Import testpipeline dev branch pulumi env run nf-core/github-prod -i pulumi import github:index/branch:Branch branch_dev_testpipeline testpipeline:dev --- pulumi/github/repos/pipelines/testpipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 0f4a1ec..918a87d 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -44,13 +44,19 @@ # 'pre-commit', # 'nf-core', -# TODO Make branches foreach (['master', 'dev', 'TEMPLATE'] as $branch) { +# Make branches foreach (['master', 'dev', 'TEMPLATE'] as $branch) { branch_default_testpipeline = github.BranchDefault( f"branch_default_{NAME}", branch="master", repository={NAME}, opts=pulumi.ResourceOptions(protect=True), ) +branch_dev_testpipeline = github.Branch( + "branch_dev_testpipeline", + branch="dev", + repository="testpipeline", + opts=pulumi.ResourceOptions(protect=True), +) branch_template_testpipeline = github.Branch( "branch_template_testpipeline", branch="TEMPLATE", From 41db6e0de3e764a04914f1a7ef8c838e724af441 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:09:57 -0500 Subject: [PATCH 22/40] chore: Check off some TODOs --- pulumi/github/repos/pipelines/testpipeline.py | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 918a87d..6fbb22d 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -1,3 +1,6 @@ +# NOTE => are tests from PHP +# TODO Convert => to actual tests https://www.pulumi.com/docs/using-pulumi/testing/ +# https://github.com/pulumi/examples/blob/74db62a03d013c2854d2cf933c074ea0a3bbf69d/testing-unit-py/test_ec2.py import pulumi import pulumi_github as github @@ -15,16 +18,16 @@ nfcore_testpipeline = github.Repository( NAME, - description="A small example pipeline used to test new nf-core infrastructure and common code.", + description="A small example pipeline used to test new nf-core infrastructure and common code.", # 'repo_description' => 'Description must be set', has_downloads=True, - has_issues=True, + has_issues=True, # 'repo_issues' => 'Enable issues', has_projects=True, - has_wiki=False, - allow_merge_commit=True, - allow_rebase_merge=True, - allow_squash_merge=False, + has_wiki=False, # 'repo_wikis' => 'Disable wikis', + allow_merge_commit=True, # 'repo_merge_commits' => 'Allow merge commits', + allow_rebase_merge=True, # 'repo_merge_rebase' => 'Allow rebase merging', + allow_squash_merge=False, # 'repo_merge_squash' => 'Do not allow squash merges', delete_branch_on_merge=True, - homepage_url=f"https://nf-co.re/{NAME}", + homepage_url=f"https://nf-co.re/{NAME}", # 'repo_url' => 'URL should be set to https://nf-co.re', name=NAME, security_and_analysis=github.RepositorySecurityAndAnalysisArgs( secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs( @@ -35,7 +38,7 @@ ), ), visibility="public", - topics=TOPICS, + topics=TOPICS, # 'repo_keywords' => 'Minimum keywords set', opts=pulumi.ResourceOptions(protect=True), ) @@ -45,18 +48,23 @@ # 'nf-core', # Make branches foreach (['master', 'dev', 'TEMPLATE'] as $branch) { +# 'repo_default_branch' => 'default branch master (released) or dev (no releases)', +# TODO Toggle this on dev and master? +# 'branch_master_exists' => 'master branch: branch must exist', branch_default_testpipeline = github.BranchDefault( f"branch_default_{NAME}", branch="master", repository={NAME}, opts=pulumi.ResourceOptions(protect=True), ) +# 'branch_dev_exists' => 'dev branch: branch must exist', branch_dev_testpipeline = github.Branch( "branch_dev_testpipeline", branch="dev", repository="testpipeline", opts=pulumi.ResourceOptions(protect=True), ) +# 'branch_template_exists' => 'TEMPLATE branch: branch must exist', branch_template_testpipeline = github.Branch( "branch_template_testpipeline", branch="TEMPLATE", @@ -68,20 +76,8 @@ # https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 # TODO Set contributors to push # TODO Set core to admin -# TODO 'repo_wikis' => 'Disable wikis', -# TODO 'repo_issues' => 'Enable issues', -# TODO 'repo_merge_commits' => 'Allow merge commits', -# TODO 'repo_merge_rebase' => 'Allow rebase merging', -# TODO 'repo_merge_squash' => 'Do not allow squash merges', -# TODO 'repo_default_branch' => 'default branch master (released) or dev (no releases)', -# TODO 'repo_keywords' => 'Minimum keywords set', -# TODO 'repo_description' => 'Description must be set', -# TODO 'repo_url' => 'URL should be set to https://nf-co.re', # TODO 'team_contributors' => 'Write access for nf-core/contributors', # TODO 'team_core' => 'Admin access for nf-core/core', -# TODO 'branch_master_exists' => 'master branch: branch must exist', -# TODO 'branch_dev_exists' => 'dev branch: branch must exist', -# TODO 'branch_template_exists' => 'TEMPLATE branch: branch must exist', # TODO 'branch_master_strict_updates' => 'master branch: do not require branch to be up to date before merging', # TODO 'branch_master_required_ci' => 'master branch: minimum set of CI tests must pass', # TODO 'branch_master_stale_reviews' => 'master branch: reviews not marked stale after new commits', From bda668639de28dbf913e5df39cb2b9aafd0b904c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:21:29 -0500 Subject: [PATCH 23/40] chore: Import Master Branch protection pulumi import github:index/repositoryRuleset:RepositoryRuleset ruleset_branch_default_testpipeline testpipeline:1220601 --- pulumi/github/repos/pipelines/testpipeline.py | 54 ++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 6fbb22d..06d918b 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -72,18 +72,60 @@ opts=pulumi.ResourceOptions(protect=True), ) # TODO Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 -# TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 -# https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 -# TODO Set contributors to push -# TODO Set core to admin -# TODO 'team_contributors' => 'Write access for nf-core/contributors', -# TODO 'team_core' => 'Admin access for nf-core/core', +# NOTE This uses the new Rulesets instead of classic branch protection rule # TODO 'branch_master_strict_updates' => 'master branch: do not require branch to be up to date before merging', # TODO 'branch_master_required_ci' => 'master branch: minimum set of CI tests must pass', # TODO 'branch_master_stale_reviews' => 'master branch: reviews not marked stale after new commits', # TODO 'branch_master_code_owner_reviews' => 'master branch: code owner reviews not required', # TODO 'branch_master_required_num_reviews' => 'master branch: 2 reviews required', # TODO 'branch_master_enforce_admins' => 'master branch: do not enforce rules for admins', +ruleset_branch_default_testpipeline = github.RepositoryRuleset( + "ruleset_branch_default_testpipeline", + bypass_actors=[ + github.RepositoryRulesetBypassActorArgs( + actor_id=2649377, + actor_type="Team", + bypass_mode="always", + ) + ], + conditions=github.RepositoryRulesetConditionsArgs( + ref_name=github.RepositoryRulesetConditionsRefNameArgs( + excludes=[], + includes=["~DEFAULT_BRANCH"], + ), + ), + enforcement="active", + name="master", + repository="testpipeline", + rules=github.RepositoryRulesetRulesArgs( + deletion=True, + non_fast_forward=True, + pull_request=github.RepositoryRulesetRulesPullRequestArgs( + required_approving_review_count=2, + ), + required_status_checks=github.RepositoryRulesetRulesRequiredStatusChecksArgs( + required_checks=[ + github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( + context="Prettier", + integration_id=0, + ), + github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( + context="nf-core", + integration_id=0, + ), + ], + strict_required_status_checks_policy=True, + ), + ), + target="branch", + opts=pulumi.ResourceOptions(protect=True), +) +# TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 +# https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 +# TODO Set contributors to push +# TODO Set core to admin +# TODO 'team_contributors' => 'Write access for nf-core/contributors', +# TODO 'team_core' => 'Admin access for nf-core/core', # TODO 'branch_dev_strict_updates' => 'dev branch: do not require branch to be up to date before merging', # TODO 'branch_dev_required_ci' => 'dev branch: minimum set of CI tests must pass', # TODO 'branch_dev_stale_reviews' => 'dev branch: reviews not marked stale after new commits', From 296ab65c24043113157cb595045827d9f9a9f097 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:23:16 -0500 Subject: [PATCH 24/40] chore: Import dev branch ruleset pulumi import github:index/repositoryRuleset:RepositoryRuleset ruleset_branch_dev_testpipeline testpipeline:1220600 --- pulumi/github/repos/pipelines/testpipeline.py | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 06d918b..cfa73b6 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -120,16 +120,49 @@ target="branch", opts=pulumi.ResourceOptions(protect=True), ) -# TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 -# https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 -# TODO Set contributors to push -# TODO Set core to admin -# TODO 'team_contributors' => 'Write access for nf-core/contributors', -# TODO 'team_core' => 'Admin access for nf-core/core', # TODO 'branch_dev_strict_updates' => 'dev branch: do not require branch to be up to date before merging', # TODO 'branch_dev_required_ci' => 'dev branch: minimum set of CI tests must pass', # TODO 'branch_dev_stale_reviews' => 'dev branch: reviews not marked stale after new commits', # TODO 'branch_dev_code_owner_reviews' => 'dev branch: code owner reviews not required', # TODO 'branch_dev_required_num_reviews' => 'dev branch: 1 review required', # TODO 'branch_dev_enforce_admins' => 'dev branch: do not enforce rules for admins', +ruleset_branch_dev_testpipeline = github.RepositoryRuleset( + "ruleset_branch_dev_testpipeline", + bypass_actors=[ + github.RepositoryRulesetBypassActorArgs( + actor_id=2649377, + actor_type="Team", + bypass_mode="always", + ), + github.RepositoryRulesetBypassActorArgs( + actor_id=4462882, + actor_type="Team", + bypass_mode="always", + ), + ], + conditions=github.RepositoryRulesetConditionsArgs( + ref_name=github.RepositoryRulesetConditionsRefNameArgs( + excludes=[], + includes=["refs/heads/dev"], + ), + ), + enforcement="active", + name="dev", + repository="testpipeline", + rules=github.RepositoryRulesetRulesArgs( + deletion=True, + non_fast_forward=True, + pull_request=github.RepositoryRulesetRulesPullRequestArgs( + required_approving_review_count=1, + ), + ), + target="branch", + opts=pulumi.ResourceOptions(protect=True), +) +# TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 +# https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 +# TODO Set contributors to push +# TODO Set core to admin +# TODO 'team_contributors' => 'Write access for nf-core/contributors', +# TODO 'team_core' => 'Admin access for nf-core/core', # TODO 'branch_template_restrict_push' => 'Restrict push to TEMPLATE to @nf-core-bot', From cccbc96b2dc88e262d3ca591fb3c5be4d380560b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:25:50 -0500 Subject: [PATCH 25/40] chore: Import template branch ruleset for testpipeline pulumi import github:index/repositoryRuleset:RepositoryRuleset ruleset_branch_TEMPLATE_testpipeline testpipeline:1220597 --- pulumi/github/repos/pipelines/testpipeline.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index cfa73b6..eec7efc 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -159,10 +159,36 @@ target="branch", opts=pulumi.ResourceOptions(protect=True), ) -# TODO Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 -# https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L275-L278 +# TODO Double check +# Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 +# TODO 'branch_template_restrict_push' => 'Restrict push to TEMPLATE to @nf-core-bot', +ruleset_branch_template_testpipeline = github.RepositoryRuleset( + "ruleset_branch_TEMPLATE_testpipeline", + bypass_actors=[ + github.RepositoryRulesetBypassActorArgs( + actor_id=2649377, + actor_type="Team", + bypass_mode="always", + ) + ], + conditions=github.RepositoryRulesetConditionsArgs( + ref_name=github.RepositoryRulesetConditionsRefNameArgs( + excludes=[], + includes=["refs/heads/TEMPLATE"], + ), + ), + enforcement="active", + name="template", + repository="testpipeline", + rules=github.RepositoryRulesetRulesArgs( + deletion=True, + non_fast_forward=True, + update=True, + ), + target="branch", + opts=pulumi.ResourceOptions(protect=True), +) # TODO Set contributors to push # TODO Set core to admin # TODO 'team_contributors' => 'Write access for nf-core/contributors', # TODO 'team_core' => 'Admin access for nf-core/core', -# TODO 'branch_template_restrict_push' => 'Restrict push to TEMPLATE to @nf-core-bot', From c786f5b75bd0e828e17991ab460d28ec9b0b66b2 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:27:09 -0500 Subject: [PATCH 26/40] chore: Remove duplicate TODO --- pulumi/github/repos/pipelines/testpipeline.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index eec7efc..7fec012 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -188,7 +188,5 @@ target="branch", opts=pulumi.ResourceOptions(protect=True), ) -# TODO Set contributors to push -# TODO Set core to admin # TODO 'team_contributors' => 'Write access for nf-core/contributors', # TODO 'team_core' => 'Admin access for nf-core/core', From 4e3785bc57aef75223c345784ea8ec910bbe58db Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:43:17 -0500 Subject: [PATCH 27/40] chore: Match up specs with code --- pulumi/github/repos/pipelines/testpipeline.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 7fec012..66caf8f 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -16,6 +16,9 @@ "workflows", ] +CORE_TEAM_ID = 2649377 +MAINTAINERS_TEAM_ID = 4462882 + nfcore_testpipeline = github.Repository( NAME, description="A small example pipeline used to test new nf-core infrastructure and common code.", # 'repo_description' => 'Description must be set', @@ -74,16 +77,14 @@ # TODO Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 # NOTE This uses the new Rulesets instead of classic branch protection rule # TODO 'branch_master_strict_updates' => 'master branch: do not require branch to be up to date before merging', -# TODO 'branch_master_required_ci' => 'master branch: minimum set of CI tests must pass', # TODO 'branch_master_stale_reviews' => 'master branch: reviews not marked stale after new commits', # TODO 'branch_master_code_owner_reviews' => 'master branch: code owner reviews not required', -# TODO 'branch_master_required_num_reviews' => 'master branch: 2 reviews required', -# TODO 'branch_master_enforce_admins' => 'master branch: do not enforce rules for admins', ruleset_branch_default_testpipeline = github.RepositoryRuleset( "ruleset_branch_default_testpipeline", bypass_actors=[ + # 'branch_master_enforce_admins' => 'master branch: do not enforce rules for admins', github.RepositoryRulesetBypassActorArgs( - actor_id=2649377, + actor_id=CORE_TEAM_ID, actor_type="Team", bypass_mode="always", ) @@ -101,8 +102,10 @@ deletion=True, non_fast_forward=True, pull_request=github.RepositoryRulesetRulesPullRequestArgs( + # 'branch_master_required_num_reviews' => 'master branch: 2 reviews required', required_approving_review_count=2, ), + # 'branch_master_required_ci' => 'master branch: minimum set of CI tests must pass', required_status_checks=github.RepositoryRulesetRulesRequiredStatusChecksArgs( required_checks=[ github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( @@ -124,18 +127,17 @@ # TODO 'branch_dev_required_ci' => 'dev branch: minimum set of CI tests must pass', # TODO 'branch_dev_stale_reviews' => 'dev branch: reviews not marked stale after new commits', # TODO 'branch_dev_code_owner_reviews' => 'dev branch: code owner reviews not required', -# TODO 'branch_dev_required_num_reviews' => 'dev branch: 1 review required', -# TODO 'branch_dev_enforce_admins' => 'dev branch: do not enforce rules for admins', ruleset_branch_dev_testpipeline = github.RepositoryRuleset( "ruleset_branch_dev_testpipeline", + # 'branch_dev_enforce_admins' => 'dev branch: do not enforce rules for admins', bypass_actors=[ github.RepositoryRulesetBypassActorArgs( - actor_id=2649377, + actor_id=CORE_TEAM_ID, actor_type="Team", bypass_mode="always", ), github.RepositoryRulesetBypassActorArgs( - actor_id=4462882, + actor_id=MAINTAINERS_TEAM_ID, actor_type="Team", bypass_mode="always", ), @@ -153,6 +155,7 @@ deletion=True, non_fast_forward=True, pull_request=github.RepositoryRulesetRulesPullRequestArgs( + # 'branch_dev_required_num_reviews' => 'dev branch: 1 review required', required_approving_review_count=1, ), ), @@ -161,15 +164,15 @@ ) # TODO Double check # Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 -# TODO 'branch_template_restrict_push' => 'Restrict push to TEMPLATE to @nf-core-bot', ruleset_branch_template_testpipeline = github.RepositoryRuleset( "ruleset_branch_TEMPLATE_testpipeline", bypass_actors=[ github.RepositoryRulesetBypassActorArgs( - actor_id=2649377, + actor_id=CORE_TEAM_ID, actor_type="Team", bypass_mode="always", ) + # TODO 'branch_template_restrict_push' => 'Restrict push to TEMPLATE to @nf-core-bot', ], conditions=github.RepositoryRulesetConditionsArgs( ref_name=github.RepositoryRulesetConditionsRefNameArgs( From a3611b481d06c5ae6796d35eff3670a098358d68 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:51:42 -0500 Subject: [PATCH 28/40] feat: Update dev branch requirements --- pulumi/github/repos/pipelines/testpipeline.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 66caf8f..2b73c7a 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -124,9 +124,6 @@ opts=pulumi.ResourceOptions(protect=True), ) # TODO 'branch_dev_strict_updates' => 'dev branch: do not require branch to be up to date before merging', -# TODO 'branch_dev_required_ci' => 'dev branch: minimum set of CI tests must pass', -# TODO 'branch_dev_stale_reviews' => 'dev branch: reviews not marked stale after new commits', -# TODO 'branch_dev_code_owner_reviews' => 'dev branch: code owner reviews not required', ruleset_branch_dev_testpipeline = github.RepositoryRuleset( "ruleset_branch_dev_testpipeline", # 'branch_dev_enforce_admins' => 'dev branch: do not enforce rules for admins', @@ -155,8 +152,25 @@ deletion=True, non_fast_forward=True, pull_request=github.RepositoryRulesetRulesPullRequestArgs( - # 'branch_dev_required_num_reviews' => 'dev branch: 1 review required', - required_approving_review_count=1, + dismiss_stale_reviews_on_push=False, # 'branch_dev_stale_reviews' => 'dev branch: reviews not marked stale after new commits', + require_code_owner_review=False, # 'branch_dev_code_owner_reviews' => 'dev branch: code owner reviews not required', + # TODO require_last_push_approval=True, + required_approving_review_count=1, # 'branch_dev_required_num_reviews' => 'dev branch: 1 review required', + # TODO required_review_thread_resolution=True, + ), + # 'branch_dev_required_ci' => 'dev branch: minimum set of CI tests must pass', + required_status_checks=github.RepositoryRulesetRulesRequiredStatusChecksArgs( + required_checks=[ + github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( + context="nf-core", + integration_id=0, + ), + github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( + context="pre-commit", + integration_id=0, + ), + ], + strict_required_status_checks_policy=True, ), ), target="branch", From 6ca606df5c2197c0cc0cdf36295e1e84c1e18559 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 14:56:21 -0500 Subject: [PATCH 29/40] refactor: Move required CI Checks out --- pulumi/github/repos/pipelines/testpipeline.py | 50 ++++++++----------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 2b73c7a..61d75c8 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -16,6 +16,21 @@ "workflows", ] +# Names of required CI checks. These are added to whatever already exists. +# public $required_status_check_contexts = [ +# 'pre-commit', +# 'nf-core', +REQUIRED_CI_CHECKS = [ + github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( + context="pre-commit", + integration_id=0, + ), + github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( + context="nf-core", + integration_id=0, + ), +] + CORE_TEAM_ID = 2649377 MAINTAINERS_TEAM_ID = 4462882 @@ -45,10 +60,6 @@ opts=pulumi.ResourceOptions(protect=True), ) -# TODO Names of required CI checks. These are added to whatever already exists. -# public $required_status_check_contexts = [ -# 'pre-commit', -# 'nf-core', # Make branches foreach (['master', 'dev', 'TEMPLATE'] as $branch) { # 'repo_default_branch' => 'default branch master (released) or dev (no releases)', @@ -74,11 +85,9 @@ repository="testpipeline", opts=pulumi.ResourceOptions(protect=True), ) -# TODO Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 +# Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 # NOTE This uses the new Rulesets instead of classic branch protection rule # TODO 'branch_master_strict_updates' => 'master branch: do not require branch to be up to date before merging', -# TODO 'branch_master_stale_reviews' => 'master branch: reviews not marked stale after new commits', -# TODO 'branch_master_code_owner_reviews' => 'master branch: code owner reviews not required', ruleset_branch_default_testpipeline = github.RepositoryRuleset( "ruleset_branch_default_testpipeline", bypass_actors=[ @@ -102,21 +111,13 @@ deletion=True, non_fast_forward=True, pull_request=github.RepositoryRulesetRulesPullRequestArgs( - # 'branch_master_required_num_reviews' => 'master branch: 2 reviews required', - required_approving_review_count=2, + required_approving_review_count=2, # 'branch_master_required_num_reviews' => 'master branch: 2 reviews required', + dismiss_stale_reviews_on_push=False, # 'branch_master_stale_reviews' => 'master branch: reviews not marked stale after new commits' + require_code_owner_review=False, # 'branch_master_code_owner_reviews' => 'master branch: code owner reviews not required', ), # 'branch_master_required_ci' => 'master branch: minimum set of CI tests must pass', required_status_checks=github.RepositoryRulesetRulesRequiredStatusChecksArgs( - required_checks=[ - github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( - context="Prettier", - integration_id=0, - ), - github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( - context="nf-core", - integration_id=0, - ), - ], + required_checks=REQUIRED_CI_CHECKS, strict_required_status_checks_policy=True, ), ), @@ -160,16 +161,7 @@ ), # 'branch_dev_required_ci' => 'dev branch: minimum set of CI tests must pass', required_status_checks=github.RepositoryRulesetRulesRequiredStatusChecksArgs( - required_checks=[ - github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( - context="nf-core", - integration_id=0, - ), - github.RepositoryRulesetRulesRequiredStatusChecksRequiredCheckArgs( - context="pre-commit", - integration_id=0, - ), - ], + required_checks=REQUIRED_CI_CHECKS, strict_required_status_checks_policy=True, ), ), From 8dc93658ba18d9a6c424cc72d4f0c9c35cfedb78 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 21:52:04 -0500 Subject: [PATCH 30/40] feat: Add contributors and core permissions --- pulumi/github/repos/pipelines/testpipeline.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 61d75c8..0e53101 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -63,7 +63,7 @@ # Make branches foreach (['master', 'dev', 'TEMPLATE'] as $branch) { # 'repo_default_branch' => 'default branch master (released) or dev (no releases)', -# TODO Toggle this on dev and master? +# TODO Toggle this on dev as default if there's not release? # 'branch_master_exists' => 'master branch: branch must exist', branch_default_testpipeline = github.BranchDefault( f"branch_default_{NAME}", @@ -197,5 +197,17 @@ target="branch", opts=pulumi.ResourceOptions(protect=True), ) -# TODO 'team_contributors' => 'Write access for nf-core/contributors', -# TODO 'team_core' => 'Admin access for nf-core/core', +# 'team_contributors' => 'Write access for nf-core/contributors', +contributors_team_repo_testpipeline = github.TeamRepository( + "contributors_team_repo_testpipeline", + team_id="contributors", + repository="testpipeline", + permission="push", +) +# 'team_core' => 'Admin access for nf-core/core', +core_team_repo_testpipeline = github.TeamRepository( + "core_team_repo_testpipeline", + team_id="core", + repository="testpipeline", + permission="admin", +) From bbb1a426a02aa58f5b27f9cda2295bee548e55f1 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 22:01:58 -0500 Subject: [PATCH 31/40] refactor: Use NAME variable where ever possible --- pulumi/github/repos/pipelines/testpipeline.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 0e53101..41ce3a8 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -68,28 +68,28 @@ branch_default_testpipeline = github.BranchDefault( f"branch_default_{NAME}", branch="master", - repository={NAME}, + repository=NAME, opts=pulumi.ResourceOptions(protect=True), ) # 'branch_dev_exists' => 'dev branch: branch must exist', branch_dev_testpipeline = github.Branch( - "branch_dev_testpipeline", + f"branch_dev_{NAME}", branch="dev", - repository="testpipeline", + repository=NAME, opts=pulumi.ResourceOptions(protect=True), ) # 'branch_template_exists' => 'TEMPLATE branch: branch must exist', branch_template_testpipeline = github.Branch( - "branch_template_testpipeline", + f"branch_template_{NAME}", branch="TEMPLATE", - repository="testpipeline", + repository=NAME, opts=pulumi.ResourceOptions(protect=True), ) # Add branch protections https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L296 # NOTE This uses the new Rulesets instead of classic branch protection rule # TODO 'branch_master_strict_updates' => 'master branch: do not require branch to be up to date before merging', ruleset_branch_default_testpipeline = github.RepositoryRuleset( - "ruleset_branch_default_testpipeline", + f"ruleset_branch_default_{NAME}", bypass_actors=[ # 'branch_master_enforce_admins' => 'master branch: do not enforce rules for admins', github.RepositoryRulesetBypassActorArgs( @@ -106,7 +106,7 @@ ), enforcement="active", name="master", - repository="testpipeline", + repository=NAME, rules=github.RepositoryRulesetRulesArgs( deletion=True, non_fast_forward=True, @@ -126,7 +126,7 @@ ) # TODO 'branch_dev_strict_updates' => 'dev branch: do not require branch to be up to date before merging', ruleset_branch_dev_testpipeline = github.RepositoryRuleset( - "ruleset_branch_dev_testpipeline", + f"ruleset_branch_dev_{NAME}", # 'branch_dev_enforce_admins' => 'dev branch: do not enforce rules for admins', bypass_actors=[ github.RepositoryRulesetBypassActorArgs( @@ -148,7 +148,7 @@ ), enforcement="active", name="dev", - repository="testpipeline", + repository=NAME, rules=github.RepositoryRulesetRulesArgs( deletion=True, non_fast_forward=True, @@ -171,7 +171,7 @@ # TODO Double check # Template branch protection https://github.com/nf-core/website/blob/33acd6a2fab2bf9251e14212ce731ef3232b5969/public_html/pipeline_health.php#L509 ruleset_branch_template_testpipeline = github.RepositoryRuleset( - "ruleset_branch_TEMPLATE_testpipeline", + f"ruleset_branch_TEMPLATE_{NAME}", bypass_actors=[ github.RepositoryRulesetBypassActorArgs( actor_id=CORE_TEAM_ID, @@ -188,7 +188,7 @@ ), enforcement="active", name="template", - repository="testpipeline", + repository=NAME, rules=github.RepositoryRulesetRulesArgs( deletion=True, non_fast_forward=True, @@ -199,15 +199,15 @@ ) # 'team_contributors' => 'Write access for nf-core/contributors', contributors_team_repo_testpipeline = github.TeamRepository( - "contributors_team_repo_testpipeline", + f"contributors_team_repo_{NAME}", team_id="contributors", - repository="testpipeline", + repository=NAME, permission="push", ) # 'team_core' => 'Admin access for nf-core/core', core_team_repo_testpipeline = github.TeamRepository( - "core_team_repo_testpipeline", + f"core_team_repo_{NAME}", team_id="core", - repository="testpipeline", + repository=NAME, permission="admin", ) From 71e576204a63eb4f15d8a4a40b6c906112142a4a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 22:14:52 -0500 Subject: [PATCH 32/40] Remove protect on repo --- pulumi/github/repos/pipelines/testpipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pulumi/github/repos/pipelines/testpipeline.py b/pulumi/github/repos/pipelines/testpipeline.py index 41ce3a8..38ad7bf 100644 --- a/pulumi/github/repos/pipelines/testpipeline.py +++ b/pulumi/github/repos/pipelines/testpipeline.py @@ -57,7 +57,6 @@ ), visibility="public", topics=TOPICS, # 'repo_keywords' => 'Minimum keywords set', - opts=pulumi.ResourceOptions(protect=True), ) From 78e9aabd8b347c6ac9260c29a1db452265e16faf Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 22:27:29 -0500 Subject: [PATCH 33/40] ci: Add template workflow for repos --- .github/workflows/repos.yml | 77 +++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/repos.yml diff --git a/.github/workflows/repos.yml b/.github/workflows/repos.yml new file mode 100644 index 0000000..3e6b532 --- /dev/null +++ b/.github/workflows/repos.yml @@ -0,0 +1,77 @@ +name: Preview or update Pulumi app nf-core/github-repos/prod +on: + push: + branches: + - main + paths: + - "pulumi/github/repos/**/*" + pull_request: + branches: + - main + paths: + - "pulumi/github/repos/**/*" + +# These are the environment variables that jobs in the workflow have access to. +# By defining them here, all jobs/steps will have access to these variables. +env: + # IMPORTANT! You must map the env vars for your cloud provider here even though you add them as secrets + # to this repository. + # See the setup page for cloud providers here: https://www.pulumi.com/docs/intro/cloud-providers/. + # For example, if you are using AWS, then you should add the following: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} + PULUMI_STACK_NAME: nf-core/github-repos/prod + PULUMI_WORKING_DIRECTORY: pulumi/github/repos/ + +jobs: + pulumi: + name: Pulumi + runs-on: ubuntu-latest + steps: + # Turnstyle is used to prevent multiple push jobs from running at the same time. We + # limit it to push jobs to allow PR jobs to run concurrently. + - name: Turnstyle + if: ${{ github.event_name == 'push' }} + uses: softprops/turnstyle@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v2 + + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.6.x + + - run: pip install -r requirements.txt + working-directory: ${{ env.PULUMI_WORKING_DIRECTORY }} + + - name: PR previews + if: ${{ github.event_name == 'pull_request' }} + uses: pulumi/actions@v3 + with: + command: preview + stack-name: ${{ env.PULUMI_STACK_NAME }} + work-dir: ${{ env.PULUMI_WORKING_DIRECTORY }} + + - name: Apply infrastructure update + if: ${{ github.event_name == 'push' }} + uses: pulumi/actions@v3 + with: + command: up + stack-name: ${{ env.PULUMI_STACK_NAME }} + work-dir: ${{ env.PULUMI_WORKING_DIRECTORY }} + + # If you'd like to run your Pulumi application outside of the official GitHub Action + + #- name: Install Pulumi CLI + # uses: pulumi/setup-pulumi@v2 + + #- name: PR previews + # run: pulumi preview -s $PULUMI_STACK_NAME --cwd $PULUMI_WORKING_DIRECTORY + # if: ${{ github.event_name == 'pull_request' }} + + #- name: Apply infrastructure update + # run: pulumi update --yes -s $PULUMI_STACK_NAME --cwd $PULUMI_WORKING_DIRECTORY + # if: ${{ github.event_name == 'push' }} From 855be9b67f1445412338c5fd6dffb1fcc758e7d6 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 22:32:37 -0500 Subject: [PATCH 34/40] ci: Remove python-version --- .github/workflows/repos.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/repos.yml b/.github/workflows/repos.yml index 3e6b532..b7b17b8 100644 --- a/.github/workflows/repos.yml +++ b/.github/workflows/repos.yml @@ -40,9 +40,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: 3.6.x + uses: actions/setup-python@v5 - run: pip install -r requirements.txt working-directory: ${{ env.PULUMI_WORKING_DIRECTORY }} From 77019ba6581f4a49db4f15443ae5793e53eb290e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Jul 2024 22:45:20 -0500 Subject: [PATCH 35/40] style: Run pre-commit --- pulumi/github/repos/import_by_hand.py | 2 -- pulumi/github/repos/loop_example.py | 2 -- pulumi/github/teams/__main__.py | 8 +++++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pulumi/github/repos/import_by_hand.py b/pulumi/github/repos/import_by_hand.py index b28f9c2..7a723a9 100644 --- a/pulumi/github/repos/import_by_hand.py +++ b/pulumi/github/repos/import_by_hand.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -import yaml - import pulumi import pulumi_github as github diff --git a/pulumi/github/repos/loop_example.py b/pulumi/github/repos/loop_example.py index 268cec4..58be0c1 100644 --- a/pulumi/github/repos/loop_example.py +++ b/pulumi/github/repos/loop_example.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -import yaml - import pulumi import pulumi_github as github diff --git a/pulumi/github/teams/__main__.py b/pulumi/github/teams/__main__.py index d825b3a..739eaa4 100644 --- a/pulumi/github/teams/__main__.py +++ b/pulumi/github/teams/__main__.py @@ -24,7 +24,8 @@ def setup_team(self, team, parent_team=None): for user in members: # Add a user to the newly created team - team_membership = github.TeamMembership( + # team_membership = + github.TeamMembership( f"{team['name']}-{user['name']}", team_id=team_resource, username=user["name"], @@ -37,7 +38,8 @@ def setup_team(self, team, parent_team=None): continue # Associate a repository with the team - team_repository = github.TeamRepository( + # team_repository = + github.TeamRepository( f"{team['name']}-{repo['name']}", team_id=team_resource, repository=self._repos[repo["name"]], @@ -75,4 +77,4 @@ def __init__(self, org_file): self.setup_team(team) -Organization("org.yaml") \ No newline at end of file +Organization("org.yaml") From aa24ea0dc3d8d41218158ddc9a032de946c00ecf Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:56:01 -0500 Subject: [PATCH 36/40] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .github/workflows/repos.yml | 13 +------------ docs/1password.md | 11 ++++++++--- pulumi/github/repos/Pulumi.yaml | 2 +- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/repos.yml b/.github/workflows/repos.yml index b7b17b8..74f7eca 100644 --- a/.github/workflows/repos.yml +++ b/.github/workflows/repos.yml @@ -41,6 +41,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 + cache: 'pip' - run: pip install -r requirements.txt working-directory: ${{ env.PULUMI_WORKING_DIRECTORY }} @@ -61,15 +62,3 @@ jobs: stack-name: ${{ env.PULUMI_STACK_NAME }} work-dir: ${{ env.PULUMI_WORKING_DIRECTORY }} - # If you'd like to run your Pulumi application outside of the official GitHub Action - - #- name: Install Pulumi CLI - # uses: pulumi/setup-pulumi@v2 - - #- name: PR previews - # run: pulumi preview -s $PULUMI_STACK_NAME --cwd $PULUMI_WORKING_DIRECTORY - # if: ${{ github.event_name == 'pull_request' }} - - #- name: Apply infrastructure update - # run: pulumi update --yes -s $PULUMI_STACK_NAME --cwd $PULUMI_WORKING_DIRECTORY - # if: ${{ github.event_name == 'push' }} diff --git a/docs/1password.md b/docs/1password.md index 757d666..6159912 100644 --- a/docs/1password.md +++ b/docs/1password.md @@ -5,12 +5,17 @@ [How to use 1Password with different accounts automatically](https://developer.1password.com/docs/cli/shell-plugins/multiple-accounts/) ```console -$ cd ~/src/nf-core +cd ~/src/nf-core + +op signin -~/src/nf-core $ op signin # Select nf-core -~/src/nf-core $ op plugin init pulumi +op plugin init pulumi +``` + +This should result in: +``` Pulumi CLI Authenticate with Pulumi Personal Access Token. diff --git a/pulumi/github/repos/Pulumi.yaml b/pulumi/github/repos/Pulumi.yaml index 6154903..ea14373 100644 --- a/pulumi/github/repos/Pulumi.yaml +++ b/pulumi/github/repos/Pulumi.yaml @@ -3,4 +3,4 @@ runtime: name: python options: virtualenv: venv -description: Managing GitHub repos +description: Managing nf-core GitHub repos From 5fdd914afd0b602eaa6d61d378470ea40ad62960 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 22 Jul 2024 12:55:24 -0500 Subject: [PATCH 37/40] chore: Add 1password link --- pulumi/github/repos/Pulumi.prod.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/pulumi/github/repos/Pulumi.prod.yaml b/pulumi/github/repos/Pulumi.prod.yaml index 37afc74..d0ca5fc 100644 --- a/pulumi/github/repos/Pulumi.prod.yaml +++ b/pulumi/github/repos/Pulumi.prod.yaml @@ -1,4 +1,5 @@ config: github:owner: nf-core +# https://start.1password.com/open/i?a=O5GICFDKPNABLLVGMKBL5JWDWA&v=rdfcz6oy6qxxrc4clu467a7dmm&i=ttqz63qvlr5qfwfde424nbl4re&h=nf-core.1password.eu environment: - github-prod From 2e802463359e984b6230f012785c7f81fca073f5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 22 Jul 2024 13:01:52 -0500 Subject: [PATCH 38/40] docs: Clean up README moving planning to GH issue Co-authored-by: mashehu --- pulumi/github/repos/README.md | 47 ++++++----------------------------- 1 file changed, 7 insertions(+), 40 deletions(-) diff --git a/pulumi/github/repos/README.md b/pulumi/github/repos/README.md index 933c12b..b005a8c 100644 --- a/pulumi/github/repos/README.md +++ b/pulumi/github/repos/README.md @@ -1,48 +1,15 @@ # Repos -Goal is to replace https://oldsite.nf-co.re/pipeline_health +Replaces the automatic rule enforcement from the [old Pipeline Health PHP code](https://github.com/nf-core/website/blob/old-site/public_html/pipeline_health.php), -This repo will be the "Actions" section at the bottom. We can then create a reporting page if we really need to see all the green checks +[Main GitHub Issue](https://github.com/nf-core/ops/issues/5) +[Tracking Milestone](https://github.com/nf-core/ops/milestone/1) -[Old Pipeline Health PHP code](https://github.com/nf-core/website/blob/old-site/public_html/pipeline_health.php) +## Useful Docs -[New Pipeline Health page](https://github.com/nf-core/website/blob/main/sites/pipelines/src/pages/pipeline_health.astro) - -## Initial Roll-out - -The new pipelines that are broken: - -- demo -- testpipeline - -- denovotranscript -- meerpipe -- pairgenomealign -- phaseimpute -- reportho - -Maybe: - -- scdownstream -- scnanoseq - -### Plan - -#### Short-term - -1. [ ] Import a pipeline that has all the right settings -2. [ ] Fix the 5 pipelines above with the correct settings from the "model" repo -3. [ ] Keep importing new pipelines until we gain confidence in it. - -#### Long-term - -1. Wrangle in `core_repos` -2. Roll out to all pipelines -3. Switch all repos to main - -## Docs - -https://www.pulumi.com/registry/packages/github/api-docs/repository/ +- https://www.pulumi.com/registry/packages/github/api-docs/repository/ +- [Old Pipeline Health PHP code](https://github.com/nf-core/website/blob/old-site/public_html/pipeline_health.php) +- [New Pipeline Health page](https://github.com/nf-core/website/blob/main/sites/pipelines/src/pages/pipeline_health.astro) ### Importing Repos From d4758586706236fcc3c30ebc63d7bde01afe4032 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 22 Jul 2024 13:30:31 -0500 Subject: [PATCH 39/40] refactor: Use Pulumi ESC for dev --- pulumi/github/repos/Pulumi.dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulumi/github/repos/Pulumi.dev.yaml b/pulumi/github/repos/Pulumi.dev.yaml index b222191..3a722fd 100644 --- a/pulumi/github/repos/Pulumi.dev.yaml +++ b/pulumi/github/repos/Pulumi.dev.yaml @@ -1,5 +1,5 @@ config: github:owner: nf-core-tf # https://start.1password.com/open/i?a=O5GICFDKPNABLLVGMKBL5JWDWA&v=rdfcz6oy6qxxrc4clu467a7dmm&i=4ajrv44kc5lcbboa37fr5oydla&h=nf-core.1password.eu - github:token: - secure: AAABAFMgBNyCNuYsps6YVPV2L7Ji5qBJj0omEQQa9HrdhT2iHo3ex0e9NsDER3Q04itGiY698X/ZQCnTM2zu9op3tcjmzfITdHxGy0FGATuUFamYsSiztHrNAKiIEJ9E0M4Al8/yJeB6X4BXvkLEgik/I+GPvZIXK3tE65Q= +environment: + - github-nf-core-tf From 9eebe4b152bc8adb740c7d620d9aaff1e308f660 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 23 Jul 2024 11:44:07 -0500 Subject: [PATCH 40/40] docs: Add some nf-core Pulumi starter info --- docs/pulumi.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/pulumi.md diff --git a/docs/pulumi.md b/docs/pulumi.md new file mode 100644 index 0000000..ad430fb --- /dev/null +++ b/docs/pulumi.md @@ -0,0 +1,72 @@ +# Pulumi + +nf-core specific docs, links and guides. + +## Quick Start + +### Repo structure + +This repo is a "Monorepo", basically a bunch of smaller projects inside of one bigger project. + +```console +tree -L 1 pulumi +pulumi +├── AWSMegatests +├── github +├── repo-backups +├── sentieon-license-server +└── test-datasets +``` + +Each of these are their own projects. + +### Install Pulumi + +[Here's the official guide](https://www.pulumi.com/docs/clouds/aws/get-started/) + +### Working with this repo + + + +1. Open up the project you want to make a change to +2. Make the change (Probably in `__main.py__`) +3. If you have a Pulumi cloud account in the nf-core org `pulumi preview` and `pulumi up` should work locally +4. Create a branch in the repo and make a PR, and a preview of the deployment should get ran. + +## Terminology + +Pulumi is pretty heavy on the terms and it was kinda confusing. A hierarchy kinda helps + +``` +Projects +├── Stacks +├──── Deployments +├──── Resources +Environments +``` + +### Projects + +Each directory in `pulumi` is a project. + +#### Stacks + +Each project can have multiple stacks. For example, `dev`, `prod`, `test`. + +Official quote: + +> What are projects and stacks? Pulumi projects and stacks let you organize Pulumi code. Consider a Pulumi project to be analogous to a GitHub repo—a single place for code—and a stack to be an instance of that code with a separate configuration. For instance, project foo may have multiple stacks for different deployment environments (dev, test, or prod), or perhaps for different cloud configurations (geographic region for example). See Organizing Projects and Stacks for some best practices on organizing your Pulumi projects and stacks. + +https://www.pulumi.com/docs/using-pulumi/organizing-projects-stacks/ + +##### Deployments + +Everytime you push to main in this repo a new deployment of the stack goes out. + +##### Resources + +These are individual pieces of infrastructure. An EC2 instance, a VPC, a GitHub repo, a GitHub team are some examples. + +### Environments + +This is Pulumi's hosted Secrete management. I'm thinking of these like, well "Environments". The nf-core AWS, the nf-core Azure, nf-core GCP, nf-core GitHub org, the nf-core-tf GitHub org.