From e519b3ff1b99b9fbbfc41adab0566e2a461fc0a7 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 27 Feb 2025 11:22:11 +0100 Subject: [PATCH 1/6] nf-test sharding --- .github/actions/get-shards/action.yml | 66 +++++++++ .github/actions/nf-test/action.yml | 97 +++++++++++++ .github/workflows/ci.yml | 136 ------------------ .github/workflows/nf-test.yml | 131 +++++++++++++++++ .../local/utils_nf_aggregate/tests/tags.yml | 2 - workflows/nf_aggregate/tests/tags.yml | 2 - 6 files changed, 294 insertions(+), 140 deletions(-) create mode 100644 .github/actions/get-shards/action.yml create mode 100644 .github/actions/nf-test/action.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/nf-test.yml delete mode 100644 subworkflows/local/utils_nf_aggregate/tests/tags.yml delete mode 100644 workflows/nf_aggregate/tests/tags.yml diff --git a/.github/actions/get-shards/action.yml b/.github/actions/get-shards/action.yml new file mode 100644 index 0000000..52356bf --- /dev/null +++ b/.github/actions/get-shards/action.yml @@ -0,0 +1,66 @@ +name: "Get number of shards" +description: "Get the number of nf-test shards for the current CI job" +inputs: + max_shards: + description: "Maximum number of shards allowed" + required: true + paths: + description: "Component paths to test" + required: false +outputs: + shard: + description: "Array of shard numbers" + value: ${{ steps.shards.outputs.shard }} + total_shards: + description: "Total number of shards" + value: ${{ steps.shards.outputs.total_shards }} +runs: + using: "composite" + steps: + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + - name: Get number of shards + id: shards + shell: bash + run: | + # Run nf-test with dynamic parameter + nftest_output=$(nf-test test \ + --dry-run \ + --profile +docker \ + --filter module,function,workflow,pipeline \ + --ci \ + --changed-since HEAD^) || { + echo "nf-test command failed with exit code $?" + echo "Full output: $nftest_output" + exit 1 + } + echo "nf-test dry-run output: $nftest_output" + + # Default values for shard and total_shards + shard="[]" + total_shards=0 + + # Check if there are related tests + if echo "$nftest_output" | grep -q 'No tests to execute'; then + echo "No related tests found." + else + # Extract the number of related tests + number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p') + if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then + shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} )) + shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .) + total_shards="$shards_to_run" + else + echo "Unexpected output format. Falling back to default values." + fi + fi + + # Write to GitHub Actions outputs + echo "shard=$shard" >> $GITHUB_OUTPUT + echo "total_shards=$total_shards" >> $GITHUB_OUTPUT + + # Debugging output + echo "Final shard array: $shard" + echo "Total number of shards: $total_shards" diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml new file mode 100644 index 0000000..fdc8e24 --- /dev/null +++ b/.github/actions/nf-test/action.yml @@ -0,0 +1,97 @@ +name: "nf-test Action" +description: "Runs nf-test with common setup steps" +inputs: + profile: + description: "Profile to use" + required: true + shard: + description: "Shard number for this CI job" + required: true + total_shards: + description: "Total number of test shards(NOT the total number of matrix jobs)" + required: true + paths: + description: "Test paths" + required: true + +runs: + using: "composite" + steps: + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + with: + version: "${{ env.NXF_VERSION }}" + + - name: Set up Python + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5 + with: + python-version: "3.11" + + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: "${{ env.NFT_VER }}" + + - name: Install pdiff + shell: bash + run: | + python -m pip install pdiff + + # TODO Skip failing conda tests and document their failures + # https://github.com/nf-core/modules/issues/7017 + - name: Run nf-test + shell: bash + env: + NFT_DIFF: ${{ env.NFT_DIFF }} + NFT_DIFF_ARGS: ${{ env.NFT_DIFF_ARGS }} + NFT_WORKDIR: ${{ env.NFT_WORKDIR }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} + run: | + nf-test test \ + --profile=+${{ inputs.profile }} \ + --tap=test.tap \ + --verbose \ + --ci \ + --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ + --filter module,function,workflow,pipeline + + # Save the absolute path of the test.tap file to the output + echo "tap_file_path=$(realpath test.tap)" >> $GITHUB_OUTPUT + + - name: Generate test summary + if: always() + shell: bash + run: | + # Add header if it doesn't exist (using a token file to track this) + if [ ! -f ".summary_header" ]; then + echo "# 🚀 nf-test results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Status | Test Name | Profile | Shard |" >> $GITHUB_STEP_SUMMARY + echo "|:------:|-----------|---------|-------|" >> $GITHUB_STEP_SUMMARY + touch .summary_header + fi + + if [ -f test.tap ]; then + while IFS= read -r line; do + if [[ $line =~ ^ok ]]; then + test_name="${line#ok }" + # Remove the test number from the beginning + test_name="${test_name#* }" + echo "| ✅ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + elif [[ $line =~ ^not\ ok ]]; then + test_name="${line#not ok }" + # Remove the test number from the beginning + test_name="${test_name#* }" + echo "| ❌ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + fi + done < test.tap + else + echo "| ⚠️ | No test results found | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + fi + + - name: Clean up + if: always() + shell: bash + run: | + sudo rm -rf /home/ubuntu/tests/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 842a9a8..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,136 +0,0 @@ -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors -name: nf-core CI -on: - pull_request: - release: - types: [published] - merge_group: - types: - - checks_requested - branches: - - master - - dev - -env: - NXF_ANSI_LOG: false - NFT_VER: "0.9.2" - NFT_WORKDIR: "~" - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" - TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} - -concurrency: - group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" - cancel-in-progress: true - -jobs: - changes: - name: Check for changes - runs-on: ubuntu-latest - outputs: - nf_test_files: ${{ steps.list.outputs.components }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: List nf-test files - id: list - uses: adamrtalbot/detect-nf-test-changes@v0.0.3 - with: - head: ${{ github.sha }} - base: origin/${{ github.base_ref }} - include: .github/include.yaml - - - name: print list of nf-test files - run: | - echo ${{ steps.list.outputs.components }} - - test: - name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} - needs: [changes] - if: needs.changes.outputs.nf_test_files != '[]' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - NXF_VER: - - "latest-everything" - nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"] - profile: - - "docker" - - steps: - - name: Check out pipeline code - uses: actions/checkout@v4 - - - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: "x64" - - - name: Install pdiff to see diff between nf-test snapshots - run: | - python -m pip install --upgrade pip - pip install pdiff - - - name: Cache nf-test installation - id: cache-software - uses: actions/cache@v3 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-${{ env.NFT_VER }}-nftest - - - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ - - - name: Run nf-test - run: | - nf-test test --verbose ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - - - uses: pcolby/tap-summary@v1 - with: - path: >- - test.tap - - - name: Output log on failure - if: failure() - run: | - sudo apt install bat > /dev/null - batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log - - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: test.xml - - confirm-pass: - runs-on: ubuntu-latest - needs: - - changes - - test - if: always() - steps: - - name: All tests ok - if: ${{ !contains(needs.*.result, 'failure') }} - run: exit 0 - - name: One or more tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 - - - name: debug-print - if: always() - run: | - echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 0000000..6e05b45 --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,131 @@ +name: Run nf-test +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [dev] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "ubuntu-latest" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFT_VER: "0.9.2" + NFT_WORKDIR: "~" + NXF_ANSI_LOG: false + TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} + +jobs: + nf-test-changes: + name: nf-test-changes + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + outputs: + shard: ${{ steps.set-shards.outputs.shard }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} + steps: + - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + + - name: get number of shards + id: set-shards + uses: ./.github/actions/get-shards + env: + NFT_VER: ${{ env.NFT_VER }} + with: + max_shards: 7 + + - name: debug + run: | + echo ${{ steps.set-shards.outputs.shard }} + echo ${{ steps.set-shards.outputs.total_shards }} + + nf-test: + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + name: "${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}" + needs: [nf-test-changes] + if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }} + profile: [docker] + NXF_VER: + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + - "24.10.2" + - "latest-everything" + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + + - name: Run nf-test + uses: ./.github/actions/nf-test + env: + NFT_DIFF: ${{ env.NFT_DIFF }} + NFT_DIFF_ARGS: ${{ env.NFT_DIFF_ARGS }} + NFT_WORKDIR: ${{ env.NFT_WORKDIR }} + TOWER_ACCESS_TOKEN: ${{ env.TOWER_ACCESS_TOKEN }} + with: + profile: ${{ matrix.profile }} + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} + + confirm-pass: + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + needs: [nf-test] + if: always() + steps: + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: One or more tests cancelled + if: ${{ contains(needs.*.result, 'cancelled') }} + run: exit 1 + + - name: All tests ok + if: ${{ contains(needs.*.result, 'success') }} + run: exit 0 + + - name: debug-print + if: always() + run: | + echo "::group::DEBUG: `needs` Contents" + echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" + echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" + echo "::endgroup::" + + - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner + if: always() + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ diff --git a/subworkflows/local/utils_nf_aggregate/tests/tags.yml b/subworkflows/local/utils_nf_aggregate/tests/tags.yml deleted file mode 100644 index 4fc1520..0000000 --- a/subworkflows/local/utils_nf_aggregate/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -subworkflows_local_utils_nf_aggregate: - - subworkflows/local/utils_nf_aggregate/** diff --git a/workflows/nf_aggregate/tests/tags.yml b/workflows/nf_aggregate/tests/tags.yml deleted file mode 100644 index 50048dc..0000000 --- a/workflows/nf_aggregate/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -workflows_nf_aggregate: - - workflows/nf_aggregate/** From 19a2031d2162f9d7ba38a621cae8d5d38632f487 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 27 Feb 2025 11:25:45 +0100 Subject: [PATCH 2/6] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6df445..298f4fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [PR #74](https://github.com/seqeralabs/nf-aggregate/pull/74) - Add process for generating Benchmark reports - [PR #75](https://github.com/seqeralabs/nf-aggregate/pull/75) - Skip failed jobs in benchmarking report - [PR #78](https://github.com/seqeralabs/nf-aggregate/pull/78) - Sync with nf-core tools 3.2.0 +- [PR #79](https://github.com/seqeralabs/nf-aggregate/pull/79) - nf-test auto sharding + better pipeline level test ## [[0.5.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.5.0)] - 2024-11-12 From 1f959eb327b214cae968bfce8f48e4577bdd9cd3 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 27 Feb 2025 11:31:43 +0100 Subject: [PATCH 3/6] fix linting --- .nf-core.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.nf-core.yml b/.nf-core.yml index e19fb5b..8b5db26 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -8,6 +8,7 @@ lint: - .github/workflows/awstest.yml - .github/workflows/branch.yml - .github/workflows/linting_comment.yml + - .github/workflows/ci.yml - CODE_OF_CONDUCT.md - LICENSE - assets/email_template.html From 4e6e100045030e60e32085acb6919ca88c98770b Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 27 Feb 2025 13:28:42 +0100 Subject: [PATCH 4/6] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 298f4fb..0f5faed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [PR #74](https://github.com/seqeralabs/nf-aggregate/pull/74) - Add process for generating Benchmark reports - [PR #75](https://github.com/seqeralabs/nf-aggregate/pull/75) - Skip failed jobs in benchmarking report - [PR #78](https://github.com/seqeralabs/nf-aggregate/pull/78) - Sync with nf-core tools 3.2.0 -- [PR #79](https://github.com/seqeralabs/nf-aggregate/pull/79) - nf-test auto sharding + better pipeline level test +- [PR #79](https://github.com/seqeralabs/nf-aggregate/pull/79) - nf-test auto sharding ## [[0.5.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.5.0)] - 2024-11-12 From b969276cb058aad9909948a26ced5e921e9adb41 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 27 Feb 2025 13:30:40 +0100 Subject: [PATCH 5/6] remove unused test tags --- modules/local/plot_run_gantt/tests/tags.yml | 2 -- modules/local/seqera_runs_dump/tests/tags.yml | 2 -- modules/nf-core/multiqc/tests/tags.yml | 2 -- subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml | 2 -- tests/tags.yml | 5 ----- 5 files changed, 13 deletions(-) delete mode 100644 modules/local/plot_run_gantt/tests/tags.yml delete mode 100644 modules/local/seqera_runs_dump/tests/tags.yml delete mode 100644 modules/nf-core/multiqc/tests/tags.yml delete mode 100644 subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml delete mode 100644 tests/tags.yml diff --git a/modules/local/plot_run_gantt/tests/tags.yml b/modules/local/plot_run_gantt/tests/tags.yml deleted file mode 100644 index ce481c7..0000000 --- a/modules/local/plot_run_gantt/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -plot_run_gantt: - - modules/local/plot_run_gantt/** diff --git a/modules/local/seqera_runs_dump/tests/tags.yml b/modules/local/seqera_runs_dump/tests/tags.yml deleted file mode 100644 index 2983b13..0000000 --- a/modules/local/seqera_runs_dump/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -seqera_runs_dump: - - modules/local/seqera_runs_dump/** diff --git a/modules/nf-core/multiqc/tests/tags.yml b/modules/nf-core/multiqc/tests/tags.yml deleted file mode 100644 index bea6c0d..0000000 --- a/modules/nf-core/multiqc/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -multiqc: - - modules/nf-core/multiqc/** diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml deleted file mode 100644 index f847611..0000000 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -subworkflows/utils_nextflow_pipeline: - - subworkflows/nf-core/utils_nextflow_pipeline/** diff --git a/tests/tags.yml b/tests/tags.yml deleted file mode 100644 index c2e26f4..0000000 --- a/tests/tags.yml +++ /dev/null @@ -1,5 +0,0 @@ -pipeline_nf-aggregate: - - "**.nf" - - "**.config" - - "**.nf.test" - - "**.json" From afc0d7cdc4be9e1f33009a7e6d38c91aa27cbd02 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 27 Feb 2025 13:31:56 +0100 Subject: [PATCH 6/6] update nf-test config file --- nf-test.config | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nf-test.config b/nf-test.config index 97ad11b..ac6caff 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,13 +1,16 @@ config { - // location for all nf-tests + // location for all nf-test tests testsDir "." // nf-test directory including temporary files for each test - workDir System.getenv("TMPDIR") ?: '.nf-test' + workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" // location of an optional nextflow.config file specific for executing tests configFile "tests/nextflow.config" - // run all test with the defined docker profile from the main nextflow.config + // run all test with defined profile(s) from the main nextflow.config profile "test" + + // list of filenames or patterns that should be trigger a full test run + triggers 'nextflow.config', 'nf-test.config', 'tests/nextflow.config' }