Skip to content

Commit 626edee

Browse files
committed
feat!: --merge-reports to support coverage
1 parent a4ec583 commit 626edee

File tree

16 files changed

+288
-39
lines changed

16 files changed

+288
-39
lines changed

docs/guide/features.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -232,10 +232,12 @@ test('my types work properly', () => {
232232
## Sharding
233233

234234
Run tests on different machines using [`--shard`](/guide/cli#shard) and [`--reporter=blob`](/guide/reporters#blob-reporter) flags.
235-
All test results can be merged at the end of your CI pipeline using `--merge-reports` command:
235+
All test and coverage results can be merged at the end of your CI pipeline using `--merge-reports` command:
236236

237237
```bash
238238
vitest --shard=1/2 --reporter=blob
239239
vitest --shard=2/2 --reporter=blob
240-
vitest --merge-reports --reporter=junit
240+
vitest --merge-reports --reporter=junit --coverage.reporter=text
241241
```
242+
243+
See [`Improving Performance | Sharding`](/guide/improving-performance#sharding) for more information.

docs/guide/improving-performance.md

+99
Original file line numberDiff line numberDiff line change
@@ -72,3 +72,102 @@ export default defineConfig({
7272
})
7373
```
7474
:::
75+
76+
## Sharding
77+
78+
Test sharding means running a small subset of test cases at a time. It can be useful when you have multiple machines that could be used to run tests simultaneously.
79+
80+
To split Vitest tests on multiple different runs, use [`--shard`](/guide/cli#shard) option with [`--reporter=blob`](/guide/reporters#blob-reporter) option:
81+
82+
```sh
83+
vitest run --reporter=blob --shard=1/3 # 1st machine
84+
vitest run --reporter=blob --shard=2/3 # 2nd machine
85+
vitest run --reporter=blob --shard=3/3 # 3rd machine
86+
```
87+
88+
Collect the results stored in `.vitest-reports` directory from each machine and merge them with [`--merge-reports`](/guide/cli#merge-reports) option:
89+
90+
```sh
91+
vitest --merge-reports
92+
```
93+
94+
<details>
95+
<summary>Github action example</summary>
96+
97+
```yaml
98+
# Inspired from https://playwright.dev/docs/test-sharding
99+
name: Tests
100+
on:
101+
push:
102+
branches:
103+
- main
104+
jobs:
105+
tests:
106+
runs-on: ubuntu-latest
107+
strategy:
108+
matrix:
109+
shardIndex: [1, 2, 3, 4]
110+
shardTotal: [4]
111+
steps:
112+
- uses: actions/checkout@v4
113+
- uses: actions/setup-node@v4
114+
- name: Install dependencies
115+
run: pnpm i
116+
117+
- name: Run tests
118+
run: pnpm run test --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
119+
120+
- name: Upload blob report to GitHub Actions Artifacts
121+
if: ${{ !cancelled() }}
122+
uses: actions/upload-artifact@v4
123+
with:
124+
name: blob-report-${{ matrix.shardIndex }}
125+
path: blob-report
126+
retention-days: 1
127+
128+
merge-reports:
129+
if: ${{ !cancelled() }}
130+
needs: [tests]
131+
132+
runs-on: ubuntu-latest
133+
steps:
134+
- uses: actions/checkout@v4
135+
- uses: actions/setup-node@v4
136+
- name: Install dependencies
137+
run: pnpm i
138+
139+
- name: Download blob reports from GitHub Actions Artifacts
140+
uses: actions/download-artifact@v4
141+
with:
142+
path: all-blob-reports
143+
pattern: blob-report-*
144+
merge-multiple: true
145+
146+
- name: Merge reports
147+
run: npx vitest --merge-reports
148+
```
149+
150+
</details>
151+
152+
:::tip
153+
Test sharding can also become useful on high CPU-count machines.
154+
155+
Vitest will run only a single Vite server in its main thread. Rest of the threads are used to run test files.
156+
In a high CPU-count machine the main thread can become a bottleneck as it cannot handle all the requests coming from the threads. For example in 32 CPU machine the main thread is responsible to handle load coming from 31 test threads.
157+
158+
To reduce the load from main thread's Vite server you can use test sharding. The load can be balanced on multiple Vite server.
159+
160+
```sh
161+
# Example for splitting tests on 32 CPU to 4 shards.
162+
# As each process needs 1 main thread, there's 7 threads for test runners (1+7)*4 = 32
163+
# Use VITEST_MAX_THREADS or VITEST_MAX_FORKS depending on the pool:
164+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=1/4 & \
165+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=2/4 & \
166+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=3/4 & \
167+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=4/4 & \
168+
wait # https://man7.org/linux/man-pages/man2/waitpid.2.html
169+
170+
vitest --merge-reports
171+
```
172+
173+
:::

packages/coverage-istanbul/src/provider.ts

+31-11
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
162162
this.pendingPromises = []
163163
}
164164

165-
async reportCoverage({ allTestsRun }: ReportContext = {}) {
165+
async generateCoverage({ allTestsRun }: ReportContext) {
166166
const coverageMap = libCoverage.createCoverageMap({})
167167
let index = 0
168168
const total = this.pendingPromises.length
@@ -202,6 +202,29 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
202202
coverageMap.merge(await transformCoverage(uncoveredCoverage))
203203
}
204204

205+
return coverageMap
206+
}
207+
208+
async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
209+
await this.generateReports(
210+
coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
211+
allTestsRun,
212+
)
213+
214+
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
215+
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
216+
217+
if (!keepResults) {
218+
this.coverageFiles = new Map()
219+
await fs.rm(this.coverageFilesDirectory, { recursive: true })
220+
221+
// Remove empty reports directory, e.g. when only text-reporter is used
222+
if (readdirSync(this.options.reportsDirectory).length === 0)
223+
await fs.rm(this.options.reportsDirectory, { recursive: true })
224+
}
225+
}
226+
227+
async generateReports(coverageMap: CoverageMap, allTestsRun: boolean | undefined) {
205228
const context = libReport.createContext({
206229
dir: this.options.reportsDirectory,
207230
coverageMap,
@@ -248,21 +271,18 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
248271
})
249272
}
250273
}
274+
}
251275

252-
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
253-
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
276+
async mergeReports(coverageMaps: unknown[]) {
277+
const coverageMap = libCoverage.createCoverageMap({})
254278

255-
if (!keepResults) {
256-
this.coverageFiles = new Map()
257-
await fs.rm(this.coverageFilesDirectory, { recursive: true })
279+
for (const coverage of coverageMaps)
280+
coverageMap.merge(coverage as CoverageMap)
258281

259-
// Remove empty reports directory, e.g. when only text-reporter is used
260-
if (readdirSync(this.options.reportsDirectory).length === 0)
261-
await fs.rm(this.options.reportsDirectory, { recursive: true })
262-
}
282+
await this.generateReports(coverageMap, true)
263283
}
264284

265-
async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
285+
private async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
266286
const allFiles = await this.testExclude.glob(this.ctx.config.root)
267287
let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file))
268288

packages/coverage-v8/src/provider.ts

+33-13
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
148148
this.pendingPromises.push(promise)
149149
}
150150

151-
async reportCoverage({ allTestsRun }: ReportContext = {}) {
152-
if (provider === 'stackblitz')
153-
this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
154-
151+
async generateCoverage({ allTestsRun }: ReportContext) {
155152
const coverageMap = libCoverage.createCoverageMap({})
156153
let index = 0
157154
const total = this.pendingPromises.length
@@ -193,6 +190,32 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
193190
coverageMap.merge(await transformCoverage(converted))
194191
}
195192

193+
return coverageMap
194+
}
195+
196+
async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
197+
if (provider === 'stackblitz')
198+
this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
199+
200+
await this.generateReports(
201+
coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
202+
allTestsRun,
203+
)
204+
205+
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
206+
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
207+
208+
if (!keepResults) {
209+
this.coverageFiles = new Map()
210+
await fs.rm(this.coverageFilesDirectory, { recursive: true })
211+
212+
// Remove empty reports directory, e.g. when only text-reporter is used
213+
if (readdirSync(this.options.reportsDirectory).length === 0)
214+
await fs.rm(this.options.reportsDirectory, { recursive: true })
215+
}
216+
}
217+
218+
async generateReports(coverageMap: CoverageMap, allTestsRun?: boolean) {
196219
const context = libReport.createContext({
197220
dir: this.options.reportsDirectory,
198221
coverageMap,
@@ -239,18 +262,15 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
239262
})
240263
}
241264
}
265+
}
242266

243-
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
244-
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
267+
async mergeReports(coverageMaps: unknown[]) {
268+
const coverageMap = libCoverage.createCoverageMap({})
245269

246-
if (!keepResults) {
247-
this.coverageFiles = new Map()
248-
await fs.rm(this.coverageFilesDirectory, { recursive: true })
270+
for (const coverage of coverageMaps)
271+
coverageMap.merge(coverage as CoverageMap)
249272

250-
// Remove empty reports directory, e.g. when only text-reporter is used
251-
if (readdirSync(this.options.reportsDirectory).length === 0)
252-
await fs.rm(this.options.reportsDirectory, { recursive: true })
253-
}
273+
await this.generateReports(coverageMap, true)
254274
}
255275

256276
private async getUntestedFiles(testedFiles: string[]): Promise<RawCoverage> {

packages/vitest/src/node/core.ts

+12-6
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ export class Vitest {
393393
if (this.reporters.some(r => r instanceof BlobReporter))
394394
throw new Error('Cannot merge reports when `--reporter=blob` is used. Remove blob reporter from the config first.')
395395

396-
const { files, errors } = await readBlobs(this.config.mergeReports, this.projects)
396+
const { files, errors, coverages } = await readBlobs(this.config.mergeReports, this.projects)
397397

398398
await this.report('onInit', this)
399399
await this.report('onPathsCollected', files.flatMap(f => f.filepath))
@@ -439,6 +439,8 @@ export class Vitest {
439439
process.exitCode = 1
440440

441441
await this.report('onFinished', files, errors)
442+
await this.initCoverageProvider()
443+
await this.coverageProvider?.mergeReports?.(coverages)
442444
}
443445

444446
async start(filters?: string[]) {
@@ -459,7 +461,9 @@ export class Vitest {
459461

460462
// if run with --changed, don't exit if no tests are found
461463
if (!files.length) {
462-
await this.reportCoverage(true)
464+
// Report coverage for uncovered files
465+
const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true })
466+
await this.reportCoverage(coverage, true)
463467

464468
this.logger.printNoTestFound(filters)
465469

@@ -645,8 +649,10 @@ export class Vitest {
645649
.finally(async () => {
646650
// can be duplicate files if different projects are using the same file
647651
const files = Array.from(new Set(specs.map(([, p]) => p)))
648-
await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors())
649-
await this.reportCoverage(allTestsRun)
652+
const coverage = await this.coverageProvider?.generateCoverage({ allTestsRun })
653+
654+
await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors(), coverage)
655+
await this.reportCoverage(coverage, allTestsRun)
650656

651657
this.runningPromise = undefined
652658
this.isFirstRun = false
@@ -946,12 +952,12 @@ export class Vitest {
946952
return Array.from(new Set(files))
947953
}
948954

949-
private async reportCoverage(allTestsRun: boolean) {
955+
private async reportCoverage(coverage: unknown, allTestsRun: boolean) {
950956
if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0)
951957
return
952958

953959
if (this.coverageProvider) {
954-
await this.coverageProvider.reportCoverage({ allTestsRun })
960+
await this.coverageProvider.reportCoverage(coverage, { allTestsRun })
955961
// notify coverage iframe reload
956962
for (const reporter of this.reporters) {
957963
if (reporter instanceof WebSocketReporter)

packages/vitest/src/node/reporters/blob.ts

+7-4
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export class BlobReporter implements Reporter {
2626
this.ctx = ctx
2727
}
2828

29-
async onFinished(files: File[] = [], errors: unknown[] = []) {
29+
async onFinished(files: File[] = [], errors: unknown[] = [], coverage: unknown) {
3030
let outputFile = this.options.outputFile ?? getOutputFile(this.ctx.config, 'blob')
3131
if (!outputFile) {
3232
const shard = this.ctx.config.shard
@@ -39,7 +39,7 @@ export class BlobReporter implements Reporter {
3939
return [project.getName(), [...project.server.moduleGraph.idToModuleMap.keys()]]
4040
})
4141

42-
const report = stringify([this.ctx.version, files, errors, moduleKeys] satisfies MergeReport)
42+
const report = stringify([this.ctx.version, files, errors, moduleKeys, coverage] satisfies MergeReport)
4343

4444
const reportFile = resolve(this.ctx.config.root, outputFile)
4545

@@ -62,8 +62,8 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
6262
const blobsFiles = await readdir(resolvedDir)
6363
const promises = blobsFiles.map(async (file) => {
6464
const content = await readFile(resolve(resolvedDir, file), 'utf-8')
65-
const [version, files, errors, moduleKeys] = parse(content) as MergeReport
66-
return { version, files, errors, moduleKeys }
65+
const [version, files, errors, moduleKeys, coverage] = parse(content) as MergeReport
66+
return { version, files, errors, moduleKeys, coverage }
6767
})
6868
const blobs = await Promise.all(promises)
6969

@@ -108,10 +108,12 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
108108
return time1 - time2
109109
})
110110
const errors = blobs.flatMap(blob => blob.errors)
111+
const coverages = blobs.map(blob => blob.coverage)
111112

112113
return {
113114
files,
114115
errors,
116+
coverages,
115117
}
116118
}
117119

@@ -120,6 +122,7 @@ type MergeReport = [
120122
files: File[],
121123
errors: unknown[],
122124
moduleKeys: MergeReportModuleKeys[],
125+
coverage: unknown,
123126
]
124127

125128
type MergeReportModuleKeys = [projectName: string, moduleIds: string[]]

packages/vitest/src/types/coverage.ts

+16-1
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,33 @@ import type { Arrayable } from './general'
55
import type { AfterSuiteRunMeta } from './worker'
66

77
type TransformResult = string | Partial<ViteTransformResult> | undefined | null | void
8+
type CoverageResults = unknown
89

910
export interface CoverageProvider {
1011
name: string
12+
13+
/** Called when provider is being initialized before tests run */
1114
initialize: (ctx: Vitest) => Promise<void> | void
1215

16+
/** Called when setting coverage options for Vitest context (`ctx.config.coverage`) */
1317
resolveOptions: () => ResolvedCoverageOptions
18+
19+
/** Callback to clean previous reports */
1420
clean: (clean?: boolean) => void | Promise<void>
1521

22+
/** Called with coverage results after a single test file has been run */
1623
onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void | Promise<void>
1724

18-
reportCoverage: (reportContext?: ReportContext) => void | Promise<void>
25+
/** Callback to generate final coverage results */
26+
generateCoverage: (reportContext: ReportContext) => CoverageResults | Promise<CoverageResults>
27+
28+
/** Callback to convert coverage results to coverage reports. Called with results returned from `generateCoverage` */
29+
reportCoverage: (coverage: CoverageResults, reportContext: ReportContext) => void | Promise<void>
30+
31+
/** Callback for `--merge-reports` options. Called with multiple coverage results generated by `generateCoverage`. */
32+
mergeReports?: (coverages: CoverageResults[]) => void | Promise<void>
1933

34+
/** Callback called for instrumenting files with coverage counters. */
2035
onFileTransform?: (
2136
sourceCode: string,
2237
id: string,

0 commit comments

Comments
 (0)