feat!: --merge-reports to support coverage

Author: AriPerkkio

docs/guide/features.md

@@ -239,3 +239,5 @@ vitest --shard=1/2 --reporter=blob
 vitest --shard=2/2 --reporter=blob
 vitest --merge-reports --reporter=junit
 ```
+
+See [`Improving Performance | Sharding`](/guide/improving-performance#sharding) for more information.

docs/guide/improving-performance.md

@@ -72,3 +72,7 @@ export default defineConfig({
 })
 ```
 :::
+
+## Sharding
+
+TODO

packages/coverage-istanbul/src/provider.ts

@@ -162,7 +162,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
     this.pendingPromises = []
   }
 
-  async reportCoverage({ allTestsRun }: ReportContext = {}) {
+  async generateCoverage({ allTestsRun }: ReportContext) {
     const coverageMap = libCoverage.createCoverageMap({})
     let index = 0
     const total = this.pendingPromises.length
@@ -202,6 +202,29 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
       coverageMap.merge(await transformCoverage(uncoveredCoverage))
     }
 
+    return coverageMap
+  }
+
+  async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
+    await this.generateReports(
+      coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
+      allTestsRun,
+    )
+
+    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
+    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+
+    if (!keepResults) {
+      this.coverageFiles = new Map()
+      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+
+      // Remove empty reports directory, e.g. when only text-reporter is used
+      if (readdirSync(this.options.reportsDirectory).length === 0)
+        await fs.rm(this.options.reportsDirectory, { recursive: true })
+    }
+  }
+
+  async generateReports(coverageMap: CoverageMap, allTestsRun: boolean | undefined) {
     const context = libReport.createContext({
       dir: this.options.reportsDirectory,
       coverageMap,
@@ -248,21 +271,18 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
         })
       }
     }
+  }
 
-    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
-    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+  async mergeReports(coverageMaps: unknown[]) {
+    const coverageMap = libCoverage.createCoverageMap({})
 
-    if (!keepResults) {
-      this.coverageFiles = new Map()
-      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+    for (const coverage of coverageMaps)
+      coverageMap.merge(coverage as CoverageMap)
 
-      // Remove empty reports directory, e.g. when only text-reporter is used
-      if (readdirSync(this.options.reportsDirectory).length === 0)
-        await fs.rm(this.options.reportsDirectory, { recursive: true })
-    }
+    await this.generateReports(coverageMap, true)
   }
 
-  async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
+  private async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
     const allFiles = await this.testExclude.glob(this.ctx.config.root)
     let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file))
 

packages/coverage-v8/src/provider.ts

@@ -148,10 +148,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
     this.pendingPromises.push(promise)
   }
 
-  async reportCoverage({ allTestsRun }: ReportContext = {}) {
-    if (provider === 'stackblitz')
-      this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
-
+  async generateCoverage({ allTestsRun }: ReportContext) {
     const coverageMap = libCoverage.createCoverageMap({})
     let index = 0
     const total = this.pendingPromises.length
@@ -193,6 +190,32 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
       coverageMap.merge(await transformCoverage(converted))
     }
 
+    return coverageMap
+  }
+
+  async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
+    if (provider === 'stackblitz')
+      this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
+
+    await this.generateReports(
+      coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
+      allTestsRun,
+    )
+
+    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
+    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+
+    if (!keepResults) {
+      this.coverageFiles = new Map()
+      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+
+      // Remove empty reports directory, e.g. when only text-reporter is used
+      if (readdirSync(this.options.reportsDirectory).length === 0)
+        await fs.rm(this.options.reportsDirectory, { recursive: true })
+    }
+  }
+
+  async generateReports(coverageMap: CoverageMap, allTestsRun?: boolean) {
     const context = libReport.createContext({
       dir: this.options.reportsDirectory,
       coverageMap,
@@ -239,18 +262,15 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
         })
       }
     }
+  }
 
-    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
-    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+  async mergeReports(coverageMaps: unknown[]) {
+    const coverageMap = libCoverage.createCoverageMap({})
 
-    if (!keepResults) {
-      this.coverageFiles = new Map()
-      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+    for (const coverage of coverageMaps)
+      coverageMap.merge(coverage as CoverageMap)
 
-      // Remove empty reports directory, e.g. when only text-reporter is used
-      if (readdirSync(this.options.reportsDirectory).length === 0)
-        await fs.rm(this.options.reportsDirectory, { recursive: true })
-    }
+    await this.generateReports(coverageMap, true)
   }
 
   private async getUntestedFiles(testedFiles: string[]): Promise<RawCoverage> {

packages/vitest/src/node/core.ts

@@ -393,7 +393,7 @@ export class Vitest {
     if (this.reporters.some(r => r instanceof BlobReporter))
       throw new Error('Cannot merge reports when `--reporter=blob` is used. Remove blob reporter from the config first.')
 
-    const { files, errors } = await readBlobs(this.config.mergeReports, this.projects)
+    const { files, errors, coverages } = await readBlobs(this.config.mergeReports, this.projects)
 
     await this.report('onInit', this)
     await this.report('onPathsCollected', files.flatMap(f => f.filepath))
@@ -439,6 +439,8 @@ export class Vitest {
       process.exitCode = 1
 
     await this.report('onFinished', files, errors)
+    await this.initCoverageProvider()
+    await this.coverageProvider?.mergeReports?.(coverages)
   }
 
   async start(filters?: string[]) {
@@ -459,7 +461,9 @@ export class Vitest {
 
     // if run with --changed, don't exit if no tests are found
     if (!files.length) {
-      await this.reportCoverage(true)
+      // Report coverage for uncovered files
+      const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true })
+      await this.reportCoverage(coverage, true)
 
       this.logger.printNoTestFound(filters)
 
@@ -645,8 +649,10 @@ export class Vitest {
       .finally(async () => {
         // can be duplicate files if different projects are using the same file
         const files = Array.from(new Set(specs.map(([, p]) => p)))
-        await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors())
-        await this.reportCoverage(allTestsRun)
+        const coverage = await this.coverageProvider?.generateCoverage({ allTestsRun })
+
+        await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors(), coverage)
+        await this.reportCoverage(coverage, allTestsRun)
 
         this.runningPromise = undefined
         this.isFirstRun = false
@@ -946,12 +952,12 @@ export class Vitest {
     return Array.from(new Set(files))
   }
 
-  private async reportCoverage(allTestsRun: boolean) {
+  private async reportCoverage(coverage: unknown, allTestsRun: boolean) {
     if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0)
       return
 
     if (this.coverageProvider) {
-      await this.coverageProvider.reportCoverage({ allTestsRun })
+      await this.coverageProvider.reportCoverage(coverage, { allTestsRun })
       // notify coverage iframe reload
       for (const reporter of this.reporters) {
         if (reporter instanceof WebSocketReporter)

packages/vitest/src/node/reporters/blob.ts

@@ -26,7 +26,7 @@ export class BlobReporter implements Reporter {
     this.ctx = ctx
   }
 
-  async onFinished(files: File[] = [], errors: unknown[] = []) {
+  async onFinished(files: File[] = [], errors: unknown[] = [], coverage: unknown) {
     let outputFile = this.options.outputFile ?? getOutputFile(this.ctx.config, 'blob')
     if (!outputFile) {
       const shard = this.ctx.config.shard
@@ -39,7 +39,7 @@ export class BlobReporter implements Reporter {
       return [project.getName(), [...project.server.moduleGraph.idToModuleMap.keys()]]
     })
 
-    const report = stringify([this.ctx.version, files, errors, moduleKeys] satisfies MergeReport)
+    const report = stringify([this.ctx.version, files, errors, moduleKeys, coverage] satisfies MergeReport)
 
     const reportFile = resolve(this.ctx.config.root, outputFile)
 
@@ -62,8 +62,8 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
   const blobsFiles = await readdir(resolvedDir)
   const promises = blobsFiles.map(async (file) => {
     const content = await readFile(resolve(resolvedDir, file), 'utf-8')
-    const [version, files, errors, moduleKeys] = parse(content) as MergeReport
-    return { version, files, errors, moduleKeys }
+    const [version, files, errors, moduleKeys, coverage] = parse(content) as MergeReport
+    return { version, files, errors, moduleKeys, coverage }
   })
   const blobs = await Promise.all(promises)
 
@@ -108,10 +108,12 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
     return time1 - time2
   })
   const errors = blobs.flatMap(blob => blob.errors)
+  const coverages = blobs.map(blob => blob.coverage)
 
   return {
     files,
     errors,
+    coverages,
   }
 }
 
@@ -120,6 +122,7 @@ type MergeReport = [
   files: File[],
   errors: unknown[],
   moduleKeys: MergeReportModuleKeys[],
+  coverage: unknown,
 ]
 
 type MergeReportModuleKeys = [projectName: string, moduleIds: string[]]

packages/vitest/src/types/coverage.ts

@@ -5,18 +5,33 @@ import type { Arrayable } from './general'
 import type { AfterSuiteRunMeta } from './worker'
 
 type TransformResult = string | Partial<ViteTransformResult> | undefined | null | void
+type CoverageResults = unknown
 
 export interface CoverageProvider {
   name: string
+
+  /** Called when provider is being initialized before tests run */
   initialize: (ctx: Vitest) => Promise<void> | void
 
+  /** Called when setting coverage options for Vitest context (`ctx.config.coverage`) */
   resolveOptions: () => ResolvedCoverageOptions
+
+  /** Callback to clean previous reports */
   clean: (clean?: boolean) => void | Promise<void>
 
+  /** Called with coverage results after a single test file has been run */
   onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void | Promise<void>
 
-  reportCoverage: (reportContext?: ReportContext) => void | Promise<void>
+  /** Callback to generate final coverage results */
+  generateCoverage: (reportContext: ReportContext) => CoverageResults | Promise<CoverageResults>
+
+  /** Callback to convert coverage results to coverage reports. Called with results returned from `generateCoverage` */
+  reportCoverage: (coverage: CoverageResults, reportContext: ReportContext) => void | Promise<void>
+
+  /** Callback for `--merge-reports` options. Called with multiple coverage results generated by `generateCoverage`. */
+  mergeReports?: (coverages: CoverageResults[]) => void | Promise<void>
 
+  /** Callback called for instrumenting files with coverage counters. */
   onFileTransform?: (
     sourceCode: string,
     id: string,

packages/vitest/src/types/reporter.ts

@@ -7,7 +7,7 @@ export interface Reporter {
   onPathsCollected?: (paths?: string[]) => Awaitable<void>
   onSpecsCollected?: (specs?: SerializableSpec[]) => Awaitable<void>
   onCollected?: (files?: File[]) => Awaitable<void>
-  onFinished?: (files?: File[], errors?: unknown[]) => Awaitable<void>
+  onFinished?: (files?: File[], errors?: unknown[], coverage?: unknown) => Awaitable<void>
   onTaskUpdate?: (packs: TaskResultPack[]) => Awaitable<void>
 
   onTestRemoved?: (trigger?: string) => Awaitable<void>

test/cli/test/create-vitest.test.ts

@@ -24,5 +24,6 @@ it(createVitest, async () => {
       },
     ],
     [],
+    undefined,
   ])
 })

test/coverage-test/coverage-report-tests/merge-reports.test.ts

@@ -0,0 +1,9 @@
+import { test } from 'vitest'
+import { readCoverageJson } from '../coverage-report-tests/utils'
+
+test('reports are merged', async () => {
+  const json = await readCoverageJson('./coverage/coverage-final.json')
+
+  // TODO: Verify json
+  console.log(json)
+})

test/coverage-test/custom-provider.ts

@@ -69,7 +69,11 @@ class CustomCoverageProvider implements CoverageProvider {
     }))
   }
 
-  reportCoverage(reportContext?: ReportContext) {
+  generateCoverage(_reportContext: ReportContext) {
+    return {}
+  }
+
+  reportCoverage(coverage: unknown, reportContext?: ReportContext) {
     this.calls.add(`reportCoverage with ${JSON.stringify(reportContext)}`)
 
     const jsonReport = JSON.stringify({

test/coverage-test/option-tests/merge-fixture-1.test.ts

@@ -0,0 +1,6 @@
+import { test } from 'vitest'
+import { add } from '../src/utils'
+
+test('cover add', () => {
+  add(1, 2)
+})

test/coverage-test/option-tests/merge-fixture-2.test.ts

@@ -0,0 +1,6 @@
+import { test } from 'vitest'
+import { multiply } from '../src/utils'
+
+test('cover multiply', () => {
+  multiply(1, 2)
+})

test/coverage-test/option-tests/merge-fixture-3.test.ts

@@ -0,0 +1,11 @@
+import { test } from 'vitest'
+import { multiply } from '../src/utils'
+import { useImportEnv } from '../src/importEnv'
+
+test('cover multiply again', () => {
+  multiply(1, 2)
+})
+
+test('also cover another file', () => {
+  useImportEnv()
+})

test/coverage-test/test/configuration-options.test-d.ts

@@ -106,6 +106,7 @@ test('provider module', () => {
       return {
         name: 'custom-provider',
         initialize(_: Vitest) {},
+        generateCoverage() {},
         resolveOptions(): ResolvedCoverageOptions {
           return {
             clean: true,