From 23f7a488ef500fb1df5cd234c7d3c2ab4ec02961 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Mon, 15 Feb 2021 15:53:32 +0100 Subject: [PATCH] feat: Be able to specify files to spell check within the config. (#948) Related to #571 ## Specify `files` Make it possible to specify which files to check in the configuration file. A new configuration field `files` has be added: ```js { // tell cspell to check all JavaScript and Markdown files. files: ["**/*.js", "**/*.md"] } ``` ## Commits * feat: Be able to specify files to spell check within the config. * dev: Use files from config in cli application * dev: add methods to support glob normalization to a common root. * refactor: move the methods to a more logical place. * dev: Correct the order to load configuration files to support VS Code Ext * dev: Normalize globs passed in on the command line. - Added lots of test to ensure behavior. - Added support for `files` to be defined in the configuration. - Fixed some issues related to the root. * dev: build lists of include and exclude globs. * dev: File normalization is now done in cspell-glob. * dev: Use a generator to flatten the results and make them unique. * dev: Normalize relative paths * dev: Use a single glob. * dev: make single glob optional * Use 0.2 for the main cspell.json file * Update launch.json --- cspell.json | 2 +- cspell.schema.json | 11 + packages/cspell-glob/src/GlobMatcher.test.ts | 103 +++++---- packages/cspell-glob/src/GlobMatcher.ts | 134 +++++------- packages/cspell-glob/src/GlobMatcherTypes.ts | 55 +++++ packages/cspell-glob/src/globHelper.test.ts | 190 +++++++++++++++++ packages/cspell-glob/src/globHelper.ts | 178 ++++++++++++++++ packages/cspell-glob/src/index.test.ts | 9 + packages/cspell-glob/src/index.ts | 10 +- .../src/Settings/CSpellSettingsServer.test.ts | 101 ++++++--- .../src/Settings/CSpellSettingsServer.ts | 73 +++++-- packages/cspell-types/cspell.schema.json | 11 + .../src/settings/CSpellSettingsDef.ts | 12 +- packages/cspell/.vscode/launch.json | 6 +- packages/cspell/cSpell.json | 8 +- .../cspell/src/__snapshots__/app.test.ts.snap | 46 ++-- packages/cspell/src/app.test.ts | 12 +- packages/cspell/src/app.ts | 8 +- packages/cspell/src/application.test.ts | 28 ++- packages/cspell/src/application.ts | 38 ++-- packages/cspell/src/util/glob.test.ts | 121 ++++++++--- packages/cspell/src/util/glob.ts | 200 ++++-------------- 22 files changed, 926 insertions(+), 430 deletions(-) create mode 100644 packages/cspell-glob/src/GlobMatcherTypes.ts create mode 100644 packages/cspell-glob/src/globHelper.test.ts create mode 100644 packages/cspell-glob/src/globHelper.ts create mode 100644 packages/cspell-glob/src/index.test.ts diff --git a/cspell.json b/cspell.json index b293a675d31c..4ae091f40f46 100644 --- a/cspell.json +++ b/cspell.json @@ -1,5 +1,5 @@ { - "version": "0.1", + "version": "0.2", "dictionaryDefinitions": [ { "name": "workspace", diff --git a/cspell.schema.json b/cspell.schema.json index b8841bd4dba4..144c4d2ce554 100644 --- a/cspell.schema.json +++ b/cspell.schema.json @@ -488,6 +488,13 @@ }, "type": "array" }, + "files": { + "description": "Glob patterns of files to be checked. Glob patterns are relative to the `globRoot` of the configuration file that defines them.", + "items": { + "$ref": "#/definitions/Glob" + }, + "type": "array" + }, "flagWords": { "description": "list of words to always be considered incorrect.", "items": { @@ -611,6 +618,10 @@ "version": { "default": "0.2", "description": "Configuration format version of the setting file.", + "enum": [ + "0.2", + "0.1" + ], "type": "string" }, "words": { diff --git a/packages/cspell-glob/src/GlobMatcher.test.ts b/packages/cspell-glob/src/GlobMatcher.test.ts index 5a95cd4d565d..08d393084e66 100644 --- a/packages/cspell-glob/src/GlobMatcher.test.ts +++ b/packages/cspell-glob/src/GlobMatcher.test.ts @@ -1,4 +1,5 @@ -import { GlobMatcher, GlobMatch, PathInterface, GlobMatchOptions } from './GlobMatcher'; +import { GlobMatcher, GlobMatchOptions } from './GlobMatcher'; +import { GlobMatch, PathInterface } from './GlobMatcherTypes'; import * as path from 'path'; import mm = require('micromatch'); @@ -162,26 +163,44 @@ describe('Validate Options', () => { expected: Partial | boolean; } test.each` - pattern | file | options | expected - ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{}} | ${{ matched: false }} - ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${{ matched: true, glob: '**/{*.yaml,*.yaml/**}' }} - ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${true} - ${'.github/**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${true} - ${'.github/**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: false }} | ${true} - ${'.github/**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{}} | ${true} - ${'.github/**/*.yaml'} | ${'.github/test.yaml'} | ${{}} | ${true} - ${'.github/**/*.yaml'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${false} - ${'**/.github/**/*.yaml'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} - ${'.github'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} - ${'**/.github/**'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} - ${'package/**'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${false} - ${'package/**'} | ${'package/.github/workflows/test.yaml'} | ${{ dot: true }} | ${true} - ${'workflows'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${false} - ${'workflows'} | ${'package/.github/workflows/test.yaml'} | ${{ dot: true }} | ${true} - ${'*.yaml|!test.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${{ matched: false, glob: '!**/{test.yaml,test.yaml/**}', isNeg: true }} - ${'*.yaml|!/test.yaml'} | ${'test.yaml'} | ${{ dot: true }} | ${{ matched: false, glob: '!test.yaml', isNeg: true }} - ${'*.{!yml}'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${false} - `('Test options: $pattern, $text, $options', ({ pattern, file, options, expected }: TestCase) => { + pattern | file | options | expected + ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{}} | ${{ matched: true }} + ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: false }} | ${{ matched: false }} + ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ mode: 'include' }} | ${{ matched: false }} + ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${{ matched: true, glob: '**/{*.yaml,*.yaml/**}' }} + ${'*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${true} + ${'**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ mode: 'exclude' }} | ${{ matched: true }} + ${'**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ mode: 'include' }} | ${{ matched: false }} + ${'.github/**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: true }} | ${true} + ${'.github/**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{ dot: false }} | ${true} + ${'.github/**/*.yaml'} | ${'.github/workflows/test.yaml'} | ${{}} | ${true} + ${'.github/**/*.yaml'} | ${'.github/test.yaml'} | ${{}} | ${true} + ${'.github/**/*.yaml'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${false} + ${'**/.github/**/*.yaml'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} + ${'.github'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} + ${'**/.github/**'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} + ${'package/**'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} + ${'package/**'} | ${'package/.github/workflows/test.yaml'} | ${{ dot: false }} | ${false} + ${'package/**'} | ${'package/.github/workflows/test.yaml'} | ${{ mode: 'include' }} | ${false} + ${'workflows'} | ${'package/.github/workflows/test.yaml'} | ${{}} | ${true} + ${'workflows'} | ${'package/.github/workflows/test.yaml'} | ${{ dot: false }} | ${false} + ${'package/'} | ${'package/src/test.yaml'} | ${{}} | ${true} + ${'package/'} | ${'package/src/test.yaml'} | ${{ dot: false }} | ${true} + ${'package/'} | ${'package/src/test.yaml'} | ${{ mode: 'include' }} | ${true} + ${'package/'} | ${'repo/package/src/test.yaml'} | ${{}} | ${true} + ${'package/'} | ${'repo/package/src/test.yaml'} | ${{ mode: 'include' }} | ${false} + ${'/package/'} | ${'package/src/test.yaml'} | ${{}} | ${true} + ${'/package/'} | ${'package/src/test.yaml'} | ${{ dot: false }} | ${true} + ${'/package/'} | ${'package/src/test.yaml'} | ${{ mode: 'include' }} | ${true} + ${'/package/'} | ${'repo/package/src/test.yaml'} | ${{}} | ${false} + ${'/package/'} | ${'repo/package/src/test.yaml'} | ${{ mode: 'include' }} | ${false} + ${'src'} | ${'package/src/test.yaml'} | ${{ mode: 'include' }} | ${false} + ${'*.yaml|!test.yaml'} | ${'.github/workflows/test.yaml'} | ${{}} | ${{ matched: false, glob: '!**/{test.yaml,test.yaml/**}', isNeg: true }} + ${'*.yaml|!/test.yaml'} | ${'test.yaml'} | ${{}} | ${{ matched: false, glob: '!{test.yaml,test.yaml/**}', isNeg: true }} + ${'*.yaml|!/node_modules'} | ${'node_modules/test.yaml'} | ${{}} | ${{ matched: false, glob: '!{node_modules,node_modules/**}', isNeg: true }} + ${'*.{!yaml}'} | ${'.github/workflows/test.yaml'} | ${{}} | ${false} + ${'test.*|!*.{yaml,yml}'} | ${'.github/workflows/test.yaml'} | ${{}} | ${{ matched: false, isNeg: true }} + `('Test options: $pattern, $file, $options', ({ pattern, file, options, expected }: TestCase) => { const root = '/Users/code/project/cspell/'; const filename = path.join(root, file); const patterns = pattern.split('|'); @@ -218,12 +237,13 @@ describe('Validate GlobMatcher', () => { ${['/*.json']} | ${undefined} | ${'/src/settings.json'} | ${false} | ${'Matches pattern but not cwd /*.json'} ${['*.js']} | ${undefined} | ${'${cwd}/src/settings.js'} | ${true} | ${'// Matches nested files, *.js'} ${['.vscode/']} | ${undefined} | ${'${cwd}/.vscode/settings.json'} | ${true} | ${'.vscode/'} - ${['.vscode/']} | ${undefined} | ${'${cwd}/.vscode'} | ${true} | ${'.vscode/'} - ${['/.vscode/']} | ${undefined} | ${'${cwd}/.vscode'} | ${true} | ${'should match root'} + ${['.vscode/']} | ${undefined} | ${'${cwd}/.vscode'} | ${false} | ${'.vscode/'} + ${['/.vscode/']} | ${undefined} | ${'${cwd}/.vscode'} | ${false} | ${'should match root'} ${['/.vscode/']} | ${undefined} | ${'${cwd}/.vscode/settings.json'} | ${true} | ${'should match root'} ${['/.vscode/']} | ${undefined} | ${'${cwd}/package/.vscode'} | ${false} | ${'should only match root'} - ${['.vscode/']} | ${undefined} | ${'${cwd}/src/.vscode/settings.json'} | ${false} | ${"shouldn't match nested .vscode/"} + ${['.vscode/']} | ${undefined} | ${'${cwd}/src/.vscode/settings.json'} | ${true} | ${'should match nested .vscode/'} ${['**/.vscode/']} | ${undefined} | ${'${cwd}/src/.vscode/settings.json'} | ${true} | ${'should match nested .vscode/'} + ${['**/.vscode/']} | ${undefined} | ${'${cwd}/src/.vscode'} | ${false} | ${'should match nested .vscode'} ${['**/.vscode']} | ${undefined} | ${'${cwd}/src/.vscode/settings.json'} | ${false} | ${'should not match nested **/.vscode'} ${['**/.vscode/**']} | ${undefined} | ${'${cwd}/src/.vscode/settings.json'} | ${true} | ${'should match nested **/.vscode'} ${['/User/user/Library/**']} | ${undefined} | ${'/src/User/user/Library/settings.json'} | ${false} | ${'No match'} @@ -270,8 +290,8 @@ function tests(): TestCase[] { [['/*.json'], undefined, '/src/settings.json', false, 'Matches pattern but not cwd /*.json'], // . [['*.js'], undefined, '${cwd}/src/settings.js', true, '// Matches nested files, *.js'], [['.vscode/'], undefined, '${cwd}/.vscode/settings.json', true, '.vscode/'], - [['.vscode/'], undefined, '${cwd}/.vscode', true, '.vscode/'], - [['.vscode/'], undefined, '${cwd}/src/.vscode/settings.json', false, "shouldn't match nested .vscode/"], + [['.vscode/'], undefined, '${cwd}/.vscode', false, '.vscode/'], + [['.vscode/'], undefined, '${cwd}/src/.vscode/settings.json', true, 'should match nested .vscode/'], [['**/.vscode/'], undefined, '${cwd}/src/.vscode/settings.json', true, 'should match nested .vscode/'], [['**/.vscode'], undefined, '${cwd}/src/.vscode/settings.json', false, 'should not match nested **/.vscode'], [['**/.vscode/**'], undefined, '${cwd}/src/.vscode/settings.json', true, 'should match nested **/.vscode'], @@ -284,8 +304,8 @@ function tests(): TestCase[] { [['/*.json'], undefined, 'src/settings.json', false, 'Matches only root level files, /*.json'], // . [['*.js'], undefined, 'src/settings.js', true, '// Matches nested files, *.js'], [['.vscode/'], undefined, '.vscode/settings.json', true, '.vscode/'], - [['.vscode/'], undefined, '.vscode', true, '.vscode/'], - [['.vscode/'], undefined, 'src/.vscode/settings.json', false, "shouldn't match nested .vscode/"], + [['.vscode/'], undefined, '.vscode', false, '.vscode/'], + [['.vscode/'], undefined, 'src/.vscode/settings.json', true, 'should match nested .vscode/'], [['**/.vscode/'], undefined, 'src/.vscode/settings.json', true, 'should match nested .vscode/'], [['**/.vscode'], undefined, 'src/.vscode/settings.json', false, 'should not match nested **/.vscode'], [['**/.vscode/**'], undefined, 'src/.vscode/settings.json', true, 'should match nested **/.vscode'], @@ -304,13 +324,13 @@ function tests(): TestCase[] { ], // . [['*.js'], '/User/code/src', '/User/code/src/src/settings.js', true, 'With Root Matches nested files, *.js'], [['.vscode/'], '/User/code/src', '/User/code/src/.vscode/settings.json', true, 'With Root .vscode/'], - [['.vscode/'], '/User/code/src', '/User/code/src/.vscode', true, 'With Root .vscode/'], // This one shouldn't match, but micromatch says it should. :-( + [['.vscode/'], '/User/code/src', '/User/code/src/.vscode', false, 'With Root .vscode/'], // This one shouldn't match, but micromatch says it should. :-( [ ['.vscode/'], '/User/code/src', '/User/code/src/src/.vscode/settings.json', - false, - "With Root shouldn't match nested .vscode/", + true, + 'With Root should match nested .vscode/', ], [ ['**/.vscode/'], @@ -355,14 +375,22 @@ function tests(): TestCase[] { ], // . [['*.js'], '/User/code/src/', '/User/code/src/src/settings.js', true, '// Matches nested files, *.js'], [['.vscode/'], '/User/code/src/', '/User/code/src/.vscode/settings.json', true, '.vscode/'], - [['.vscode/'], '/User/code/src/', '/User/code/src/.vscode', true, '.vscode/'], // This one shouldn't match, but micromatch says it should. :-( + [['.vscode/'], '/User/code/src/', '/User/code/src/.vscode', false, '.vscode/'], // This one shouldn't match, but micromatch says it should. :-( [ - ['.vscode/'], + ['/.vscode/'], '/User/code/src/', '/User/code/src/src/.vscode/settings.json', false, "shouldn't match nested .vscode/", ], + [ + ['.vscode/'], + '/User/code/src/', + '/User/code/src/src/.vscode/settings.json', + true, + 'should match nested .vscode/', + ], + [['.vscode/'], '/User/code/src/', '/User/code/src/src/.vscode', false, 'should match nested file .vscode'], [ ['**/.vscode/'], '/User/code/src/', @@ -379,8 +407,8 @@ function tests(): TestCase[] { [['/*.json'], '/', '/settings.json', true, 'Matches only root level files, /*.json'], // . [['*.js'], '/', '/src/settings.js', true, '// Matches nested files, *.js'], [['.vscode/'], '/', '/.vscode/settings.json', true, '.vscode/'], - [['.vscode/'], '/', '/.vscode', true, '.vscode/'], - [['.vscode/'], '/', '/src/.vscode/settings.json', false, "shouldn't match nested .vscode/"], + [['.vscode/'], '/', '/.vscode', false, '.vscode/'], + [['.vscode/'], '/', '/src/.vscode/settings.json', true, 'should match nested .vscode/'], [['**/.vscode/'], '/', '/src/.vscode/settings.json', true, 'should match nested .vscode/'], [['/User/user/Library/**'], '/', '/src/User/user/Library/settings.json', false, 'No match'], [['/User/user/Library/**'], '/', '/User/user/Library/settings.json', true, 'Match system root'], @@ -392,8 +420,9 @@ function tests(): TestCase[] { [['/*.json'], '', '${cwd}/src/settings.json', false, 'Matches only root level files, /*.json'], // . [['*.js'], '', '${cwd}/src/settings.js', true, '// Matches nested files, *.js'], [['.vscode/'], '', '${cwd}/.vscode/settings.json', true, '.vscode/'], - [['.vscode/'], '', '${cwd}/.vscode', true, '.vscode/'], - [['.vscode/'], '', '${cwd}/src/.vscode/settings.json', false, "shouldn't match nested .vscode/"], + [['.vscode/'], '', '${cwd}/.vscode', false, '.vscode/'], + [['.vscode/'], '', '${cwd}/src/.vscode/settings.json', true, 'should match nested .vscode/'], + [['/.vscode/'], '', '${cwd}/src/.vscode/settings.json', false, "shouldn't match nested .vscode/"], [['**/.vscode/'], '', '${cwd}/src/.vscode/settings.json', true, 'should match nested .vscode/'], [['/User/user/Library/**'], '', '${cwd}/src/User/user/Library/settings.json', false, 'No match'], [['/User/user/Library/**'], '', '${cwd}/User/user/Library/settings.json', true, 'Match system root'], diff --git a/packages/cspell-glob/src/GlobMatcher.ts b/packages/cspell-glob/src/GlobMatcher.ts index e3d816884c44..c72fe2319626 100644 --- a/packages/cspell-glob/src/GlobMatcher.ts +++ b/packages/cspell-glob/src/GlobMatcher.ts @@ -1,49 +1,58 @@ import mm = require('micromatch'); import * as Path from 'path'; +import { normalizeGlobPatterns } from './globHelper'; +import { PathInterface, GlobMatch, GlobPattern, GlobPatternWithRoot } from './GlobMatcherTypes'; // cspell:ignore fname -export interface PathInterface { - normalize(p: string): string; - join(...paths: string[]): string; - resolve(...paths: string[]): string; - relative(from: string, to: string): string; - isAbsolute(p: string): boolean; - sep: string; -} - -export type GlobMatch = GlobMatchRule | GlobMatchNoRule; - -export interface GlobMatchRule { - matched: boolean; - glob: string; - root: string; - index: number; - isNeg: boolean; -} - -export interface GlobMatchNoRule { - matched: false; -} - export type GlobMatchOptions = Partial; +export type MatcherMode = 'exclude' | 'include'; + interface NormalizedGlobMatchOptions { + /** + * The matcher has two modes (`include` or `exclude`) that impact how globs behave. + * + * `include` - designed for searching for file. By default it matches a sub-set of file. + * In include mode, the globs need to be more explicit to match. + * - `dot` is by default false. + * - `nested` is by default false. + * + * `exclude` - designed to emulate `.gitignore`. By default it matches a larger range of files. + * - `dot` is by default true. + * - `nested` is by default true. + * + * @default: 'exclude' + */ + mode: MatcherMode; + + /** + * The default directory from which a glob is relative. + * Any globs that are not relative to the root will ignored. + * @default: process.cwd() + */ root: string; - dot: boolean; - nodePath: PathInterface; -} -export type GlobPattern = SimpleGlobPattern | GlobPatternWithRoot | GlobPatternWithOptionalRoot; + /** + * Allows matching against directories with a leading `.`. + * + * @default: mode == 'exclude' + */ + dot: boolean; -export type SimpleGlobPattern = string; -export interface GlobPatternWithOptionalRoot { - glob: string; - root?: string; -} + /** + * Allows matching against nested directories or files without needing to add `**` + * + * @default: mode == 'exclude' + */ + nested: boolean; -export interface GlobPatternWithRoot extends GlobPatternWithOptionalRoot { - root: string; + /** + * Mostly used for testing purposes. It allows explicitly specifying `path.win32` or `path.posix`. + * + * @default: require('path') + */ + nodePath: PathInterface; } export class GlobMatcher { @@ -82,18 +91,27 @@ export class GlobMatcher { const options = typeof rootOrOptions === 'string' ? { root: rootOrOptions, nodePath: _nodePath } : rootOrOptions ?? {}; + const { mode = 'exclude' } = options; + const isExcludeMode = mode !== 'include'; - const { root = _nodePath.resolve(), dot = false, nodePath = _nodePath } = options; + const { + root = _nodePath.resolve(), + dot = isExcludeMode, + nodePath = _nodePath, + nested = isExcludeMode, + } = options; const normalizedRoot = nodePath.resolve(nodePath.normalize(root)); - this.options = { root: normalizedRoot, dot, nodePath }; + this.options = { root: normalizedRoot, dot, nodePath, nested, mode }; patterns = Array.isArray(patterns) ? patterns : typeof patterns === 'string' ? patterns.split(/\r?\n/g) : [patterns]; - const globPatterns = patterns.map((p) => normalizeGlobPatternWithRoot(p, normalizedRoot, nodePath)); + const globPatterns = normalizeGlobPatterns(patterns, this.options) + // Only keep globs that do not match the root when using exclude mode. + .filter((g) => isExcludeMode || g.root === normalizedRoot); this.patterns = globPatterns; this.root = normalizedRoot; @@ -183,45 +201,3 @@ function buildMatcherFn(patterns: GlobPatternWithRoot[], options: NormalizedGlob }; return fn; } - -type MutationsToSupportGitIgnore = [RegExp, string]; - -const mutations: MutationsToSupportGitIgnore[] = [ - [/^[^/#][^/]*$/, '**/{$&,$&/**}'], // no slashes will match files names or folders - [/^\/(?!\/)/, ''], // remove leading slash to match from the root - [/\/$/, '$&**'], // if it ends in a slash, make sure matches the folder -]; - -export function isGlobPatternWithOptionalRoot(g: GlobPattern): g is GlobPatternWithOptionalRoot { - return typeof g !== 'string' && typeof g.glob === 'string'; -} - -export function isGlobPatternWithRoot(g: GlobPatternWithRoot | GlobPatternWithOptionalRoot): g is GlobPatternWithRoot { - return !!g.root; -} - -function normalizePattern(pattern: string): string { - pattern = pattern.replace(/^(!!)+/, ''); - const isNeg = pattern.startsWith('!'); - pattern = isNeg ? pattern.slice(1) : pattern; - pattern = mutations.reduce((p, [regex, replace]) => p.replace(regex, replace), pattern); - return isNeg ? '!' + pattern : pattern; -} - -function normalizeGlobPatternWithRoot(g: GlobPattern, root: string, path: PathInterface): GlobPatternWithRoot { - g = !isGlobPatternWithOptionalRoot(g) - ? { - glob: g.trim(), - root, - } - : g; - - const gr = isGlobPatternWithRoot(g) ? g : { ...g, root }; - if (gr.root.startsWith('${cwd}')) { - gr.root = path.join(path.resolve(), gr.root.replace('${cwd}', '')); - } - gr.root = path.resolve(root, path.normalize(gr.root)); - gr.glob = normalizePattern(gr.glob); - - return gr; -} diff --git a/packages/cspell-glob/src/GlobMatcherTypes.ts b/packages/cspell-glob/src/GlobMatcherTypes.ts new file mode 100644 index 000000000000..65544e0f11e8 --- /dev/null +++ b/packages/cspell-glob/src/GlobMatcherTypes.ts @@ -0,0 +1,55 @@ +// cspell:ignore fname + +export interface PathInterface { + normalize(p: string): string; + join(...paths: string[]): string; + resolve(...paths: string[]): string; + relative(from: string, to: string): string; + isAbsolute(p: string): boolean; + sep: string; +} + +export type GlobMatch = GlobMatchRule | GlobMatchNoRule; + +export interface GlobMatchRule { + matched: boolean; + glob: string; + root: string; + index: number; + isNeg: boolean; +} + +export interface GlobMatchNoRule { + matched: false; +} + +export type GlobPattern = SimpleGlobPattern | GlobPatternWithRoot | GlobPatternWithOptionalRoot; + +export type SimpleGlobPattern = string; + +export interface GlobPatternWithOptionalRoot { + /** + * a glob pattern + */ + glob: string; + /** + * The root from which the glob pattern is relative. + * @default: options.root + */ + root?: string; + /** + * Optional value useful for tracing which file a glob pattern was defined in. + */ + source?: string; +} + +export interface GlobPatternWithRoot extends GlobPatternWithOptionalRoot { + root: string; +} + +export interface GlobPatternNormalized extends GlobPatternWithRoot { + /** the original glob pattern before it was normalized */ + rawGlob: string; + /** the original root */ + rawRoot: string | undefined; +} diff --git a/packages/cspell-glob/src/globHelper.test.ts b/packages/cspell-glob/src/globHelper.test.ts new file mode 100644 index 000000000000..91804d1db7d7 --- /dev/null +++ b/packages/cspell-glob/src/globHelper.test.ts @@ -0,0 +1,190 @@ +import { fileOrGlobToGlob, normalizeGlobPatterns } from './globHelper'; +import { win32, posix } from 'path'; +import * as path from 'path'; +import { GlobPattern, GlobPatternNormalized, GlobPatternWithOptionalRoot, PathInterface } from './GlobMatcherTypes'; +import mm = require('micromatch'); + +describe('Validate fileOrGlobToGlob', () => { + function g(glob: string, root: string) { + return { glob, root }; + } + + function p(root: string, path: PathInterface): string { + const cwd = path === win32 ? 'E:\\user\\projects' : '/User/projects'; + return path.resolve(cwd, root); + } + + function pp(root: string): string { + return p(root, posix); + } + + function pw(root: string): string { + return p(root, win32); + } + + test.each` + file | root | path | expected | comment + ${'*.json'} | ${'.'} | ${posix} | ${g('*.json', pp('.'))} | ${'posix'} + ${'*.json'} | ${'.'} | ${win32} | ${g('*.json', pw('.'))} | ${'win32'} + ${pp('./*.json')} | ${'.'} | ${posix} | ${g('*.json', pp('.'))} | ${''} + ${pw('./*.json')} | ${'.'} | ${win32} | ${g('*.json', pw('.'))} | ${''} + ${pp('./package.json')} | ${'.'} | ${posix} | ${g('package.json', pp('.'))} | ${''} + ${pw('.\\package.json')} | ${'.'} | ${win32} | ${g('package.json', pw('.'))} | ${''} + ${pp('./package.json')} | ${'.'} | ${posix} | ${g('package.json', pp('.'))} | ${''} + ${'.\\package.json'} | ${'.'} | ${win32} | ${g('package.json', pw('.'))} | ${''} + ${'./a/package.json'} | ${'.'} | ${posix} | ${g('a/package.json', pp('.'))} | ${''} + ${pw('.\\a\\package.json')} | ${'.'} | ${win32} | ${g('a/package.json', pw('.'))} | ${''} + ${'/user/tester/projects'} | ${'.'} | ${posix} | ${g('/user/tester/projects', pp('.'))} | ${'Directory not matching root.'} + ${'C:\\user\\tester\\projects'} | ${'.'} | ${win32} | ${g('C:/user/tester/projects', pw('.'))} | ${'Directory not matching root.'} + ${'/user/tester/projects/**/*.json'} | ${'.'} | ${posix} | ${g('/user/tester/projects/**/*.json', pp('.'))} | ${'A glob like path not matching the root.'} + ${'C:\\user\\tester\\projects\\**\\*.json'} | ${'.'} | ${win32} | ${g('C:/user/tester/projects/**/*.json', pw('.'))} | ${'A glob like path not matching the root.'} + `('fileOrGlobToGlob file: "$file" root: "$root" $comment', ({ file, root, path, expected }) => { + root = p(root, path); + const r = fileOrGlobToGlob(file, root, path); + expect(r).toEqual(expected); + }); +}); + +describe('Validate Glob Normalization to root', () => { + function mg( + patterns: GlobPattern | GlobPattern[], + root?: string, + source = 'cspell.json' + ): GlobPatternWithOptionalRoot[] { + root = path.resolve(root || '.'); + patterns = Array.isArray(patterns) ? patterns : typeof patterns === 'string' ? patterns.split('|') : [patterns]; + source = path.join(root, source); + + return patterns.map((p) => (typeof p === 'string' ? { glob: p } : p)).map((g) => ({ root, source, ...g })); + } + + function j( + patterns: GlobPatternWithOptionalRoot[], + ...additional: (GlobPatternWithOptionalRoot[] | GlobPatternWithOptionalRoot)[] + ): GlobPatternWithOptionalRoot[] { + function* flatten() { + for (const a of additional) { + if (Array.isArray(a)) { + yield* a; + } else { + yield a; + } + } + } + + return patterns.concat([...flatten()]); + } + + function e(...expected: Partial[]) { + return expected + .map((e) => { + const p: Partial = {}; + if (e.root) { + p.root = path.resolve(e.root); + } + if (e.rawRoot) { + p.rawRoot = path.resolve(e.rawRoot); + } + return { ...e, ...p }; + }) + .map((e) => expect.objectContaining(e)); + } + + interface TestCase { + globs: GlobPatternWithOptionalRoot[]; + root: string; + expectedGlobs: GlobPatternNormalized[]; + comment: string; + } + + test.each` + globs | root | expectedGlobs | comment + ${mg('*.json')} | ${'.'} | ${e({ rawGlob: '*.json', glob: '**/{*.json,*.json/**}' })} | ${'Glob with same root'} + ${mg('!*.json')} | ${'.'} | ${e({ rawGlob: '!*.json', glob: '!**/{*.json,*.json/**}' })} | ${'Negative glob'} + ${mg('!*.json', 'project/a')} | ${'.'} | ${e({ rawGlob: '!*.json', glob: '!project/a/**/{*.json,*.json/**}' })} | ${'Negative in Sub dir glob.'} + ${j(mg('*.json', 'project/a'), mg('*.ts', '.'))} | ${'.'} | ${e({ rawGlob: '*.json', glob: 'project/a/**/{*.json,*.json/**}' }, { glob: '**/{*.ts,*.ts/**}' })} | ${'Sub dir glob.'} + ${j(mg('*.json', '../tests/a'), mg('*.ts', '.'))} | ${'.'} | ${e({ glob: '**/{*.ts,*.ts/**}' })} | ${'Glob not in root is removed.'} + ${mg('*.json')} | ${'project'} | ${e({ rawGlob: '*.json', glob: '**/{*.json,*.json/**}' })} | ${'Root deeper than glob'} + ${mg('!*.json')} | ${'project'} | ${e({ rawGlob: '!*.json', glob: '!**/{*.json,*.json/**}' })} | ${'Root deeper than glob'} + ${j(mg('*.json', 'project/a'), mg('*.ts', '.'))} | ${'project'} | ${e({ rawGlob: '*.json', glob: 'a/**/{*.json,*.json/**}' }, { glob: '**/{*.ts,*.ts/**}' })} | ${'Root in the middle.'} + ${j(mg('/node_modules', 'project/a'), mg('*.ts', '.'))} | ${'project'} | ${e({ rawGlob: '/node_modules', glob: 'a/{node_modules,node_modules/**}' }, { glob: '**/{*.ts,*.ts/**}' })} | ${'Root in the middle. /node_modules'} + ${j(mg('!/node_modules', 'project/a'))} | ${'project'} | ${e({ rawGlob: '!/node_modules', glob: '!a/{node_modules,node_modules/**}' })} | ${'Root in the middle. /node_modules'} + ${j(mg('*.json', '../tests/a'), mg('*.ts', '.'))} | ${'project'} | ${e({ glob: '**/{*.ts,*.ts/**}' })} | ${'Glob not in root is removed.'} + ${j(mg('*.json', '../tests/a'))} | ${'project'} | ${e()} | ${'Glob not in root is removed.'} + ${j(mg('*/*.json', 'project/a'))} | ${'project'} | ${e({ glob: 'a/*/*.json' })} | ${'nested a/*/*.json'} + ${j(mg('*/*.json', '.'))} | ${'project'} | ${e({ glob: '*.json' })} | ${'nested */*.json'} + `('tests normalization nested "$comment" root: "$root"', ({ globs, root, expectedGlobs }: TestCase) => { + root = path.resolve(root); + const r = normalizeGlobPatterns(globs, { root, nested: true, nodePath: path }); + expect(r).toEqual(expectedGlobs); + }); + + test.each` + globs | root | expectedGlobs | comment + ${mg('*.json')} | ${'.'} | ${e({ rawGlob: '*.json', glob: '*.json' })} | ${'Glob with same root'} + ${j(mg('*.json', 'project/a'), mg('*.ts', '.'))} | ${'.'} | ${e({ rawGlob: '*.json', glob: 'project/a/*.json' }, { glob: '*.ts' })} | ${'Sub dir glob.'} + ${j(mg('*.json', '../tests/a'), mg('*.ts', '.'))} | ${'.'} | ${e({ glob: '*.ts' })} | ${'Glob not in root is removed.'} + ${mg('*.json')} | ${'project'} | ${e({ glob: '*.json', root: '.' })} | ${'Root deeper than glob'} + ${j(mg('*.json', 'project/a'), mg('*.ts', '.'))} | ${'project'} | ${e({ rawGlob: '*.json', glob: 'a/*.json' }, { glob: '*.ts', root: '.' })} | ${'Root in the middle.'} + ${j(mg('/node_modules', 'project/a'), mg('*.ts', 'project'))} | ${'project'} | ${e({ rawGlob: '/node_modules', glob: 'a/node_modules' }, { glob: '*.ts' })} | ${'Root in the middle. /node_modules'} + ${j(mg('*.json', '../tests/a'), mg('**/*.ts', '.'))} | ${'project'} | ${e({ glob: '**/*.ts' })} | ${'Glob not in root is removed.'} + ${j(mg('*.json', '../tests/a'))} | ${'project'} | ${e()} | ${'Glob not in root is removed.'} + ${j(mg('*/*.json', 'project/a'))} | ${'project'} | ${e({ glob: 'a/*/*.json' })} | ${'nested a/*/*.json'} + ${j(mg('*/*.json', '.'))} | ${'project'} | ${e({ glob: '*.json' })} | ${'nested */*.json'} + ${j(mg('project/*/*.json', '.'))} | ${'project/sub'} | ${e({ glob: '*.json' })} | ${'nested project/*/*.json'} + `('tests normalization not nested "$comment" root: "$root"', ({ globs, root, expectedGlobs }: TestCase) => { + root = path.resolve(root); + const r = normalizeGlobPatterns(globs, { root, nested: false, nodePath: path }); + expect(r).toEqual(expectedGlobs); + }); +}); + +describe('Validate minimatch assumptions', () => { + interface TestCase { + pattern: string; + file: string; + options: mm.Options; + expected: boolean; + } + + const jsPattern = '*.{js,jsx}'; + const mdPattern = '*.md'; + const nodePattern = '{node_modules,node_modules/**}'; + const nestedPattern = `{**/temp/**,{${jsPattern},${mdPattern},${nodePattern}}}`; + + test.each` + pattern | file | options | expected | comment + ${'*.json'} | ${'package.json'} | ${{}} | ${true} | ${''} + ${'**/*.json'} | ${'package.json'} | ${{}} | ${true} | ${''} + ${'node_modules'} | ${'node_modules/cspell/package.json'} | ${{}} | ${false} | ${''} + ${'node_modules/'} | ${'node_modules/cspell/package.json'} | ${{}} | ${false} | ${''} + ${'node_modules/'} | ${'node_modules'} | ${{}} | ${false} | ${''} + ${'node_modules/**'} | ${'node_modules/cspell/package.json'} | ${{}} | ${true} | ${''} + ${'node_modules/**/*'} | ${'node_modules/package.json'} | ${{}} | ${true} | ${''} + ${'node_modules/**'} | ${'node_modules'} | ${{}} | ${true} | ${'Note: this seems to be a bug with micromatch (minimatch return false)'} + ${'node_modules/**/*'} | ${'node_modules'} | ${{}} | ${false} | ${'Note: this is a work around for `/**` not working.'} + ${'*.json'} | ${'src/package.json'} | ${{}} | ${false} | ${''} + ${'*.json'} | ${'src/package.json'} | ${{ matchBase: true }} | ${true} | ${'check matchBase behavior, option not used by cspell'} + ${'*.yml'} | ${'.github/workflows/test.yml'} | ${{ matchBase: true }} | ${true} | ${'check matchBase behavior, option not used by cspell'} + ${'**/*.yml'} | ${'.github/workflows/test.yml'} | ${{}} | ${false} | ${''} + ${'**/*.yml'} | ${'.github/workflows/test.yml'} | ${{ dot: true }} | ${true} | ${'dot is used by default for excludes'} + ${'{*.json,*.yaml}'} | ${'package.json'} | ${{}} | ${true} | ${''} + ${nestedPattern} | ${'index.js'} | ${{}} | ${true} | ${'Nested {} is supported'} + ${nestedPattern} | ${'node_modules/cspell/package.json'} | ${{}} | ${true} | ${'Nested {} is supported'} + ${nestedPattern} | ${'testing/temp/file.bin'} | ${{}} | ${true} | ${'Nested {} is supported'} + ${'# comment'} | ${'comment'} | ${{}} | ${false} | ${'Comments do not match'} + ${' *.js '} | ${'index.js'} | ${{}} | ${false} | ${'Spaces are NOT ignored'} + ${'!*.js'} | ${'index.js'} | ${{}} | ${false} | ${'Negations work'} + ${'!!*.js'} | ${'index.js'} | ${{}} | ${true} | ${'double negative'} + ${'{!*.js,*.ts}'} | ${'index.js'} | ${{}} | ${false} | ${'nested negative'} + ${'{!*.js,*.ts}'} | ${'index.ts'} | ${{}} | ${true} | ${'nested negative'} + ${'{*.js,!index.js}'} | ${'index.js'} | ${{}} | ${true} | ${'nested negative does not work as expected'} + ${'{!!index.js,*.ts}'} | ${'index.js'} | ${{}} | ${false} | ${'nested negative does not work as expected'} + `( + 'assume glob "$pattern" matches "$file" is $expected - $comment', + ({ pattern, file, options, expected }: TestCase) => { + const r = mm.isMatch(file, pattern, options); + expect(r).toBe(expected); + } + ); +}); diff --git a/packages/cspell-glob/src/globHelper.ts b/packages/cspell-glob/src/globHelper.ts new file mode 100644 index 000000000000..55ff6d436a0b --- /dev/null +++ b/packages/cspell-glob/src/globHelper.ts @@ -0,0 +1,178 @@ +import { + GlobPattern, + GlobPatternNormalized, + GlobPatternWithOptionalRoot, + GlobPatternWithRoot, + PathInterface, +} from './GlobMatcherTypes'; +import * as Path from 'path'; + +const { posix } = Path; +const { relative } = posix; + +const relRegExp = /^\.[\\/]/; + +/** + * This function tries its best to determine if `fileOrGlob` is a path to a file or a glob pattern. + * @param fileOrGlob - file (with absolute path) or glob. + * @param root - absolute path to the directory that will be considered the root when testing the glob pattern. + * @param path - optional node path methods - used for testing + */ +export function fileOrGlobToGlob( + fileOrGlob: string | GlobPattern, + root: string, + path: PathInterface = Path +): GlobPatternWithRoot { + const pathToGlob = path.sep === '\\' ? (p: string) => p.replace(/\\/g, '/') : (p: string) => p; + + if (typeof fileOrGlob !== 'string') { + return { root, ...fileOrGlob }; + } + + if (fileOrGlob.startsWith(root) || relRegExp.test(fileOrGlob)) { + const rel = path.relative(root, path.resolve(root, fileOrGlob)); + return { + glob: pathToGlob(rel), + root, + }; + } + return { + glob: pathToGlob(fileOrGlob), + root, + }; +} + +type MutationsToSupportGitIgnore = [RegExp, string]; + +const mutationsNestedOnly: MutationsToSupportGitIgnore[] = [ + [/^[/]([^/]*)$/, '{$1,$1/**}'], // Only a leading slash will match root files and directories. + [/^[^/#][^/]*$/, '**/{$&,$&/**}'], // no slashes will match files names or folders + [/^[^/#][^/]*\/$/, '**/$&**/*'], // ending slash, should match any nested directory +]; + +const mutationsGeneral: MutationsToSupportGitIgnore[] = [ + [/^\//, ''], // remove leading slash to match from the root + [/\/$/, '$&**/*'], // if it ends in a slash, make sure matches the folder +]; + +const mutationsNested = mutationsNestedOnly.concat(mutationsGeneral); + +export function isGlobPatternWithOptionalRoot(g: GlobPattern): g is GlobPatternWithOptionalRoot { + return typeof g !== 'string' && typeof g.glob === 'string'; +} + +export function isGlobPatternWithRoot(g: GlobPatternWithRoot | GlobPatternWithOptionalRoot): g is GlobPatternWithRoot { + return !!g.root; +} + +function normalizePattern(pattern: string, nested: boolean): string { + pattern = pattern.replace(/^(!!)+/, ''); + const isNeg = pattern.startsWith('!'); + pattern = isNeg ? pattern.slice(1) : pattern; + const mutations = nested ? mutationsNested : mutationsGeneral; + pattern = mutations.reduce((p, [regex, replace]) => p.replace(regex, replace), pattern); + return isNeg ? '!' + pattern : pattern; +} + +export interface NormalizeOptions { + nested: boolean; + root: string; + nodePath: PathInterface; +} + +/** + * + * @param patterns - glob patterns to normalize. + * @param options - Normalization options. + */ +export function normalizeGlobPatterns(patterns: GlobPattern[], options: NormalizeOptions): GlobPatternNormalized[] { + return patterns + .map((g) => normalizeGlobPatternWithRoot(g, options)) + .map((g) => mapGlobToRoot(g, options.root)) + .filter(isNotUndefined); +} + +function normalizeGlobPatternWithRoot(g: GlobPattern, options: NormalizeOptions): GlobPatternNormalized { + const { root, nodePath: path, nested } = options; + + g = !isGlobPatternWithOptionalRoot(g) + ? { + glob: g.trim(), + root, + } + : g; + + const gr = isGlobPatternWithRoot(g) ? { ...g } : { ...g, root }; + if (gr.root.startsWith('${cwd}')) { + gr.root = path.join(path.resolve(), gr.root.replace('${cwd}', '')); + } + gr.root = path.resolve(root, path.normalize(gr.root)); + gr.glob = normalizePattern(gr.glob, nested); + + const gn: GlobPatternNormalized = { ...gr, rawGlob: g.glob, rawRoot: g.root }; + return gn; +} + +function makeRelative(from: string, to: string): string { + const rel = relative(from.replace(/\\/g, '/'), to.replace(/\\/g, '/')); + return rel; +} + +function mapGlobToRoot(glob: GlobPatternNormalized, root: string): GlobPatternNormalized | undefined { + if (glob.root === root) { + return glob; + } + + const isNeg = glob.glob.startsWith('!'); + const g = isNeg ? glob.glob.slice(1) : glob.glob; + const prefix = isNeg ? '!' : ''; + + const globIsUnderRoot = glob.root.startsWith(root); + const rootIsUnderGlob = root.startsWith(glob.root); + + // Root and Glob are not in the same part of the directory tree. + if (!globIsUnderRoot && !rootIsUnderGlob) return undefined; + + // prefix with root + if (globIsUnderRoot) { + const rel = makeRelative(root, glob.root); + + return { + ...glob, + glob: prefix + posix.join(rel, g), + root, + }; + } + + // The root is under the glob root + // The more difficult case, the glob is higher than the root + // A best effort is made, but does not do advanced matching. + + if (g.startsWith('**')) return { ...glob, root }; + + const rel = makeRelative(glob.root, root) + '/'; + if (g.startsWith(rel)) { + return { ...glob, glob: prefix + g.slice(rel.length), root }; + } + + const relParts = rel.split('/'); + const globParts = g.split('/'); + + for (let i = 0; i < relParts.length && i < globParts.length; ++i) { + const relSeg = relParts[i]; + const globSeg = globParts[i]; + // the trailing / allows for us to test against an empty segment. + if (!relSeg || globSeg === '**') { + return { ...glob, glob: prefix + globParts.slice(i).join('/'), root }; + } + if (relSeg !== globSeg && globSeg !== '*') { + break; + } + } + + return glob; +} + +function isNotUndefined(a: T | undefined): a is T { + return a !== undefined; +} diff --git a/packages/cspell-glob/src/index.test.ts b/packages/cspell-glob/src/index.test.ts new file mode 100644 index 000000000000..210fea73d93b --- /dev/null +++ b/packages/cspell-glob/src/index.test.ts @@ -0,0 +1,9 @@ +import * as index from './index'; + +describe('Validate index loads', () => { + test('the modules is ok', () => { + expect(index).toBeDefined(); + expect(index.GlobMatcher).toBeDefined(); + expect(typeof index.fileOrGlobToGlob).toBe('function'); + }); +}); diff --git a/packages/cspell-glob/src/index.ts b/packages/cspell-glob/src/index.ts index 66c6b79b24fa..7c25c1dbcbfc 100644 --- a/packages/cspell-glob/src/index.ts +++ b/packages/cspell-glob/src/index.ts @@ -1 +1,9 @@ -export * from './GlobMatcher'; +export { GlobMatchOptions, GlobMatcher } from './GlobMatcher'; +export * from './GlobMatcherTypes'; +export { + fileOrGlobToGlob, + isGlobPatternWithRoot, + isGlobPatternWithOptionalRoot, + normalizeGlobPatterns, + NormalizeOptions, +} from './globHelper'; diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts index f3ce5ae19626..fc2316be0c37 100644 --- a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts +++ b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts @@ -31,18 +31,10 @@ describe('Validate CSpellSettingsServer', () => { const left = { name: 'Left' }; const right = { name: 'Right' }; expect(mergeSettings(left, right)).toEqual({ - words: [], name: 'Left|Right', id: '|', - userWords: [], - ignoreWords: [], - flagWords: [], - patterns: [], enabledLanguageIds: [], languageSettings: [], - ignoreRegExpList: [], - dictionaries: [], - dictionaryDefinitions: [], source: { name: 'Left|Right', sources: [left, right] }, }); }); @@ -54,16 +46,8 @@ describe('Validate CSpellSettingsServer', () => { enabled: true, name: '|enabledName', id: 'left|enabledId', - words: [], - userWords: [], - ignoreWords: [], - flagWords: [], - patterns: [], enabledLanguageIds: [], languageSettings: [], - ignoreRegExpList: [], - dictionaries: [], - dictionaryDefinitions: [], source: { name: 'left|enabledName', sources: [left, enabled] }, }); }); @@ -77,16 +61,8 @@ describe('Validate CSpellSettingsServer', () => { enabled: right.enabled, name: '|', id: [left.id, right.id].join('|'), - words: [], - userWords: [], - ignoreWords: [], - flagWords: [], - patterns: [], enabledLanguageIds: [], languageSettings: [], - ignoreRegExpList: [], - dictionaries: [], - dictionaryDefinitions: [], source: { name: 'left|right', sources: [left, right] }, }); }); @@ -96,20 +72,81 @@ describe('Validate CSpellSettingsServer', () => { enabled: false, name: '|', id: '|', - words: [], - userWords: [], - ignoreWords: [], - flagWords: [], - patterns: [], enabledLanguageIds: [], languageSettings: [], - ignoreRegExpList: [], - dictionaries: [], - dictionaryDefinitions: [], source: { name: 'left|right', sources: [{ enabled: true }, { enabled: false }] }, }); }); + test('tests mergeSettings with ignorePaths, files, and overrides', () => { + const left: CSpellUserSettings = { + id: 'left', + files: ['left/**/*.*'], + ignorePaths: ['node_modules'], + overrides: [ + { + filename: '*.ts', + dictionaries: ['ts-extra'], + }, + ], + }; + const right: CSpellUserSettings = { + id: 'right', + enabled: true, + files: ['right/**/*.*'], + overrides: [{ filename: '*.jsxx', languageId: 'javascript' }], // cspell:ignore jsxx + }; + expect(mergeSettings({}, right)).toEqual(right); + expect(mergeSettings(left, {})).toEqual(left); + expect(mergeSettings(left, right)).toEqual({ + enabled: right.enabled, + name: '|', + id: [left.id, right.id].join('|'), + enabledLanguageIds: [], + languageSettings: [], + files: left.files?.concat(right.files || []), + ignorePaths: left.ignorePaths?.concat(right.ignorePaths || []), + overrides: left.overrides?.concat(right.overrides || []), + source: { name: 'left|right', sources: [left, right] }, + }); + }); + + test('tests mergeSettings with ignorePaths, files, and overrides compatibility', () => { + const left: CSpellUserSettings = { + id: 'left', + files: ['left/**/*.*'], + ignorePaths: ['node_modules'], + overrides: [ + { + filename: '*.ts', + dictionaries: ['ts-extra'], + }, + ], + }; + const right: CSpellUserSettings = { + id: 'right', + version: '0.1', + enabled: true, + files: ['right/**/*.*'], + ignorePaths: ['node_modules'], + overrides: [{ filename: '*.jsxx', languageId: 'javascript' }], // cspell:ignore jsxx + }; + expect(mergeSettings({}, right)).toEqual(right); + expect(mergeSettings(left, {})).toEqual(left); + expect(mergeSettings(left, right)).toEqual({ + enabled: right.enabled, + name: '|', + id: [left.id, right.id].join('|'), + version: right.version, + enabledLanguageIds: [], + languageSettings: [], + files: left.files?.concat(right.files || []), + ignorePaths: right.ignorePaths, + overrides: right.overrides, + source: { name: 'left|right', sources: [left, right] }, + }); + }); + test('tests mergeSettings when left/right are the same', () => { expect(mergeSettings(_defaultSettings, _defaultSettings)).toBe(_defaultSettings); }); diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts index 482137a3fbe8..4eb46ed535fa 100644 --- a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts +++ b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts @@ -26,22 +26,32 @@ export const defaultFileName = 'cspell.json'; export const ENV_CSPELL_GLOB_ROOT = 'CSPELL_GLOB_ROOT'; const cspellCosmiconfig = { + /* + * Logic of the locations: + * - Support backward compatibility with the VS Code Spell Checker + * the spell checker extension can only write to `.json` files because + * it would be too difficult to automatically modify a `.js` or `.cjs` file. + * - To support `cspell.config.js` in a VS Code environment, have a `cspell.json` import + * the `cspell.config.js`. + */ searchPlaces: [ - 'cspell.config.js', - 'cspell.config.cjs', - 'cspell.config.json', - 'cspell.config.yaml', - 'cspell.config.yml', - 'cspell.yaml', - 'cspell.yml', + // Original locations '.cspell.json', 'cspell.json', - // Alternate locations '.cSpell.json', 'cSpell.json', + // Alternate locations '.vscode/cspell.json', '.vscode/cSpell.json', '.vscode/.cspell.json', + // Standard Locations + 'cspell.config.js', // Supports dynamic config + 'cspell.config.cjs', // Supports dynamic config + 'cspell.config.json', + 'cspell.config.yaml', + 'cspell.config.yml', + 'cspell.yaml', + 'cspell.yml', ], loaders: { '.json': (_filename: string, content: string) => json.parse(content), @@ -222,9 +232,16 @@ export function readSettingsFiles(filenames: string[]): CSpellSettings { /** * Merges two lists of strings and removes duplicates. Order is NOT preserved. */ -function mergeList(left: T[] = [], right: T[] = []) { - const setOfWords = new Set([...left, ...right]); - return [...setOfWords.keys()]; +function mergeList(left: undefined, right: undefined): undefined; +function mergeList(left: T[], right: T[]): T[]; +function mergeList(left: undefined, right: T[]): T[]; +function mergeList(left: T[], right: undefined): T[]; +function mergeList(left: T[] | undefined, right: T[] | undefined): T[] | undefined; +function mergeList(left: T[] | undefined, right: T[] | undefined): T[] | undefined { + if (left === undefined) return right; + if (right === undefined) return left; + const uniqueItems = new Set([...left, ...right]); + return [...uniqueItems.keys()]; } function tagLanguageSettings(tag: string, settings: LanguageSetting[] = []): LanguageSetting[] { @@ -271,9 +288,9 @@ function merge(left: CSpellSettings, right: CSpellSettings): CSpellSettings { const leftId = left.id || left.languageId || ''; const rightId = right.id || right.languageId || ''; - const includeRegExpList = takeRightThenLeft(left.includeRegExpList, right.includeRegExpList); + const includeRegExpList = takeRightOtherwiseLeft(left.includeRegExpList, right.includeRegExpList); - const optionals = includeRegExpList.length ? { includeRegExpList } : {}; + const optionals = includeRegExpList?.length ? { includeRegExpList } : {}; const settings: CSpellSettings = { ...left, @@ -295,7 +312,11 @@ function merge(left: CSpellSettings, right: CSpellSettings): CSpellSettings { tagLanguageSettings(rightId, right.languageSettings) ), enabled: right.enabled !== undefined ? right.enabled : left.enabled, + files: mergeList(left.files, right.files), + ignorePaths: versionBasedMergeList(left.ignorePaths, right.ignorePaths, right.version), + overrides: versionBasedMergeList(left.overrides, right.overrides, right.version), source: mergeSources(left, right), + globRoot: undefined, import: undefined, __imports: mergeImportRefs(left, right), __importRef: undefined, @@ -303,6 +324,17 @@ function merge(left: CSpellSettings, right: CSpellSettings): CSpellSettings { return settings; } +function versionBasedMergeList( + left: T[] | undefined, + right: T[] | undefined, + version: CSpellSettings['version'] +): T[] | undefined { + if (version === '0.1') { + return takeRightOtherwiseLeft(left, right); + } + return mergeList(left, right); +} + function hasLeftAncestor(s: CSpellSettings, left: CSpellSettings): boolean { return hasAncestor(s, left, 0); } @@ -332,11 +364,20 @@ export function mergeInDocSettings(left: CSpellSettings, right: CSpellSettings): return merged; } -function takeRightThenLeft(left: T[] = [], right: T[] = []) { - if (right.length) { +/** + * If right is non-empty return right, otherwise return left. + * @param left - left hand values + * @param right - right hand values + */ +function takeRightOtherwiseLeft(left: undefined, right: undefined): undefined; +function takeRightOtherwiseLeft(left: T[], right: undefined): T[]; +function takeRightOtherwiseLeft(left: undefined, right: T[]): T[]; +function takeRightOtherwiseLeft(left: T[] | undefined, right: T[] | undefined): T[] | undefined; +function takeRightOtherwiseLeft(left: T[] | undefined, right: T[] | undefined): T[] | undefined { + if (right?.length) { return right; } - return left; + return left || right; } export function calcOverrideSettings(settings: CSpellSettings, filename: string): CSpellSettings { diff --git a/packages/cspell-types/cspell.schema.json b/packages/cspell-types/cspell.schema.json index b8841bd4dba4..144c4d2ce554 100644 --- a/packages/cspell-types/cspell.schema.json +++ b/packages/cspell-types/cspell.schema.json @@ -488,6 +488,13 @@ }, "type": "array" }, + "files": { + "description": "Glob patterns of files to be checked. Glob patterns are relative to the `globRoot` of the configuration file that defines them.", + "items": { + "$ref": "#/definitions/Glob" + }, + "type": "array" + }, "flagWords": { "description": "list of words to always be considered incorrect.", "items": { @@ -611,6 +618,10 @@ "version": { "default": "0.2", "description": "Configuration format version of the setting file.", + "enum": [ + "0.2", + "0.1" + ], "type": "string" }, "words": { diff --git a/packages/cspell-types/src/settings/CSpellSettingsDef.ts b/packages/cspell-types/src/settings/CSpellSettingsDef.ts index 1f7e49d822cd..ec9780f0a5a4 100644 --- a/packages/cspell-types/src/settings/CSpellSettingsDef.ts +++ b/packages/cspell-types/src/settings/CSpellSettingsDef.ts @@ -29,7 +29,7 @@ export interface FileSettings extends ExtendableSettings { * Configuration format version of the setting file. * @default "0.2" */ - version?: string | '0.2' | '0.1'; + version?: '0.2' | '0.1'; /** Words to add to dictionary -- should only be in the user config file. */ userWords?: string[]; @@ -54,11 +54,11 @@ export interface FileSettings extends ExtendableSettings { */ globRoot?: FsPath; - // /** - // * Glob patterns of files to be checked. - // * Glob patterns are relative to the `globRoot` of the configuration file that defines them. - // */ - // files?: Glob[]; + /** + * Glob patterns of files to be checked. + * Glob patterns are relative to the `globRoot` of the configuration file that defines them. + */ + files?: Glob[]; /** * Glob patterns of files to be ignored diff --git a/packages/cspell/.vscode/launch.json b/packages/cspell/.vscode/launch.json index 64e8a07bdc74..007aa3b09a29 100644 --- a/packages/cspell/.vscode/launch.json +++ b/packages/cspell/.vscode/launch.json @@ -10,10 +10,8 @@ "name": "cspell: Run", "program": "${workspaceRoot}/bin.js", "args": [ - "--root=..", - "-v", - "**/*.ts", - "*.md" + "-c=../../cspell.json", + "**/*" ], "cwd": "${workspaceRoot}", "outFiles": [ diff --git a/packages/cspell/cSpell.json b/packages/cspell/cSpell.json index 8c1bec4febcd..22b40a906882 100644 --- a/packages/cspell/cSpell.json +++ b/packages/cspell/cSpell.json @@ -1,12 +1,13 @@ { - "version": "0.1", + "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/master/cspell.schema.json", + "version": "0.2", "id": "cspell-project-config", "name": "cspell Project Config", "language": "en", "words": [ "configstore" ], - "maxNumberOfProblems": 10000, + "maxNumberOfProblems": 1000, "ignorePaths": [ "dictionaries/**", "migrated_dictionaries/**", @@ -21,6 +22,5 @@ ".vscode/**" ], "dictionaryDefinitions": [], - "ignoreWords": [ - ] + "ignoreWords": [] } diff --git a/packages/cspell/src/__snapshots__/app.test.ts.snap b/packages/cspell/src/__snapshots__/app.test.ts.snap index 535c9237c998..3db76ab87ec4 100644 --- a/packages/cspell/src/__snapshots__/app.test.ts.snap +++ b/packages/cspell/src/__snapshots__/app.test.ts.snap @@ -1,10 +1,10 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Validate cli app LICENSE 1`] = `Array []`; +exports[`Validate cli app LICENSE Expect Error: undefined 1`] = `Array []`; -exports[`Validate cli app bad config 1`] = `Array []`; +exports[`Validate cli app bad config Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app check LICENSE 1`] = ` +exports[`Validate cli app check LICENSE Expect Error: undefined 1`] = ` Array [ "MIT License", "", @@ -31,7 +31,7 @@ Array [ ] `; -exports[`Validate cli app check help 1`] = ` +exports[`Validate cli app check help Expect Error: outputHelp 1`] = ` Array [ "Usage: cspell check [options] ", "", @@ -47,9 +47,9 @@ Array [ ] `; -exports[`Validate cli app check missing 1`] = `Array []`; +exports[`Validate cli app check missing Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app check with spelling errors 1`] = ` +exports[`Validate cli app check with spelling errors Expect Error: [Function CheckFailed] 1`] = ` Array [ " [NEDERLANDS]", " U wordt vriendelijk verzocht om dit bestand (\\"README_nl_NL.txt\\")", @@ -148,15 +148,15 @@ Array [ ] `; -exports[`Validate cli app cspell-bad.json 1`] = `Array []`; +exports[`Validate cli app cspell-bad.json Expect Error: undefined 1`] = `Array []`; -exports[`Validate cli app cspell-import-missing.json 1`] = `Array []`; +exports[`Validate cli app cspell-import-missing.json Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app current_file --verbose 1`] = `Array []`; +exports[`Validate cli app current_file --verbose Expect Error: undefined 1`] = `Array []`; -exports[`Validate cli app current_file 1`] = `Array []`; +exports[`Validate cli app current_file Expect Error: undefined 1`] = `Array []`; -exports[`Validate cli app current_file languageId 1`] = `Array []`; +exports[`Validate cli app current_file languageId Expect Error: undefined 1`] = `Array []`; exports[`Validate cli app link 1`] = `Array []`; @@ -168,11 +168,11 @@ exports[`Validate cli app link ls 1`] = `Array []`; exports[`Validate cli app link remove 1`] = `Array []`; -exports[`Validate cli app must find force no error 1`] = `Array []`; +exports[`Validate cli app must find force no error Expect Error: undefined 1`] = `Array []`; -exports[`Validate cli app must find with error 1`] = `Array []`; +exports[`Validate cli app must find with error Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app no-args 1`] = ` +exports[`Validate cli app no-args Expect Error: outputHelp 1`] = ` Array [ "Usage: cspell lint [options] [files...]", "", @@ -225,13 +225,13 @@ Array [ ] `; -exports[`Validate cli app not found error by default 1`] = `Array []`; +exports[`Validate cli app not found error by default Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app samples/Dutch.txt 1`] = `Array []`; +exports[`Validate cli app samples/Dutch.txt Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app trace hello 1`] = `Array []`; +exports[`Validate cli app trace hello Expect Error: undefined 1`] = `Array []`; -exports[`Validate cli app trace help 1`] = ` +exports[`Validate cli app trace help Expect Error: outputHelp 1`] = ` Array [ "Usage: cspell trace [options] ", "", @@ -251,12 +251,12 @@ Array [ ] `; -exports[`Validate cli app trace missing dictionary 1`] = `Array []`; +exports[`Validate cli app trace missing dictionary Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app trace not-in-any-dictionary 1`] = `Array []`; +exports[`Validate cli app trace not-in-any-dictionary Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app with spelling errors --debug Dutch.txt 1`] = `Array []`; +exports[`Validate cli app with spelling errors --debug Dutch.txt Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app with spelling errors --silent Dutch.txt 1`] = `Array []`; +exports[`Validate cli app with spelling errors --silent Dutch.txt Expect Error: [Function CheckFailed] 1`] = `Array []`; -exports[`Validate cli app with spelling errors Dutch.txt 1`] = `Array []`; +exports[`Validate cli app with spelling errors Dutch.txt Expect Error: [Function CheckFailed] 1`] = `Array []`; diff --git a/packages/cspell/src/app.test.ts b/packages/cspell/src/app.test.ts index 32fc55120766..211531136478 100644 --- a/packages/cspell/src/app.test.ts +++ b/packages/cspell/src/app.test.ts @@ -131,9 +131,7 @@ describe('Validate cli', () => { ${'must find force no error'} | ${['*.not', '--no-must-find-files']} | ${undefined} | ${true} | ${false} | ${false} ${'cspell-bad.json'} | ${['-c', pathSamples('cspell-bad.json'), __filename]} | ${undefined} | ${true} | ${false} | ${false} ${'cspell-import-missing.json'} | ${['-c', pathSamples('linked/cspell-import-missing.json'), __filename]} | ${app.CheckFailed} | ${true} | ${false} | ${false} - `('app $msg', executeTest); - - async function executeTest({ testArgs, errorCheck, eError, eLog, eInfo }: TestCase) { + `('app $msg Expect Error: $errorCheck', async ({ testArgs, errorCheck, eError, eLog, eInfo }: TestCase) => { const commander = getCommander(); const args = argv(...testArgs); const result = app.run(commander, args); @@ -146,7 +144,7 @@ describe('Validate cli', () => { eLog ? expect(log).toHaveBeenCalled() : expect(log).not.toHaveBeenCalled(); eInfo ? expect(info).toHaveBeenCalled() : expect(info).not.toHaveBeenCalled(); expect(capture.text).toMatchSnapshot(); - } + }); test.each` msg | testArgs | errorCheck | eError | eLog | eInfo @@ -155,9 +153,7 @@ describe('Validate cli', () => { ${'link list'} | ${['link', 'list']} | ${undefined} | ${false} | ${true} | ${false} ${'link add'} | ${['link', 'add', 'cspell-dict-cpp/cspell-ext.json']} | ${undefined} | ${false} | ${true} | ${false} ${'link remove'} | ${['link', 'remove', 'cspell-dict-cpp/cspell-ext.json']} | ${undefined} | ${false} | ${true} | ${false} - `('app $msg', executeLinkTest); - - async function executeLinkTest({ testArgs, errorCheck, eError, eLog, eInfo }: TestCase) { + `('app $msg', async ({ testArgs, errorCheck, eError, eLog, eInfo }: TestCase) => { listGlobalImports.mockImplementation(_listGlobalImports()); addPathsToGlobalImports.mockImplementation(_addPathsToGlobalImports()); removePathsFromGlobalImports.mockImplementation(_removePathsFromGlobalImports()); @@ -173,7 +169,7 @@ describe('Validate cli', () => { eLog ? expect(log).toHaveBeenCalled() : expect(log).not.toHaveBeenCalled(); eInfo ? expect(info).toHaveBeenCalled() : expect(info).not.toHaveBeenCalled(); expect(capture.text).toMatchSnapshot(); - } + }); }); function _listGlobalImports(): typeof Link['listGlobalImports'] { diff --git a/packages/cspell/src/app.ts b/packages/cspell/src/app.ts index 85f2f89895c9..c260a4717866 100644 --- a/packages/cspell/src/app.ts +++ b/packages/cspell/src/app.ts @@ -175,11 +175,11 @@ export async function run(program?: commander.Command, argv?: string[]): Promise .action((files: string[], options: Options) => { const { mustFindFiles } = options; const emitters: Emitters = getEmitters(options); - if (!files.length) { - spellCheckCommand.outputHelp(); - throw new CheckFailed('outputHelp', 1); - } return App.lint(files, options, emitters).then((result) => { + if (!files.length && !result.files) { + spellCheckCommand.outputHelp(); + throw new CheckFailed('outputHelp', 1); + } if (options.summary && !options.silent) { console.error( 'CSpell: Files checked: %d, Issues found: %d in %d files', diff --git a/packages/cspell/src/application.test.ts b/packages/cspell/src/application.test.ts index fbffed47cfb2..3ea6d15a2b8a 100644 --- a/packages/cspell/src/application.test.ts +++ b/packages/cspell/src/application.test.ts @@ -6,6 +6,10 @@ const getStdinResult = { value: '', }; +const samplesRoot = path.resolve(__dirname, '../samples'); + +const sampleOptions = { root: samplesRoot }; + jest.mock('get-stdin', () => { return jest.fn(() => Promise.resolve(getStdinResult.value)); }); @@ -14,8 +18,8 @@ describe('Validate the Application', () => { jest.setTimeout(10000); // make sure we have enough time on Travis. test('Tests running the application', () => { - const files = ['samples/text.txt']; - const options = {}; + const files = ['text.txt']; + const options = sampleOptions; const logger = new Logger(); const lint = App.lint(files, options, logger); return lint.then((result) => { @@ -28,8 +32,8 @@ describe('Validate the Application', () => { }); test('Tests running the application verbose', () => { - const files = ['samples/text.txt']; - const options = { verbose: true }; + const files = ['text.txt']; + const options = { ...sampleOptions, verbose: true }; const logger = new Logger(); const lint = App.lint(files, options, logger); return lint.then((result) => { @@ -42,8 +46,8 @@ describe('Validate the Application', () => { }); test('Tests running the application words only', () => { - const files = ['samples/text.txt']; - const options = { wordsOnly: true, unique: true }; + const files = ['text.txt']; + const options = { ...sampleOptions, wordsOnly: true, unique: true }; const logger = new Logger(); const lint = App.lint(files, options, logger); return lint.then((result) => { @@ -96,7 +100,7 @@ describe('Validate the Application', () => { test('running the application from stdin', async () => { const files = ['stdin']; - const options = { wordsOnly: true, unique: true }; + const options = { ...sampleOptions, wordsOnly: true, unique: true }; const logger = new Logger(); // cspell:ignore texxt getStdinResult.value = ` @@ -132,8 +136,14 @@ describe('Application, Validate Samples', () => { sampleTests().map((sample) => test(`Test file: "${sample.file}"`, async () => { const logger = new Logger(); - const { file, issues, options = { wordsOnly: true, unique: false } } = sample; - const result = await App.lint([file], options, logger); + const root = path.resolve(path.dirname(sample.file)); + const { file, issues, options: sampleOptions = {} } = sample; + const options = { + root, + ...sampleOptions, + }; + + const result = await App.lint([path.resolve(file)], options, logger); expect(result.files).toBe(1); expect(logger.issues.map((issue) => issue.text)).toEqual(issues); expect(result.issues).toBe(issues.length); diff --git a/packages/cspell/src/application.ts b/packages/cspell/src/application.ts index 757a9a1d7f2c..a6315ecce5a1 100644 --- a/packages/cspell/src/application.ts +++ b/packages/cspell/src/application.ts @@ -5,7 +5,7 @@ import { calcExcludeGlobInfo, extractGlobExcludesFromConfig, extractPatterns, - normalizeExcludeGlobsToRoot, + normalizeGlobsToRoot, } from './util/glob'; import * as cspell from 'cspell-lib'; import * as fsp from 'fs-extra'; @@ -13,7 +13,7 @@ import * as path from 'path'; import * as commentJson from 'comment-json'; import * as util from './util/util'; import { traceWords, TraceResult, CheckTextInfo, getDictionary } from 'cspell-lib'; -import { CSpellSettings, CSpellUserSettings } from '@cspell/cspell-types'; +import { CSpellSettings, CSpellUserSettings, Glob } from '@cspell/cspell-types'; import getStdin from 'get-stdin'; export { TraceResult, IncludeExcludeFlag } from 'cspell-lib'; import { IOptions } from './util/IOptions'; @@ -292,13 +292,23 @@ function runLint(cfg: CSpellApplicationConfiguration) { } async function run(): Promise { - header(); - if (cfg.root) { process.env[cspell.ENV_CSPELL_GLOB_ROOT] = cfg.root; } - const configInfo: ConfigInfo = await readConfig(cfg.configFile); + const configInfo: ConfigInfo = await readConfig(cfg.configFile, cfg.root); + const cliGlobs: Glob[] = cfg.files; + const allGlobs: Glob[] = cliGlobs.concat(configInfo.config.files || []); + const combinedGlobs = normalizeGlobsToRoot(allGlobs, cfg.root, false); + const includeGlobs = combinedGlobs.filter((g) => !g.startsWith('!')); + const excludeGlobs = combinedGlobs.filter((g) => g.startsWith('!')); + const fileGlobs: string[] = includeGlobs; + if (!fileGlobs.length) { + // Nothing to do. + return runResult(); + } + header(fileGlobs); + cfg.info(`Config Files Found:\n ${configInfo.source}\n`, MessageTypes.Info); const configErrors = await countConfigErrors(configInfo); @@ -306,17 +316,19 @@ function runLint(cfg: CSpellApplicationConfiguration) { // Get Exclusions from the config files. const { root } = cfg; - const ignoreGlobs = normalizeExcludeGlobsToRoot(configInfo.config.ignorePaths || [], root); + const ignoreGlobs = normalizeGlobsToRoot(configInfo.config.ignorePaths || [], root, true).concat(excludeGlobs); const globOptions = { root, cwd: root, ignore: ignoreGlobs }; const exclusionGlobs = extractGlobExcludesFromConfig(root, configInfo.source, configInfo.config).concat( cfg.excludes ); - const files = filterFiles(await findFiles(cfg.files, globOptions), exclusionGlobs); + const files = filterFiles(await findFiles(fileGlobs, globOptions), exclusionGlobs); return processFiles(fileLoader(files), configInfo, files.length); } - function header() { + function header(files: string[]) { + const formattedFiles = files.length > 100 ? files.slice(0, 100).concat(['...']) : files; + cfg.info( ` cspell; @@ -327,7 +339,7 @@ Options: exclude: ${extractPatterns(cfg.excludes) .map((a) => a.glob.glob) .join('\n ')} - files: ${cfg.files} + files: ${formattedFiles} wordsOnly: ${yesNo(!!cfg.options.wordsOnly)} unique: ${yesNo(!!cfg.options.unique)} `, @@ -359,12 +371,12 @@ Options: } } -async function readConfig(configFile: string | undefined): Promise { +async function readConfig(configFile: string | undefined, root: string | undefined): Promise { if (configFile) { const config = (await cspell.loadConfig(configFile)) || {}; return { source: configFile, config }; } - const config = await cspell.searchForConfig(); + const config = await cspell.searchForConfig(root); return { source: config?.__importRef?.filename || 'not found', config: config || {} }; } @@ -374,7 +386,7 @@ function runResult(init: Partial = {}): RunResult { } export async function trace(words: string[], options: TraceOptions): Promise { - const configFile = await readConfig(options.config); + const configFile = await readConfig(options.config, undefined); const config = cspell.mergeSettings(cspell.getDefaultSettings(), cspell.getGlobalSettings(), configFile.config); const results = await traceWords(words, config); return results; @@ -383,7 +395,7 @@ export async function trace(words: string[], options: TraceOptions): Promise { - const pSettings = readConfig(options.config); + const pSettings = readConfig(options.config, path.dirname(filename)); const [foundSettings, text] = await Promise.all([pSettings, readFile(filename)]); const settingsFromCommandLine = util.clean({ languageId: options.languageId || undefined, diff --git a/packages/cspell/src/util/glob.test.ts b/packages/cspell/src/util/glob.test.ts index 363d6e0313d0..2d53de623f35 100644 --- a/packages/cspell/src/util/glob.test.ts +++ b/packages/cspell/src/util/glob.test.ts @@ -1,8 +1,8 @@ import * as path from 'path'; -import { _testing_, calcGlobs, normalizeExcludeGlobsToRoot } from './glob'; +import { calcGlobs, normalizeGlobsToRoot } from './glob'; import { GlobMatcher } from 'cspell-glob'; import mm = require('micromatch'); -// import minimatch = require('minimatch'); +import minimatch = require('minimatch'); const getStdinResult = { value: '', @@ -12,30 +12,55 @@ jest.mock('get-stdin', () => { return jest.fn(() => Promise.resolve(getStdinResult.value)); }); -describe('Validate internal functions', () => { - test('normalizePattern relative', () => { - const root = process.cwd(); - const r = _testing_.normalizePattern('../../packages/**/*.ts', root); - expect(r.root).toBe(path.dirname(path.dirname(root))); - expect(r.pattern).toBe('packages/**/*.ts'); - }); +describe('Validate minimatch assumptions', () => { + interface TestCase { + pattern: string; + file: string; + options: minimatch.IOptions; + expected: boolean; + } - test('normalizePattern relative absolute', () => { - const root = process.cwd(); - const p = '/packages/**/*.ts'; - const r = _testing_.normalizePattern(p, root); - expect(r.root).toBe(root); - expect(r.pattern).toBe(p); - }); + const jsPattern = '*.{js,jsx}'; + const mdPattern = '*.md'; + const nodePattern = '{node_modules,node_modules/**}'; + const nestedPattern = `{**/temp/**,{${jsPattern},${mdPattern},${nodePattern}}}`; - test('normalizePattern absolute', () => { - const root = process.cwd(); - const p = path.join(__dirname, '**', '*.ts'); - const r = _testing_.normalizePattern(p, root); - expect(r.root).toBe(path.sep); - expect(r.pattern).toBe(p); + test.each` + pattern | file | options | expected | comment + ${'*.json'} | ${'package.json'} | ${{}} | ${true} | ${''} + ${'**/*.json'} | ${'package.json'} | ${{}} | ${true} | ${''} + ${'node_modules'} | ${'node_modules/cspell/package.json'} | ${{}} | ${false} | ${''} + ${'node_modules/'} | ${'node_modules/cspell/package.json'} | ${{}} | ${false} | ${'tailing slash (not like .gitignore)'} + ${'node_modules/'} | ${'node_modules'} | ${{}} | ${false} | ${''} + ${'node_modules/**'} | ${'node_modules/cspell/package.json'} | ${{}} | ${true} | ${''} + ${'node_modules/**/*'} | ${'node_modules/package.json'} | ${{}} | ${true} | ${''} + ${'node_modules/**'} | ${'node_modules'} | ${{}} | ${false} | ${'Note: Minimatch and Micromatch do not give the same result.'} + ${'node_modules/**/*'} | ${'node_modules'} | ${{}} | ${false} | ${''} + ${'*.json'} | ${'src/package.json'} | ${{}} | ${false} | ${''} + ${'*.json'} | ${'src/package.json'} | ${{ matchBase: true }} | ${true} | ${'check matchBase behavior, option not used by cspell'} + ${'*.yml'} | ${'.github/workflows/test.yml'} | ${{ matchBase: true }} | ${true} | ${'check matchBase behavior, option not used by cspell'} + ${'**/*.yml'} | ${'.github/workflows/test.yml'} | ${{}} | ${false} | ${''} + ${'**/*.yml'} | ${'.github/workflows/test.yml'} | ${{ dot: true }} | ${true} | ${'dot is used by default for excludes'} + ${'{*.json,*.yaml}'} | ${'package.json'} | ${{}} | ${true} | ${''} + ${nestedPattern} | ${'index.js'} | ${{}} | ${true} | ${'Nested {} is supported'} + ${nestedPattern} | ${'node_modules/cspell/package.json'} | ${{}} | ${true} | ${'Nested {} is supported'} + ${nestedPattern} | ${'testing/temp/file.bin'} | ${{}} | ${true} | ${'Nested {} is supported'} + ${'# comment'} | ${'comment'} | ${{}} | ${false} | ${'Comments do not match'} + ${' *.js '} | ${'index.js'} | ${{}} | ${true} | ${'Spaces are ignored'} + ${'!*.js'} | ${'index.js'} | ${{}} | ${false} | ${'Negations work'} + ${'!!*.js'} | ${'index.js'} | ${{}} | ${true} | ${'double negative'} + ${'{!*.js,*.ts}'} | ${'index.js'} | ${{}} | ${false} | ${'nested negative - do not work (are not expected to)'} + ${'{!*.js,*.ts}'} | ${'!index.js'} | ${{}} | ${true} | ${'nested negative - exact match'} + ${'{!*.js,*.ts}'} | ${'index.ts'} | ${{}} | ${true} | ${'nested negative'} + ${'{*.js,!index.js}'} | ${'index.js'} | ${{}} | ${true} | ${'nested negative does not work as expected'} + ${'{!!index.js,*.ts}'} | ${'index.js'} | ${{}} | ${false} | ${'nested negative does not work as expected'} + `('assume glob "$pattern" matches "$file" is $expected', ({ pattern, file, options, expected }: TestCase) => { + const r = minimatch(file, pattern, options); + expect(r).toBe(expected); }); +}); +describe('Validate internal functions', () => { test('exclude globs default', () => { const ex: string[] = []; const r = calcGlobs(ex); @@ -62,14 +87,15 @@ describe('Validate internal functions', () => { glob: string; globRoot: string; root: string; - expectedGlob: string[]; + expectedGlobs: string[]; file: string; expectedToMatch: boolean; } test.each` - glob | globRoot | root | expectedGlob | file | expectedToMatch + glob | globRoot | root | expectedGlobs | file | expectedToMatch ${'*.json'} | ${'.'} | ${'.'} | ${['**/{*.json,*.json/**}']} | ${'./package.json'} | ${true} + ${'*.json'} | ${'.'} | ${'.'} | ${['**/{*.json,*.json/**}']} | ${'./.git/package.json'} | ${true} ${'*.json'} | ${'./project/p1'} | ${'.'} | ${['project/p1/**/{*.json,*.json/**}']} | ${'./project/p1/package.json'} | ${true} ${'*.json'} | ${'./project/p1'} | ${'.'} | ${['project/p1/**/{*.json,*.json/**}']} | ${'./project/p1/src/package.json'} | ${true} ${'*.json'} | ${'.'} | ${'./project/p2'} | ${['**/{*.json,*.json/**}']} | ${'./project/p2/package.json'} | ${true} @@ -77,20 +103,55 @@ describe('Validate internal functions', () => { ${'**/src/*.json'} | ${'.'} | ${'./project/p2'} | ${['**/src/*.json']} | ${'./project/p2/x/src/config.json'} | ${true} ${'**/src/*.json'} | ${'./project/p1'} | ${'.'} | ${['project/p1/**/src/*.json']} | ${'./project/p1/src/config.json'} | ${true} `( - 'mapGlobToRoot "$glob"@"$globRoot" -> "@root" = "$expectedGlob"', - ({ glob, globRoot, root, expectedGlob, file, expectedToMatch }: TestMapGlobToRoot) => { + 'mapGlobToRoot exclude "$glob"@"$globRoot" -> "$root" = "$expectedGlobs"', + ({ glob, globRoot, root, expectedGlobs, file, expectedToMatch }: TestMapGlobToRoot) => { + globRoot = path.resolve(globRoot); + root = path.resolve(root); + file = path.resolve(file); + const globMatcher = new GlobMatcher(glob, { + root: globRoot, + mode: 'exclude', + }); + const patterns = globMatcher.patterns; + const r = normalizeGlobsToRoot(patterns, root, true); + expect(r).toEqual(expectedGlobs); + + const relToRoot = path.relative(root, file); + + expect(globMatcher.match(file)).toBe(expectedToMatch); + expect(mm.isMatch(relToRoot, expectedGlobs, { dot: true })).toBe(expectedToMatch); + } + ); + + test.each` + glob | globRoot | root | expectedGlobs | file | expectedToMatch + ${'*.json'} | ${'.'} | ${'.'} | ${['*.json']} | ${'./package.json'} | ${true} + ${'*.json'} | ${'.'} | ${'.'} | ${['*.json']} | ${'./.git/package.json'} | ${false} + ${'*.json'} | ${'./project/p1'} | ${'.'} | ${['project/p1/*.json']} | ${'./project/p1/package.json'} | ${true} + ${'*.json'} | ${'./project/p1'} | ${'.'} | ${['project/p1/*.json']} | ${'./project/p1/src/package.json'} | ${false} + ${'*.json'} | ${'.'} | ${'./project/p2'} | ${[]} | ${'./project/p2/package.json'} | ${false} + ${'**/*.json'} | ${'.'} | ${'./project/p2'} | ${['**/*.json']} | ${'./project/p2/package.json'} | ${true} + ${'src/*.json'} | ${'.'} | ${'./project/p2'} | ${[]} | ${''} | ${false} + ${'**/src/*.json'} | ${'.'} | ${'./project/p2'} | ${['**/src/*.json']} | ${'./project/p2/x/src/config.json'} | ${true} + ${'**/src/*.json'} | ${'./project/p1'} | ${'.'} | ${['project/p1/**/src/*.json']} | ${'./project/p1/src/config.json'} | ${true} + `( + 'mapGlobToRoot include "$glob"@"$globRoot" -> "$root" = "$expectedGlobs"', + ({ glob, globRoot, root, expectedGlobs, file, expectedToMatch }: TestMapGlobToRoot) => { globRoot = path.resolve(globRoot); root = path.resolve(root); file = path.resolve(file); - const globMatcher = new GlobMatcher(glob, globRoot); + const globMatcher = new GlobMatcher(glob, { + root: globRoot, + mode: 'include', + }); const patterns = globMatcher.patterns; - const r = normalizeExcludeGlobsToRoot(patterns, root); - expect(r).toEqual(expectedGlob); + const r = normalizeGlobsToRoot(patterns, root, false); + expect(r).toEqual(expectedGlobs); const relToRoot = path.relative(root, file); expect(globMatcher.match(file)).toBe(expectedToMatch); - expect(mm.isMatch(relToRoot, expectedGlob)).toBe(expectedToMatch); + expect(mm.isMatch(relToRoot, expectedGlobs)).toBe(expectedToMatch); } ); }); diff --git a/packages/cspell/src/util/glob.ts b/packages/cspell/src/util/glob.ts index ad34d090e573..ce29ea655d7f 100644 --- a/packages/cspell/src/util/glob.ts +++ b/packages/cspell/src/util/glob.ts @@ -1,10 +1,8 @@ import glob, { IGlob } from 'glob'; import * as path from 'path'; -import * as fsp from 'fs-extra'; import { IOptions } from './IOptions'; -import { GlobMatcher, GlobPatternWithRoot } from 'cspell-glob'; +import { GlobMatcher, GlobPatternWithRoot, fileOrGlobToGlob, normalizeGlobPatterns } from 'cspell-glob'; import { CSpellUserSettings, Glob } from '@cspell/cspell-types'; -import { posix, relative } from 'path'; export interface GlobOptions extends IOptions { cwd?: string; @@ -14,67 +12,35 @@ export interface GlobOptions extends IOptions { const defaultExcludeGlobs = ['node_modules/**']; +// Note this is to allow experimenting with using a single glob +const useJoinPatterns = process.env['CSPELL_SINGLE_GLOB']; + /** - * Attempt to normalize a pattern based upon the root. - * If the pattern is absolute, check to see if it exists and adjust the root, otherwise it is assumed to be based upon the current root. - * If the pattern starts with a relative path, adjust the root to match. - * The challenge is with the patterns that begin with `/`. Is is an absolute path or relative pattern? - * @param pat glob pattern - * @param root absolute path | empty - * @returns the adjusted root and pattern. + * + * @param pattern - glob patterns and NOT file paths. It can be a file path turned into a glob. + * @param options - search options. */ -function normalizePattern(pat: string, root: string): PatternRoot { - // Absolute pattern - if (path.isAbsolute(pat)) { - const dir = findBaseDir(pat); - if (dir.length > 1 && exists(dir)) { - // Assume it is an absolute path - return { - pattern: pat, - root: path.sep, - }; - } - } - // normal pattern - if (!/^\.\./.test(pat)) { - return { - pattern: pat, - root, - }; - } - // relative pattern - pat = path.sep === '\\' ? pat.replace(/\\/g, '/') : pat; - const patParts = pat.split('/'); - const rootParts = root.split(path.sep); - let i = 0; - for (; i < patParts.length && patParts[i] === '..'; ++i) { - rootParts.pop(); - } - return { - pattern: patParts.slice(i).join('/'), - root: rootParts.join(path.sep), - }; -} - export async function globP(pattern: string | string[], options?: GlobOptions): Promise { const root = options?.root || process.cwd(); - const opts = options || {}; + const opts = options || { root }; const rawPatterns = typeof pattern === 'string' ? [pattern] : pattern; - const normPatterns = rawPatterns.map((pat) => normalizePattern(pat, root)); + const normPatterns = useJoinPatterns ? joinPatterns(rawPatterns) : rawPatterns; const globPState: GlobPState = { - options: { ...opts }, + options: { ...opts, root }, }; const globResults = normPatterns.map(async (pat) => { - globPState.options = { ...opts, root: pat.root, cwd: pat.root }; - const absolutePaths = (await _globP(pat.pattern, globPState)).map((filename) => - path.resolve(pat.root, filename) - ); + globPState.options = { ...opts, root: root, cwd: root }; + const absolutePaths = (await _globP(pat, globPState)).map((filename) => path.resolve(root, filename)); const relativeToRoot = absolutePaths.map((absFilename) => path.relative(root, absFilename)); return relativeToRoot; }); - const results = (await Promise.all(globResults)).reduce((prev, next) => prev.concat(next), []); - return results; + const results = new Set(flatten(await Promise.all(globResults))); + return [...results]; +} + +function joinPatterns(globs: string[]): string[] { + return globs.length <= 1 ? globs : [`{${globs.join(',')}}`]; } interface GlobPState { @@ -98,29 +64,6 @@ function _globP(pattern: string, state: GlobPState): Promise { }); } -interface PatternRoot { - pattern: string; - root: string; -} - -function findBaseDir(pat: string) { - const globChars = /[*@()?|[\]{},]/; - while (globChars.test(pat)) { - pat = path.dirname(pat); - } - return pat; -} - -function exists(filename: string): boolean { - try { - fsp.accessSync(filename); - } catch (e) { - return false; - } - - return true; -} - export function calcGlobs(commandLineExclude: string[] | undefined): { globs: string[]; source: string } { const globs = (commandLineExclude || []) .map((glob) => glob.split(/(?= 0; - const globIsUnderRoot = glob.root.startsWith(root); - const rootIsUnderGlob = root.startsWith(glob.root); - - // Root and Glob are not in the same part of the directory tree. - if (!globIsUnderRoot && !rootIsUnderGlob) return undefined; - - // prefix with root - if (globIsUnderRoot) { - const rel = makeRelative(root, glob.root); - - const globs = [posix.join(rel, glob.glob)]; - if (!globHasSlash) { - // need to add ** to be able to match deeper. - globs.push(posix.join(rel, '**', glob.glob)); - } - return globs; - } - - // The root is under the glob root - // The more difficult case, the glob is higher than the root - // A best effort is made, but does not do advanced matching. - - // no slashes matches everything "*.json" - if (!globHasSlash) return glob.glob; - - if (glob.glob.startsWith('**')) return glob.glob; - - const rel = makeRelative(glob.root, root); - if (glob.glob.startsWith(rel)) { - return glob.glob.slice(rel.length); - } - - const rel2 = '/' + rel; - if (glob.glob.startsWith(rel2)) return glob.glob.slice(rel2.length); - - return undefined; -} - -function flatten(s: (string | string[])[]): string[] { - function* f(): Iterable { - for (const i of s) { - if (Array.isArray(i)) { - yield* i; - continue; - } - yield i; - } - } - - return [...f()]; -} - -function mapToGlobPatternWithRoot(g: Glob, root: string): GlobPatternWithRoot { - if (typeof g === 'string') { - return { - glob: g, - root, - }; - } - - return { - glob: g.glob, - root: g.root || root, - }; -} - /** * * @param globs Glob patterns. * @param root */ -export function normalizeExcludeGlobsToRoot(globs: Glob[], root: string): string[] { - const withRoots = globs.map((g) => mapToGlobPatternWithRoot(g, root)); - return flatten(withRoots.map((g) => mapGlobToRoot(g, root)).filter(isNotUndefined)); +export function normalizeGlobsToRoot(globs: Glob[], root: string, isExclude: boolean): string[] { + const withRoots = globs.map((g) => { + const source = typeof g === 'string' ? 'command line' : undefined; + return { source, ...fileOrGlobToGlob(g, root) }; + }); + + const normalized = normalizeGlobPatterns(withRoots, { root, nested: isExclude, nodePath: path }); + const filteredGlobs = normalized.filter((g) => g.root === root).map((g) => g.glob); + return filteredGlobs; } -function isNotUndefined(t: T | undefined): t is T { - return t !== undefined; +function* flatten(src: Iterable): IterableIterator { + for (const item of src) { + if (Array.isArray(item)) { + yield* item; + } else { + yield item; + } + } }