From ed8a5dc17ffa6de901887d3bd5b6bacf67217866 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Sat, 21 Aug 2021 18:10:12 +0200 Subject: [PATCH] feat: Improve `trace` words command results. (#1558) * dev: use findWord to search for words in Trie It is needed to return more comprehensive results. * dev: Add find method to `SpellingDictionary` * test: improve code coverage * dev: enrich trace results. * dev: display new trace results * dev: fix name of dictionaries from settings. * dev: sort the list of dictionaries before showing them. --- .../SpellingDictionary/Dictionaries.test.ts | 44 +++++++- .../src/SpellingDictionary/Dictionaries.ts | 37 ++++--- .../SpellingDictionary/SpellingDictionary.ts | 26 +++-- .../SpellingDictionaryCollection.test.ts | 70 +++++++++++- .../SpellingDictionaryCollection.ts | 56 +++++++--- .../SpellingDictionaryFromTrie.ts | 66 ++++++++--- .../SpellingDictionaryMethods.test.ts | 15 +++ .../SpellingDictionaryMethods.ts | 20 +--- .../createSpellingDictionary.test.ts | 2 + .../createSpellingDictionary.ts | 1 + packages/cspell-lib/src/trace.test.ts | 82 +++++++++++--- packages/cspell-lib/src/trace.ts | 66 ++++++++--- packages/cspell-lib/src/util/util.ts | 4 + packages/cspell-trie-lib/src/lib/trie.ts | 32 +++++- .../cspell/src/__snapshots__/app.test.ts.snap | 1 + packages/cspell/src/app.ts | 46 +------- packages/cspell/src/application.ts | 4 +- packages/cspell/src/traceEmitter.ts | 104 ++++++++++++++++++ 18 files changed, 525 insertions(+), 151 deletions(-) create mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.test.ts create mode 100644 packages/cspell/src/traceEmitter.ts diff --git a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts index 2852445c6f39..44ced06b66d4 100644 --- a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts +++ b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts @@ -30,7 +30,7 @@ describe('Validate getDictionary', () => { ${'rhone'} | ${ignoreCaseFalse} | ${false} ${'rhone'} | ${ignoreCaseTrue} | ${true} ${'snarf'} | ${ignoreCaseTrue} | ${false} - `('tests that userWords are included in the dictionary', async ({ word, opts, expected }) => { + `('tests that userWords are included in the dictionary $word', async ({ word, opts, expected }) => { const settings = { ...getDefaultSettings(), dictionaries: [], @@ -54,6 +54,37 @@ describe('Validate getDictionary', () => { expect(dict.has(word, opts)).toBe(expected); }); + test.each` + word | expected + ${'zero'} | ${{ found: 'zero', forbidden: false, noSuggest: false }} + ${'zeros'} | ${{ found: 'zeros', forbidden: false, noSuggest: true }} + ${'google'} | ${{ found: 'google', forbidden: false, noSuggest: true }} + ${'Café'} | ${{ found: 'café', forbidden: false, noSuggest: false }} + ${'CAFÉ'} | ${{ found: 'café', forbidden: false, noSuggest: false }} + ${'café'} | ${{ found: 'café', forbidden: false, noSuggest: false }} + ${'cafe'} | ${{ found: 'cafe', forbidden: false, noSuggest: false }} + ${'CAFE'} | ${{ found: 'cafe', forbidden: false, noSuggest: false }} + ${'Rhône'} | ${{ found: 'Rhône', forbidden: false, noSuggest: false }} + ${'RHÔNE'} | ${{ found: 'rhône', forbidden: false, noSuggest: false }} + ${'rhône'} | ${{ found: 'rhône', forbidden: false, noSuggest: false }} + ${'rhone'} | ${{ found: 'rhone', forbidden: false, noSuggest: false }} + ${'snarf'} | ${{ found: 'snarf', forbidden: true, noSuggest: false }} + ${'hte'} | ${{ found: 'hte', forbidden: true, noSuggest: false }} + ${'colour'} | ${{ found: 'colour', forbidden: true, noSuggest: false }} + `('find words $word', async ({ word, expected }) => { + const settings: CSpellUserSettings = { + ...getDefaultSettings(), + noSuggestDictionaries: ['companies'], + words: ['one', 'two', 'three', 'café', '!snarf'], + userWords: ['four', 'five', 'six', 'Rhône'], + ignoreWords: ['zeros'], + flagWords: ['hte', 'colour'], + }; + + const dict = await Dictionaries.getDictionary(settings); + expect(dict.find(word)).toEqual(expected); + }); + test.each` word | opts | expected ${'zero'} | ${undefined} | ${false} @@ -91,6 +122,17 @@ describe('Validate getDictionary', () => { expect(dict.has(word, opts)).toBe(expected); }); + test('Dictionary NOT Found', async () => { + const settings: CSpellUserSettings = { + dictionaryDefinitions: [{ name: 'my-words', path: './not-found.txt' }], + dictionaries: ['my-words'], + }; + + const dict = await Dictionaries.getDictionary(settings); + expect(dict.getErrors()).toEqual([expect.objectContaining(new Error('my-words: failed to load'))]); + expect(dict.dictionaries.map((d) => d.name)).toEqual(['my-words', '[words]', '[ignoreWords]', '[flagWords]']); + }); + test('Refresh Dictionary Cache', async () => { const tempDictPath = path.join(__dirname, '..', '..', 'temp', 'words.txt'); await fs.mkdirp(path.dirname(tempDictPath)); diff --git a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts index e247130f549f..426bbc6b447e 100644 --- a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts +++ b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts @@ -13,11 +13,7 @@ export function loadDictionaries( ): Promise[] { const defsToLoad = filterDictDefsToLoad(dictIds, defs); - return defsToLoad - .map((def) => loadDictionary(def.path, def)) - .map((p) => p.catch(() => undefined)) - .filter((p) => !!p) - .map((a) => a as Promise); + return defsToLoad.map((def) => loadDictionary(def.path, def)); } export function refreshDictionaryCache(maxAge?: number): Promise { @@ -42,14 +38,29 @@ export function getDictionary(settings: CSpellUserSettings): Promise { const wordsA = [ @@ -28,6 +28,15 @@ describe('Verify using multiple dictionaries', () => { const wordsC = ['ant', 'snail', 'beetle', 'worm', 'stink bug', 'centipede', 'millipede', 'flea', 'fly']; const wordsD = ['red*', 'green*', 'blue*', 'pink*', 'black*', '*berry', '+-fruit', '*bug', 'pinkie']; const wordsF = ['!pink*', '+berry', '+bug', '!stinkbug']; + + const wordsLegacy = ['error', 'code', 'system', 'ctrl']; + + // cspell:ignore pinkberry + const wordsNoSug = ['colour', 'behaviour', 'favour', 'pinkberry']; + + const dictNoSug = createSpellingDictionary(wordsNoSug, 'words-no-suggest', 'test', { noSuggest: true }); + const dictLegacy = createSpellingDictionary(wordsLegacy, 'legacy-dict', 'test', { useCompounds: true }); + test('checks for existence', async () => { const dicts = await Promise.all([ createSpellingDictionary(wordsA, 'wordsA', 'test', {}), @@ -143,6 +152,61 @@ describe('Verify using multiple dictionaries', () => { expect(dictCollection.has(word)).toEqual(expected); }); + test.each` + word | expected + ${'redberry'} | ${{ found: 'redberry', forbidden: false, noSuggest: false }} + ${'pinkberry'} | ${{ found: 'pinkberry', forbidden: false, noSuggest: true }} + ${'pink'} | ${{ found: 'pink', forbidden: true, noSuggest: false }} + ${'bug'} | ${{ found: 'bug', forbidden: false, noSuggest: false }} + ${'blackberry'} | ${{ found: 'blackberry', forbidden: false, noSuggest: false }} + ${'pinkbug'} | ${{ found: 'pinkbug', forbidden: false, noSuggest: false }} + ${'colour'} | ${{ found: 'colour', forbidden: false, noSuggest: true }} + ${'behaviour'} | ${{ found: 'behaviour', forbidden: false, noSuggest: true }} + `('find: "$word"', ({ word, expected }) => { + const dicts = [ + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), + createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), + dictNoSug, + ]; + + const dictCollection = createCollection(dicts, 'test'); + expect(dictCollection.find(word)).toEqual(expected); + }); + + // cspell:ignore error* *code ctrl* *code *berry* + test.each` + word | expected + ${'redberry'} | ${{ found: 'redberry', forbidden: false, noSuggest: false }} + ${'pinkberry'} | ${{ found: 'pinkberry', forbidden: false, noSuggest: true }} + ${'berryberry'} | ${{ found: 'berry+berry', forbidden: false, noSuggest: false }} + ${'errorcode'} | ${{ found: 'error+code', forbidden: false, noSuggest: false }} + ${'ctrlcode'} | ${{ found: 'ctrl+code', forbidden: false, noSuggest: false }} + ${'pink'} | ${{ found: 'pink', forbidden: true, noSuggest: false }} + ${'bug'} | ${{ found: 'bug', forbidden: false, noSuggest: false }} + ${'blackberry'} | ${{ found: 'blackberry', forbidden: false, noSuggest: false }} + ${'pinkbug'} | ${{ found: 'pinkbug', forbidden: false, noSuggest: false }} + ${'colour'} | ${{ found: 'colour', forbidden: false, noSuggest: true }} + ${'behaviour'} | ${{ found: 'behaviour', forbidden: false, noSuggest: true }} + `('find compound: "$word"', ({ word, expected }) => { + const dicts = [ + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), + createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), + dictNoSug, + dictLegacy, + ]; + + const dictCollection = createCollection(dicts, 'test'); + expect(dictCollection.find(word, { useCompounds: true })).toEqual(expected); + }); + // cspell:ignore pinkbug redberry // Note: `pinkbug` is not forbidden because compound forbidden words is not yet supported. test.each` diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts index 7c42567bbc8d..600805586d90 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts @@ -1,25 +1,27 @@ +import { CASE_INSENSITIVE_PREFIX } from 'cspell-trie-lib'; +import { genSequence } from 'gensequence'; +import { getDefaultSettings } from '../Settings'; +import { memorizer, memorizerKeyBy } from '../util/Memorizer'; +import { isDefined } from '../util/util'; import { CompoundWordsMethod, + FindResult, + HasOptions, + SearchOptions, + SpellingDictionary, + SpellingDictionaryOptions, SuggestionCollector, - suggestionCollector, SuggestionResult, - hasOptionToSearchOption, + SuggestOptions, +} from './SpellingDictionary'; +import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; +import { defaultNumSuggestions, + hasOptionToSearchOption, SuggestArgs, suggestArgsToSuggestOptions, + suggestionCollector, } from './SpellingDictionaryMethods'; -import { - SpellingDictionary, - HasOptions, - SearchOptions, - SuggestOptions, - SpellingDictionaryOptions, -} from './SpellingDictionary'; -import { CASE_INSENSITIVE_PREFIX } from 'cspell-trie-lib'; -import { genSequence } from 'gensequence'; -import { getDefaultSettings } from '../Settings'; -import { memorizer, memorizerKeyBy } from '../util/Memorizer'; -import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; function identityString(w: string): string { return w; @@ -45,6 +47,16 @@ export class SpellingDictionaryCollection implements SpellingDictionary { return !!isWordInAnyDictionary(this.dictionaries, word, options) && !this.isForbidden(word); } + public find(word: string, hasOptions?: HasOptions): FindResult | undefined { + const options = hasOptionToSearchOption(hasOptions); + const { + found = false, + forbidden = false, + noSuggest = false, + } = findInAnyDictionary(this.dictionaries, word, options) || {}; + return { found, forbidden, noSuggest }; + } + public isNoSuggestWord(word: string, options?: HasOptions): boolean { return this._isNoSuggestWord(word, options); } @@ -118,7 +130,7 @@ export class SpellingDictionaryCollection implements SpellingDictionary { private _isNoSuggestWord = memorizerKeyBy( (word: string, options?: HasOptions) => { if (!this.containsNoSuggestWords) return false; - return !!isNoSuggestWordInAnyDictionary(this.dictionaries, word, options || false); + return !!isNoSuggestWordInAnyDictionary(this.dictionaries, word, options || {}); }, (word: string, options?: HasOptions) => { const opts = hasOptionToSearchOption(options); @@ -140,6 +152,20 @@ function isWordInAnyDictionary( return genSequence(dicts).first((dict) => dict.has(word, options)); } +function findInAnyDictionary( + dicts: SpellingDictionary[], + word: string, + options: SearchOptions +): FindResult | undefined { + const found = dicts.map((dict) => dict.find(word, options)).filter(isDefined); + if (!found.length) return undefined; + return found.reduce((a, b) => ({ + found: a.forbidden ? a.found : b.forbidden ? b.found : a.found || b.found, + forbidden: a.forbidden || b.forbidden, + noSuggest: a.noSuggest || b.noSuggest, + })); +} + function isNoSuggestWordInAnyDictionary( dicts: SpellingDictionary[], word: string, diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts index 1870756065e8..30d9aadf140d 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts @@ -5,6 +5,7 @@ import { SuggestionResult, CompoundWordsMethod, importTrie, + FindWordOptions, } from 'cspell-trie-lib'; import { createMapper } from '../util/repMap'; import { getDefaultSettings } from '../Settings'; @@ -18,7 +19,14 @@ import { suggestArgsToSuggestOptions, wordSuggestFormsArray, } from './SpellingDictionaryMethods'; -import { SpellingDictionary, HasOptions, SuggestOptions, SpellingDictionaryOptions } from './SpellingDictionary'; +import { + SpellingDictionary, + HasOptions, + SuggestOptions, + SpellingDictionaryOptions, + FindResult, +} from './SpellingDictionary'; +import { FindFullResult } from '../../../cspell-trie-lib/dist/lib/find'; export class SpellingDictionaryFromTrie implements SpellingDictionary { static readonly cachedWordsLimit = 50000; private _size = 0; @@ -59,37 +67,63 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { return this._size; } public has(word: string, hasOptions?: HasOptions): boolean { - const searchOptions = hasOptionToSearchOption(hasOptions); - const useCompounds = searchOptions.useCompounds ?? this.options.useCompounds; - const { ignoreCase = true } = searchOptions; - return this._has(word, useCompounds, ignoreCase); + const { useCompounds, ignoreCase } = this.resolveOptions(hasOptions); + const r = this._find(word, useCompounds, ignoreCase); + return !!r && !r.forbidden && !!r.found; + } + + public find(word: string, hasOptions?: HasOptions): FindResult | undefined { + const { useCompounds, ignoreCase } = this.resolveOptions(hasOptions); + const r = this._find(word, useCompounds, ignoreCase); + const { forbidden = this.isForbidden(word) } = r || {}; + if (!r && !forbidden) return undefined; + const { found = forbidden ? word : false } = r || {}; + const noSuggest = found !== false && this.containsNoSuggestWords; + return { found, forbidden, noSuggest }; + } + + private resolveOptions(hasOptions?: HasOptions): { + useCompounds: HasOptions['useCompounds'] | undefined; + ignoreCase: boolean; + } { + const { useCompounds = this.options.useCompounds, ignoreCase = true } = hasOptionToSearchOption(hasOptions); + return { useCompounds, ignoreCase }; } - private _has = memorizer( + private _find = memorizer( (word: string, useCompounds: number | boolean | undefined, ignoreCase: boolean) => - this.hasAnyForm(word, useCompounds, ignoreCase), + this.findAnyForm(word, useCompounds, ignoreCase), SpellingDictionaryFromTrie.cachedWordsLimit ); - private hasAnyForm(word: string, useCompounds: number | boolean | undefined, ignoreCase: boolean) { + private findAnyForm( + word: string, + useCompounds: number | boolean | undefined, + ignoreCase: boolean + ): FindAnyFormResult | undefined { const mWord = this.mapWord(word.normalize('NFC')); - if (this.trie.hasWord(mWord, true)) { - return true; + const opts: FindWordOptions = { caseSensitive: !ignoreCase }; + const findResult = this.trie.findWord(mWord, opts); + if (findResult.found !== false) { + return findResult; } const forms = wordSearchForms(mWord, this.isDictionaryCaseSensitive, ignoreCase); for (const w of forms) { - if (this.trie.hasWord(w, !ignoreCase)) { - return true; + const findResult = this.trie.findWord(w, opts); + if (findResult.found !== false) { + return findResult; } } if (useCompounds) { + opts.useLegacyWordCompounds = useCompounds; for (const w of forms) { - if (this.trie.has(w, useCompounds)) { - return true; + const findResult = this.trie.findWord(w, opts); + if (findResult.found !== false) { + return findResult; } } } - return false; + return undefined; } public isNoSuggestWord(word: string, options?: HasOptions): boolean { @@ -148,6 +182,8 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { } } +type FindAnyFormResult = FindFullResult; + export async function createSpellingDictionaryTrie( data: Iterable, name: string, diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.test.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.test.ts new file mode 100644 index 000000000000..5434fe5db0b2 --- /dev/null +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.test.ts @@ -0,0 +1,15 @@ +import { impersonateCollector, suggestionCollector } from './SpellingDictionaryMethods'; + +describe('SpellingDictionaryMethods', () => { + test('impersonateCollector', () => { + const collector = suggestionCollector('hello', { numSuggestions: 1, changeLimit: 3, ignoreCase: true }); + const ic = impersonateCollector(collector, 'Hello'); + const suggestion = { word: 'hello', cost: 1 }; + ic.add(suggestion); + expect(ic.suggestions).toEqual([suggestion]); + expect(ic.maxCost).toBeGreaterThan(200); + expect(ic.maxNumSuggestions).toBe(1); + expect(ic.word).toBe('Hello'); + expect(collector.word).toBe('hello'); + }); +}); diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.ts index 80f5c1f60002..dc9e9c15f947 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryMethods.ts @@ -1,20 +1,10 @@ +import { CompoundWordsMethod, SuggestionCollector, SuggestionResult } from 'cspell-trie-lib'; import { genSequence } from 'gensequence'; -import { SuggestionCollector, SuggestionResult, CompoundWordsMethod } from 'cspell-trie-lib'; -import { ucFirst, removeAccents, isUpperCase } from '../util/text'; +import { isUpperCase, removeAccents, ucFirst } from '../util/text'; import { FunctionArgs } from '../util/types'; -import { SpellingDictionary, HasOptions, SearchOptions, SuggestOptions } from './SpellingDictionary'; +import { HasOptions, SearchOptions, SpellingDictionary, SuggestOptions } from './SpellingDictionary'; -// cspell:word café - -export { - CompoundWordsMethod, - JOIN_SEPARATOR, - SuggestionCollector, - suggestionCollector, - SuggestionResult, - WORD_SEPARATOR, - CASE_INSENSITIVE_PREFIX, -} from 'cspell-trie-lib'; +export { suggestionCollector } from 'cspell-trie-lib'; export type FilterSuggestionsPredicate = (word: SuggestionResult) => boolean; @@ -131,7 +121,7 @@ export function wordDictionaryFormsCollector(prefixNoCase: string): (word: strin } export function hasOptionToSearchOption(opt: HasOptions | undefined): SearchOptions { - return !opt ? {} : typeof opt === 'object' ? opt : { useCompounds: opt }; + return !opt ? {} : opt; } export function suggestArgsToSuggestOptions(args: SuggestArgs): SuggestOptions { diff --git a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts index c24dc3fe5bc1..b2fe1588805e 100644 --- a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts +++ b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts @@ -17,6 +17,8 @@ describe('Validate createSpellingDictionary', () => { expect(d.suggest('error')).toEqual([]); expect(d.mapWord('café')).toBe('café'); expect(d.has('fun')).toBe(false); + expect(d.find('hello')).toBeUndefined(); + expect(d.isNoSuggestWord('hello', {})).toBe(false); }); test('createSpellingDictionary', () => { diff --git a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts index 982e01e4fbce..cc2378d1e6b4 100644 --- a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts +++ b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts @@ -41,6 +41,7 @@ export function createFailedToLoadDictionary(error: SpellingDictionaryLoadError) type: 'error', containsNoSuggestWords: false, has: () => false, + find: () => undefined, isNoSuggestWord: () => false, isForbidden: () => false, suggest: () => [], diff --git a/packages/cspell-lib/src/trace.test.ts b/packages/cspell-lib/src/trace.test.ts index f30946ea2e04..291943046705 100644 --- a/packages/cspell-lib/src/trace.test.ts +++ b/packages/cspell-lib/src/trace.test.ts @@ -1,27 +1,74 @@ import { traceWords, getDefaultSettings } from '.'; import { CSpellSettings } from '@cspell/cspell-types'; +import { mergeSettings } from './Settings'; +import { TraceResult } from './trace'; describe('Verify trace', () => { jest.setTimeout(10000); + test('tests tracing a word', async () => { const words = ['apple']; - const config = getDefaultSettings(); - const results = await traceWords(words, config); - expect(Object.keys(results)).not.toHaveLength(0); - const foundIn = results.filter((r) => r.found); - expect(foundIn).toEqual( + const config = getSettings({ ignoreWords: ['apple'], flagWords: ['apple'] }); + const results = await traceWords(words, config, {}); + expect(results.map(({ dictName, found }) => ({ dictName, found }))).toEqual( expect.arrayContaining([ - expect.objectContaining({ - dictName: 'en_us', - dictSource: expect.stringContaining('en_US.trie.gz'), - }), + { dictName: 'en-gb', found: true }, + { dictName: 'en_us', found: true }, + { dictName: 'cpp', found: true }, + { dictName: 'typescript', found: false }, + { dictName: 'companies', found: true }, + { dictName: 'softwareTerms', found: false }, + { dictName: '[ignoreWords]', found: true }, + { dictName: '[words]', found: false }, + { dictName: '[flagWords]', found: true }, ]) ); }); + // cspell:ignore *error* *code* + test.each` + word | languageId | locale | ignoreCase | allowCompoundWords | dictName | dictActive | found | forbidden | noSuggest | foundWord + ${'apple'} | ${undefined} | ${undefined} | ${true} | ${undefined} | ${'en_us'} | ${true} | ${true} | ${false} | ${false} | ${'apple'} + ${'apple'} | ${undefined} | ${undefined} | ${true} | ${undefined} | ${'en-gb'} | ${false} | ${true} | ${false} | ${false} | ${'apple'} + ${'Apple'} | ${undefined} | ${undefined} | ${false} | ${undefined} | ${'en_us'} | ${true} | ${true} | ${false} | ${false} | ${'apple'} + ${'Apple'} | ${undefined} | ${undefined} | ${false} | ${undefined} | ${'companies'} | ${true} | ${true} | ${false} | ${false} | ${'apple'} + ${'Apple'} | ${undefined} | ${undefined} | ${false} | ${undefined} | ${'cpp'} | ${false} | ${true} | ${false} | ${false} | ${'apple'} + ${'café'} | ${undefined} | ${undefined} | ${true} | ${undefined} | ${'en_us'} | ${true} | ${true} | ${false} | ${false} | ${'café'} + ${'errorcode'} | ${undefined} | ${undefined} | ${true} | ${undefined} | ${'en_us'} | ${true} | ${false} | ${false} | ${false} | ${undefined} + ${'errorcode'} | ${undefined} | ${undefined} | ${true} | ${true} | ${'en_us'} | ${true} | ${true} | ${false} | ${false} | ${'error+code'} + ${'errorcode'} | ${'cpp'} | ${undefined} | ${true} | ${undefined} | ${'cpp'} | ${true} | ${true} | ${false} | ${false} | ${'error+code'} + ${'hte'} | ${undefined} | ${undefined} | ${true} | ${undefined} | ${'en_us'} | ${true} | ${false} | ${false} | ${false} | ${undefined} + ${'hte'} | ${undefined} | ${undefined} | ${true} | ${undefined} | ${'[flagWords]'} | ${true} | ${true} | ${true} | ${false} | ${'hte'} + ${'Colour'} | ${undefined} | ${undefined} | ${true} | ${undefined} | ${'[ignoreWords]'} | ${true} | ${true} | ${false} | ${true} | ${'colour'} + `('trace word "$word" in $dictName', async (params) => { + const { word, languageId, ignoreCase, locale, allowCompoundWords } = params; + const { dictName, dictActive, found, forbidden, noSuggest, foundWord } = params; + const words = [word]; + const config = getSettings({ allowCompoundWords, flagWords: ['hte'], ignoreWords: ['colour'] }); + const results = await traceWords(words, config, { locale, languageId, ignoreCase }); + const byName = results.reduce((a, b) => { + a[b.dictName] = b; + return a; + }, {} as Record); + + // console.log(JSON.stringify(byName)); + + expect(byName[dictName]).toEqual( + oc({ + dictActive, + dictName, + forbidden, + found, + foundWord, + noSuggest, + word, + }) + ); + }); + test('tracing with missing dictionary.', async () => { const words = ['apple']; - const defaultConfig = getDefaultSettings(); + const defaultConfig = getSettings(); const dictionaryDefinitions = (defaultConfig.dictionaryDefinitions || []).concat([ { name: 'bad dict', @@ -32,15 +79,12 @@ describe('Verify trace', () => { ...defaultConfig, dictionaryDefinitions, }; - const results = await traceWords(words, config); + const results = await traceWords(words, config, {}); expect(Object.keys(results)).not.toHaveLength(0); const foundIn = results.filter((r) => r.found); expect(foundIn).toEqual( expect.arrayContaining([ - expect.objectContaining({ - dictName: 'en_us', - dictSource: expect.stringContaining('en_US.trie.gz'), - }), + expect.objectContaining({ dictName: 'en_us', dictSource: expect.stringContaining('en_US.trie.gz') }), ]) ); @@ -60,3 +104,11 @@ describe('Verify trace', () => { ); }); }); + +function oc(t: T): T { + return expect.objectContaining(t); +} + +function getSettings(...settings: CSpellSettings[]): CSpellSettings { + return settings.reduce((a, b) => mergeSettings(a, b), getDefaultSettings()); +} diff --git a/packages/cspell-lib/src/trace.ts b/packages/cspell-lib/src/trace.ts index 0f9e7aaa3a89..1e5e808e6658 100644 --- a/packages/cspell-lib/src/trace.ts +++ b/packages/cspell-lib/src/trace.ts @@ -1,36 +1,65 @@ -import { finalizeSettings } from './Settings'; -import { CSpellSettings } from '@cspell/cspell-types'; -import { getDictionary, SpellingDictionaryCollection } from './SpellingDictionary'; -import * as util from './util/util'; +import { CSpellSettings, DictionaryId, LocaleId } from '@cspell/cspell-types'; import { genSequence } from 'gensequence'; +import { LanguageId } from './LanguageIds'; +import { finalizeSettings, mergeSettings } from './Settings'; +import { calcSettingsForLanguageId } from './Settings/LanguageSettings'; +import { getDictionary, HasOptions, SpellingDictionaryCollection } from './SpellingDictionary'; +import * as util from './util/util'; export interface TraceResult { word: string; found: boolean; + foundWord: string | undefined; forbidden: boolean; + noSuggest: boolean; dictName: string; dictSource: string; + dictActive: boolean; configSource: string; errors: Error[] | undefined; } -export async function traceWords(words: string[], settings: CSpellSettings): Promise { +export interface TraceOptions { + languageId?: LanguageId | LanguageId[]; + locale?: LocaleId; + ignoreCase?: boolean; +} + +export async function traceWords( + words: string[], + settings: CSpellSettings, + options: TraceOptions | undefined +): Promise { + const { languageId, locale: language, ignoreCase = true } = options || {}; + async function finalize(config: CSpellSettings): Promise<{ + activeDictionaries: DictionaryId[]; config: CSpellSettings; dicts: SpellingDictionaryCollection; }> { - const settings = finalizeSettings(config); + const withLocale = mergeSettings(config, { language }); + const withLanguageId = calcSettingsForLanguageId( + withLocale, + languageId ?? withLocale.languageId ?? 'plaintext' + ); + const settings = finalizeSettings(withLanguageId); const dictionaries = (settings.dictionaries || []) .concat((settings.dictionaryDefinitions || []).map((d) => d.name)) .filter(util.uniqueFn); const dictSettings: CSpellSettings = { ...settings, dictionaries }; + const dictBase = await getDictionary(settings); const dicts = await getDictionary(dictSettings); + const activeDictionaries = dictBase.dictionaries.map((d) => d.name); return { + activeDictionaries, config: settings, dicts, }; } - const { config, dicts } = await finalize(settings); + const { config, dicts, activeDictionaries } = await finalize(settings); + + const setOfActiveDicts = new Set(activeDictionaries); + const opts: HasOptions = { ignoreCase, useCompounds: config.allowCompoundWords }; const r = await Promise.all( genSequence(words) @@ -48,15 +77,20 @@ export async function traceWords(words: string[], settings: CSpellSettings): Pro const s = genSequence(r) .concatMap((p) => { const { word, config, dicts } = p; - return dicts.dictionaries.map((dict) => ({ - word, - found: dict.has(word), - forbidden: dict.isForbidden(word), - dictName: dict.name, - dictSource: dict.source, - configSource: config.name || '', - errors: normalizeErrors(dict.getErrors?.()), - })); + return dicts.dictionaries + .map((dict) => ({ dict, findResult: dict.find(word, opts) })) + .map(({ dict, findResult }) => ({ + word, + found: !!findResult?.found, + foundWord: findResult?.found || undefined, + forbidden: findResult?.forbidden || false, + noSuggest: findResult?.noSuggest || false, + dictName: dict.name, + dictSource: dict.source, + dictActive: setOfActiveDicts.has(dict.name), + configSource: config.name || '', + errors: normalizeErrors(dict.getErrors?.()), + })); }) .toArray(); diff --git a/packages/cspell-lib/src/util/util.ts b/packages/cspell-lib/src/util/util.ts index ccc404b4b0e5..46c7f5cb48b9 100644 --- a/packages/cspell-lib/src/util/util.ts +++ b/packages/cspell-lib/src/util/util.ts @@ -52,3 +52,7 @@ export function scanMap(accFn: (acc: T, value: T) => T, init?: T): (value: T) return acc; }; } + +export function isDefined(v: T | undefined): v is T { + return v !== undefined; +} diff --git a/packages/cspell-trie-lib/src/lib/trie.ts b/packages/cspell-trie-lib/src/lib/trie.ts index 32f5d3561fc6..7578a95dea2f 100644 --- a/packages/cspell-trie-lib/src/lib/trie.ts +++ b/packages/cspell-trie-lib/src/lib/trie.ts @@ -21,6 +21,7 @@ import { PartialFindOptions, findWord, findWordNode, + FindFullResult, } from './find'; export { COMPOUND_FIX, OPTIONAL_COMPOUND_FIX, CASE_INSENSITIVE_PREFIX, FORBID_PREFIX } from './constants'; @@ -100,11 +101,10 @@ export class Trie { } has(word: string, minLegacyCompoundLength?: boolean | number): boolean { - if (this.hasWord(word, true)) return true; + if (this.hasWord(word, false)) return true; if (minLegacyCompoundLength) { - const len = minLegacyCompoundLength !== true ? minLegacyCompoundLength : defaultLegacyMinCompoundLength; - const findOptions = createFindOptions({ legacyMinCompoundLength: len }); - return !!findLegacyCompound(this.root, word, findOptions).found; + const f = this.findWord(word, { useLegacyWordCompounds: minLegacyCompoundLength }); + return !!f.found; } return false; } @@ -116,11 +116,26 @@ export class Trie { * @returns true if the word was found and is not forbidden. */ hasWord(word: string, caseSensitive: boolean): boolean { - const findOptions = this.createFindOptions({ matchCase: caseSensitive }); - const f = findWord(this.root, word, findOptions); + const f = this.findWord(word, { caseSensitive }); return !!f.found && !f.forbidden; } + findWord(word: string, options?: FindWordOptions): FindFullResult { + if (options?.useLegacyWordCompounds) { + const len = + options.useLegacyWordCompounds !== true + ? options.useLegacyWordCompounds + : defaultLegacyMinCompoundLength; + const findOptions = this.createFindOptions({ + legacyMinCompoundLength: len, + matchCase: options.caseSensitive, + }); + return findLegacyCompound(this.root, word, findOptions); + } + const findOptions = this.createFindOptions({ matchCase: options?.caseSensitive }); + return findWord(this.root, word, findOptions); + } + /** * Determine if a word is in the forbidden word list. * @param word the word to lookup. @@ -273,3 +288,8 @@ export class Trie { return findOptions; } } + +export interface FindWordOptions { + caseSensitive?: boolean; + useLegacyWordCompounds?: boolean | number; +} diff --git a/packages/cspell/src/__snapshots__/app.test.ts.snap b/packages/cspell/src/__snapshots__/app.test.ts.snap index 0f265b098283..6919e069255b 100644 --- a/packages/cspell/src/__snapshots__/app.test.ts.snap +++ b/packages/cspell/src/__snapshots__/app.test.ts.snap @@ -524,6 +524,7 @@ Array [ "Usage: cspell trace [options] ", "", "Trace words", + " Search for words in the configuration and dictionaries.", "", "Options:", " -c, --config Configuration file to use. By default cspell", diff --git a/packages/cspell/src/app.ts b/packages/cspell/src/app.ts index 430cefded74e..2aae2c6c04c0 100644 --- a/packages/cspell/src/app.ts +++ b/packages/cspell/src/app.ts @@ -16,6 +16,7 @@ import { import { tableToLines } from './util/table'; import { Emitters, isProgressFileComplete, MessageType, ProgressItem, Issue } from './emitters'; import { isSpellingDictionaryLoadError, SpellingDictionaryLoadError, ImportError } from 'cspell-lib'; +import { emitTraceResults } from './traceEmitter'; interface Options extends CSpellApplicationOptions { legacy?: boolean; @@ -211,7 +212,10 @@ export async function run(program?: commander.Command, argv?: string[]): Promise type TraceCommandOptions = TraceOptions; prog.command('trace') - .description('Trace words') + .description( + `Trace words + Search for words in the configuration and dictionaries.` + ) .option( '-c, --config ', 'Configuration file to use. By default cspell looks for cspell.json in the current directory.' @@ -226,7 +230,7 @@ export async function run(program?: commander.Command, argv?: string[]): Promise .arguments('') .action(async (words: string[], options: TraceCommandOptions) => { const results = await App.trace(words, options); - results.forEach(emitTraceResult); + emitTraceResults(results, { cwd: process.cwd() }); const numFound = results.reduce((n, r) => n + (r.found ? 1 : 0), 0); if (!numFound) { console.error('No matches found'); @@ -362,44 +366,6 @@ function collect(value: string, previous: string[] | undefined): string[] { return previous.concat([value]); } -function emitTraceResult(r: App.TraceResult) { - const terminalWidth = process.stdout.columns || 120; - const widthName = 20; - const errors = r.errors?.map((e) => e.message)?.join('\n\t') || ''; - const w = r.forbidden ? chalk.red(r.word) : chalk.green(r.word); - const f = r.forbidden - ? chalk.red('!') - : r.found - ? chalk.whiteBright('*') - : errors - ? chalk.red('X') - : chalk.dim('-'); - const n = chalk.yellowBright(pad(r.dictName, widthName)); - const used = [r.word.length, 1, widthName].reduce((a, b) => a + b, 3); - const widthSrc = terminalWidth - used; - const c = errors ? chalk.red : chalk.white; - const s = c(trimMid(r.dictSource, widthSrc)); - const line = [w, f, n, s].join(' '); - console.log(line); - if (errors) { - console.error('\t' + chalk.red(errors)); - } -} - -function pad(s: string, w: number): string { - return (s + ' '.repeat(w)).substr(0, w); -} - -function trimMid(s: string, w: number): string { - s = s.trim(); - if (s.length <= w) { - return s; - } - const l = Math.floor((w - 3) / 2); - const r = Math.ceil((w - 3) / 2); - return s.substr(0, l) + '...' + s.substr(-r); -} - function formatIssue(templateStr: string, issue: Issue, maxIssueTextWidth: number) { function clean(t: string) { return t.replace(/\s+/, ' '); diff --git a/packages/cspell/src/application.ts b/packages/cspell/src/application.ts index 376a9e5da66c..e639b45a6338 100644 --- a/packages/cspell/src/application.ts +++ b/packages/cspell/src/application.ts @@ -17,9 +17,11 @@ export function lint(files: string[], options: CSpellApplicationOptions, emitter } export async function trace(words: string[], options: TraceOptions): Promise { + const { local } = options; + const { languageId, locale = local } = options; const configFile = await readConfig(options.config, undefined); const config = cspell.mergeSettings(cspell.getDefaultSettings(), cspell.getGlobalSettings(), configFile.config); - const results = await traceWords(words, config); + const results = await traceWords(words, config, { languageId, locale }); return results; } diff --git a/packages/cspell/src/traceEmitter.ts b/packages/cspell/src/traceEmitter.ts new file mode 100644 index 000000000000..51113ddf9009 --- /dev/null +++ b/packages/cspell/src/traceEmitter.ts @@ -0,0 +1,104 @@ +import { TraceResult } from './application'; +import chalk = require('chalk'); +import * as Path from 'path'; + +export interface EmitTraceOptions { + /** current working directory */ + cwd: string; +} + +const colWidthDictionaryName = 20; + +export function emitTraceResults(results: TraceResult[], options: EmitTraceOptions): void { + const maxWordLength = results + .map((r) => r.foundWord || r.word) + .reduce((a, b) => Math.max(a, b.length), 'Word'.length); + + const cols: ColWidths = { + word: maxWordLength, + dictName: colWidthDictionaryName, + terminalWidth: process.stdout.columns || 120, + }; + + const col = new Intl.Collator(); + results.sort((a, b) => col.compare(a.dictName, b.dictName)); + + emitHeader(cols); + results.forEach((r) => emitTraceResult(r, cols, options)); +} + +interface ColWidths { + word: number; + dictName: number; + terminalWidth: number; +} + +function emitHeader(colWidths: ColWidths): void { + const line = [ + pad('Word', colWidths.word), + 'F', + pad('Dictionary', colWidths.dictName), + pad('Dictionary Location', 30), + ]; + console.log(chalk.underline(line.join(' '))); +} + +function emitTraceResult(r: TraceResult, colWidths: ColWidths, options: EmitTraceOptions): void { + const { word: wordColWidth, terminalWidth, dictName: widthName } = colWidths; + const errors = r.errors?.map((e) => e.message)?.join('\n\t') || ''; + const word = pad(r.foundWord || r.word, wordColWidth); + const cWord = word.replace(/[+]/g, chalk.yellow('+')); + const w = r.forbidden ? chalk.red(cWord) : chalk.green(cWord); + const f = calcFoundChar(r); + const a = r.dictActive ? '*' : ' '; + const dictName = pad(r.dictName.slice(0, widthName - 1) + a, widthName); + const dictColor = r.dictActive ? chalk.yellowBright : chalk.rgb(200, 128, 50); + const n = dictColor(dictName); + const used = [r.word.length, 1, widthName].reduce((a, b) => a + b, 3); + const widthSrc = terminalWidth - used; + const c = errors ? chalk.red : chalk.white; + const s = c(formatDictionaryLocation(r.dictSource, widthSrc, options.cwd)); + const line = [w, f, n, s].join(' '); + console.log(line); + if (errors) { + console.error('\t' + chalk.red(errors)); + } +} + +function pad(s: string, w: number): string { + return (s + ' '.repeat(w)).substr(0, w); +} + +function trimMid(s: string, w: number): string { + s = s.trim(); + if (s.length <= w) { + return s; + } + const l = Math.floor((w - 3) / 2); + const r = Math.ceil((w - 3) / 2); + return s.substr(0, l) + '...' + s.substr(-r); +} + +function calcFoundChar(r: TraceResult): string { + const errors = r.errors?.map((e) => e.message)?.join('\n\t') || ''; + + let color = chalk.dim; + color = r.found ? chalk.whiteBright : color; + color = r.forbidden ? chalk.red : color; + color = r.noSuggest ? chalk.blueBright : color; + color = errors ? chalk.red : color; + + let char = '-'; + char = r.found ? '*' : char; + char = r.forbidden ? '!' : char; + char = r.noSuggest ? 'N' : char; + char = errors ? 'X' : char; + + return color(char); +} + +function formatDictionaryLocation(dictSource: string, maxWidth: number, cwd: string): string { + const relPath = cwd ? Path.relative(cwd, dictSource) : dictSource; + const usePath = relPath.length < dictSource.length ? relPath : dictSource; + return trimMid(usePath, maxWidth); +}