feat: add option to ignore diacritics

Add `ignoreDiacritics` to ignore diacritics in search

Author: piitaya

docs/api/options.md

@@ -14,6 +14,13 @@ tags:
 
 Indicates whether comparisons should be case sensitive.
 
+### `ignoreDiacritics`
+
+- Type: `boolean`
+- Default: `false`
+
+Indicates whether comparisons should ignore diacritics (accents).
+
 ### `includeScore`
 
 - Type: `boolean`

src/core/config.js

@@ -16,6 +16,8 @@ export const BasicOptions = {
   // When `true`, the algorithm continues searching to the end of the input even if a perfect
   // match is found before the end of the same input.
   isCaseSensitive: false,
+  // When `true`, the algorithm will ignore diacritics (accents) in comparisons
+  ignoreDiacritics: false,
   // When true, the matching function will continue to the end of a search pattern even if
   includeScore: false,
   // List of properties that will be searched. This also supports nested properties.

src/core/index.js

@@ -12,6 +12,7 @@ export default class Fuse {
   constructor(docs, options = {}, index) {
     this.options = { ...Config, ...options }
 
+    console.log(this.options);
     if (
       this.options.useExtendedSearch &&
       !process.env.EXTENDED_SEARCH_ENABLED

src/helpers/diacritics.js

@@ -0,0 +1,3 @@
+export const stripDiacritics = String.prototype.normalize
+    ? ((str) => str.normalize('NFD').replace(/[\u0300-\u036F]/g, ''))
+    : ((str) => str);

src/index.d.ts

@@ -292,6 +292,8 @@ export type FuseOptionKey<T> = FuseOptionKeyObject<T> | string | string[]
 export interface IFuseOptions<T> {
   /** Indicates whether comparisons should be case sensitive. */
   isCaseSensitive?: boolean
+  /** Indicates whether comparisons should ignore diacritics (accents). */
+  ignoreDiacritics?: boolean
   /** Determines how close the match must be to the fuzzy location (specified by `location`). An exact letter match which is `distance` characters away from the fuzzy location would score as a complete mismatch. A `distance` of `0` requires the match be at the exact `location` specified. A distance of `1000` would require a perfect match to be within `800` characters of the `location` to be found using a `threshold` of `0.8`. */
   distance?: number
   /** When true, the matching function will continue to the end of a search pattern even if a perfect match has already been located in the string. */

src/search/bitap/index.js

@@ -2,6 +2,7 @@ import search from './search'
 import createPatternAlphabet from './createPatternAlphabet'
 import { MAX_BITS } from './constants'
 import Config from '../../core/config'
+import { stripDiacritics } from '../../helpers/diacritics'
 
 export default class BitapSearch {
   constructor(
@@ -14,6 +15,7 @@ export default class BitapSearch {
       findAllMatches = Config.findAllMatches,
       minMatchCharLength = Config.minMatchCharLength,
       isCaseSensitive = Config.isCaseSensitive,
+      ignoreDiacritics = Config.ignoreDiacritics,
       ignoreLocation = Config.ignoreLocation
     } = {}
   ) {
@@ -25,10 +27,13 @@ export default class BitapSearch {
       findAllMatches,
       minMatchCharLength,
       isCaseSensitive,
+      ignoreDiacritics,
       ignoreLocation
     }
 
-    this.pattern = isCaseSensitive ? pattern : pattern.toLowerCase()
+    pattern = isCaseSensitive ? pattern : pattern.toLowerCase()
+    pattern = ignoreDiacritics ? stripDiacritics(pattern) : pattern;
+    this.pattern = pattern;
 
     this.chunks = []
 
@@ -66,11 +71,10 @@ export default class BitapSearch {
   }
 
   searchIn(text) {
-    const { isCaseSensitive, includeMatches } = this.options
+    const { isCaseSensitive, ignoreDiacritics, includeMatches } = this.options
 
-    if (!isCaseSensitive) {
-      text = text.toLowerCase()
-    }
+    text = isCaseSensitive ? text : text.toLowerCase()
+    text = ignoreDiacritics ? stripDiacritics(text) : text
 
     // Exact match
     if (this.pattern === text) {

src/search/extended/FuzzyMatch.js

@@ -13,6 +13,7 @@ export default class FuzzyMatch extends BaseMatch {
       findAllMatches = Config.findAllMatches,
       minMatchCharLength = Config.minMatchCharLength,
       isCaseSensitive = Config.isCaseSensitive,
+      ignoreDiacritics = Config.ignoreDiacritics,
       ignoreLocation = Config.ignoreLocation
     } = {}
   ) {
@@ -25,6 +26,7 @@ export default class FuzzyMatch extends BaseMatch {
       findAllMatches,
       minMatchCharLength,
       isCaseSensitive,
+      ignoreDiacritics,
       ignoreLocation
     })
   }

src/search/extended/index.js

@@ -2,6 +2,7 @@ import parseQuery from './parseQuery'
 import FuzzyMatch from './FuzzyMatch'
 import IncludeMatch from './IncludeMatch'
 import Config from '../../core/config'
+import { stripDiacritics } from '../../helpers/diacritics'
 
 // These extended matchers can return an array of matches, as opposed
 // to a singl match
@@ -40,6 +41,7 @@ export default class ExtendedSearch {
     pattern,
     {
       isCaseSensitive = Config.isCaseSensitive,
+      ignoreDiacritics = Config.ignoreDiacritics,
       includeMatches = Config.includeMatches,
       minMatchCharLength = Config.minMatchCharLength,
       ignoreLocation = Config.ignoreLocation,
@@ -52,6 +54,7 @@ export default class ExtendedSearch {
     this.query = null
     this.options = {
       isCaseSensitive,
+      ignoreDiacritics,
       includeMatches,
       minMatchCharLength,
       findAllMatches,
@@ -61,7 +64,9 @@ export default class ExtendedSearch {
       distance
     }
 
-    this.pattern = isCaseSensitive ? pattern : pattern.toLowerCase()
+    pattern = isCaseSensitive ? pattern : pattern.toLowerCase()
+    pattern = ignoreDiacritics ? stripDiacritics(pattern) : pattern
+    this.pattern = pattern
     this.query = parseQuery(this.pattern, this.options)
   }
 
@@ -79,9 +84,10 @@ export default class ExtendedSearch {
       }
     }
 
-    const { includeMatches, isCaseSensitive } = this.options
+    const { includeMatches, isCaseSensitive, ignoreDiacritics } = this.options
 
     text = isCaseSensitive ? text : text.toLowerCase()
+    text = ignoreDiacritics ? stripDiacritics(text) : text
 
     let numMatches = 0
     let allIndices = []

test/extended-search.test.js

@@ -109,3 +109,46 @@ describe('ignoreLocation when useExtendedSearch is true', () => {
     expect(result).toHaveLength(1)
   })
 })
+
+describe('Searching using extended search ignoring diactrictics', () => {
+  const list = [
+    {
+      text: 'déjà'
+    },
+    {
+      text: 'cafe'
+    }
+  ]
+
+  const options = {
+    useExtendedSearch: true,
+    ignoreDiacritics: true,
+    threshold: 0,
+    keys: ['text']
+  }
+  const fuse = new Fuse(list, options)
+
+  test('Search: query with diactrictics, list with diactrictics', () => {
+    let result = fuse.search('déjà')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(0)
+  })
+
+  test('Search: query without diactrictics, list with diactrictics', () => {
+    let result = fuse.search('deja')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(0)
+  })
+
+  test('Search: query with diactrictics, list without diactrictics', () => {
+    let result = fuse.search('café')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(1)
+  })
+
+  test('Search: query without diactrictics, list without diactrictics', () => {
+    let result = fuse.search('cafe')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(1)
+  })
+})

test/fuzzy-search.test.js

@@ -1209,3 +1209,45 @@ describe('Breaking values', () => {
     expect(result).toHaveLength(1)
   })
 })
+
+describe('Searching ignoring diactrictics', () => {
+  const list = [
+    {
+      text: 'déjà'
+    },
+    {
+      text: 'cafe'
+    }
+  ]
+
+  const options = {
+    ignoreDiacritics: true,
+    threshold: 0,
+    keys: ['text']
+  }
+  const fuse = new Fuse(list, options)
+
+  test('Search: query with diactrictics, list with diactrictics', () => {
+    let result = fuse.search('déjà')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(0)
+  })
+
+  test('Search: query without diactrictics, list with diactrictics', () => {
+    let result = fuse.search('deja')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(0)
+  })
+
+  test('Search: query with diactrictics, list without diactrictics', () => {
+    let result = fuse.search('café')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(1)
+  })
+
+  test('Search: query without diactrictics, list without diactrictics', () => {
+    let result = fuse.search('cafe')
+    expect(result).toHaveLength(1)
+    expect(result[0].refIndex).toBe(1)
+  })
+})