Merge pull request #50 from stockholmux/master

stockholmux · web-flow · commit d71c48e14c37 · 2017-04-17T17:21:37.000-04:00
version bump / 1.0
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,62 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
 .DS_Store
-node_modules
 *.sock
 testing.js
diff --git a/History.md b/History.md
@@ -1,3 +1,9 @@
+1.0.0 / 
+==================
+
+ * Updated tests and benchmarks to take a connection object from an command line argument
+ * Updated creaky dependencies 
+ * Added the ability to plug in new natural language processor and options
 
 0.2.4 / 2013-08-10 
 ==================
diff --git a/Readme.md b/Readme.md
@@ -2,13 +2,17 @@
 
   reds is a light-weight Redis search for node.js. This module was originally developed to provide search capabilities for [Kue](http://automattic.github.io/kue/) a priority job queue, however it is very much a light general purpose search library that could be integrated into a blog, a documentation server, etc.
 
+## Upgrading
+
+Version 1.0.0 is syntactically compatible with previous versions of reds (0.2.5). However, [natural](https://github.com/NaturalNode/natural) has been updated. Documents indexed with older installs of reds (using natural v0.2.0) may need to be re-indexed to avoid some edge cases.
+
 ## Installation
 
       $ npm install reds
 
 ## Example
 
-The first thing you'll want to do is create a `Search` instance, which allows you to pass a `key`, used for namespacing within Redis so that you may have several searches in the same db.
+The first thing you'll want to do is create a `Search` instance, which allows you to pass a `key`, used for namespacing within Redis so that you may have several searches in the same db. You may specify your own [node_redis](https://github.com/NodeRedis/node_redis) instance with the `reds.setClient` function.
 
     var search = reds.createSearch('pets');
 
@@ -93,6 +97,24 @@ search.remove('bcd');
 search.query('foo bar').end(function(err, ids){});
 ```
 
+## Extending reds
+
+Starting in 1.0.0, you can easily extend and expand how reds functions. When creating a new search, supply an object as the second argument. There are currently three properties that can be configured:
+
+- `nlpProcess` the natural language processing function. You can alter how the words are processed (split, stemmed, and converted to metaphones) using this function.
+- `writeIndex` how the items are written to the index. 
+- `removeIndex` how the items are removed from the index.
+
+See the `lib/reds.js` file for the implementation of each. Please keep in mind that changing these functions may invalidate your previously stored index.
+
+```js
+reds.createSearch('pets', {
+  nlpProcess  : yourNlpProcessingFunction,
+  writeIndex  : yourWriteIndexFunction,
+  removeIndex : yourRemoveIndexFunction
+});
+```
+
 ## About
 
   Currently reds strips stop words and applies the metaphone and porter stemmer algorithms to the remaining words before mapping the constants in Redis sets. For example the following text:
diff --git a/benchmarks/index.js b/benchmarks/index.js
@@ -2,10 +2,20 @@
 /**
  * Module dependencies.
  */
-
-var reds = require('../').createSearch('reds');
+var argv = require('yargs')
+    .demand('connection')
+    .argv;
+var redis = require('redis');
+var connectionObj = require(argv.connection);
+var reds;
 var fs = require('fs');
 
+
+reds = require('../');
+
+reds.setClient(redis.createClient(connectionObj));
+
+reds = reds.createSearch('reds');
 // test data
 
 var tiny = fs.readFileSync('package.json', 'utf8');
diff --git a/lib/reds.js b/lib/reds.js
@@ -1,4 +1,3 @@
-
 /*!
  * reds
  * Copyright(c) 2011 TJ Holowaychuk <tj@vision-media.ca>
@@ -20,7 +19,7 @@ function noop(){};
  * Library version.
  */
 
-exports.version = '0.1.4';
+exports.version = '1.0.0';
 
 /**
  * Expose `Search`.
@@ -45,6 +44,19 @@ var types = {
   or: 'zunionstore'
 };
 
+/**
+ * Alternate way to set client 
+ * provide your own behaviour.
+ *
+ * @param {RedisClient} inClient
+ * @return {RedisClient}
+ * @api public
+ */
+
+exports.setClient = function(inClient) {
+  return exports.client = inClient;
+}
+
 /**
  * Create a redis client, override to
  * provide your own behaviour.
@@ -58,17 +70,89 @@ exports.createClient = function(){
     || (exports.client = redis.createClient());
 };
 
+/**
+ * Process indexing string into NLP components (words, counts, map, keys)
+ * 
+ * @param {String} str
+ * @param {String} [key]
+ */
+exports.nlpProcess = function(str, key) {
+  var words = exports.stem(exports.stripStopWords(exports.words(str)));
+  var counts = exports.countWords(words);
+  var map = exports.metaphoneMap(words);
+  var keys = Object.keys(map);
+  //the key argument is only needed for removing items from the index
+  //by not executing the `metaphoneKeys` function, it speeds up indexing a bit
+  var metaphoneKeys = !key ? null : exports.metaphoneKeys(key, words)
+  
+
+  return {
+    words   : words,
+    counts  : counts,
+    map     : map,
+    keys    : keys,
+    metaphoneKeys
+            : metaphoneKeys
+  };
+}
+
+/**
+ * Writes index to Redis
+ * 
+ * @param {Object} db Redis Client
+ * @param {Number|String} id
+ * @param {String} key
+ * @param {Object} nlpObj Object with word map, counts, and keys
+ * @param {Function} fn
+ * 
+ */
+exports.writeIndex = function(db, id, key, nlpObj, fn) {
+  var cmds = [];
+  nlpObj.keys.forEach(function(word, i){
+    cmds.push(['zadd', key + ':word:' + nlpObj.map[word], nlpObj.counts[word], id]);
+    cmds.push(['zadd', key + ':object:' + id, nlpObj.counts[word], nlpObj.map[word]]);
+  });
+  db.multi(cmds).exec(fn || noop);
+}
+
+/**
+ * Removes index from Redis
+ * 
+ * @param {Object} db Redis Client
+ * @param {Number|String} id
+ * @param {String} key
+ * @param {Function} fn
+ * 
+ */
+exports.removeIndex = function(db, id, key, fn) {
+  db.zrevrangebyscore(key + ':object:' + id, '+inf', 0, function(err, constants){
+    if (err) return fn(err);
+    var multi = db.multi().del(key + ':object:' + id);
+    constants.forEach(function(c){
+      multi.zrem(key + ':word:' + c, id);
+    });
+    multi.exec(fn);
+  });
+}
+
+
 /**
  * Return a new reds `Search` with the given `key`.
- *
+ * @param {Object} opts
  * @param {String} key
  * @return {Search}
  * @api public
  */
 
-exports.createSearch = function(key){
+exports.createSearch = function(key,opts){
   if (!key) throw new Error('createSearch() requires a redis key for namespacing');
-  return new Search(key);
+  
+  opts = !opts ? {} : opts;
+  opts.nlpProcess = !opts.nlpProcess ? exports.nlpProcess : opts.nlpProcess;
+  opts.writeIndex = !opts.writeIndex ? exports.writeIndex : opts.writeIndex;
+  opts.removeIndex = !opts.removeIndex ? exports.removeIndex : opts.removeIndex;
+  
+  return new Search(key, opts.nlpProcess, opts.writeIndex, opts.removeIndex);
 };
 
 /**
@@ -256,8 +340,8 @@ Query.prototype.end = function(fn){
   var key = this.search.key;
   var db = this.search.client;
   var query = this.str;
-  var words = exports.stem(exports.stripStopWords(exports.words(query)));
-  var keys = exports.metaphoneKeys(key, words);
+  var nlpObj = this.search.nlpProcess(query, key);
+  var keys = nlpObj.metaphoneKeys;
   var type = this._type;
   var start = this._start || 0;
   var stop = this._stop || -1;
@@ -287,9 +371,13 @@ Query.prototype.end = function(fn){
  * @api public
  */
 
-function Search(key) {
+function Search(key, nlpProcess, writeIndex, removeIndex) {
   this.key = key;
   this.client = exports.createClient();
+
+  this.nlpProcess = nlpProcess;
+  this.writeIndex = writeIndex;
+  this.removeIndex = removeIndex;
 }
 
 /**
@@ -304,17 +392,9 @@ function Search(key) {
 Search.prototype.index = function(str, id, fn){
   var key = this.key;
   var db = this.client;
-  var words = exports.stem(exports.stripStopWords(exports.words(str)));
-  var counts = exports.countWords(words);
-  var map = exports.metaphoneMap(words);
-  var keys = Object.keys(map);
+  var nlpObj = this.nlpProcess(str);
 
-  var cmds = [];
-  keys.forEach(function(word, i){
-    cmds.push(['zadd', key + ':word:' + map[word], counts[word], id]);
-    cmds.push(['zadd', key + ':object:' + id, counts[word], map[word]]);
-  });
-  db.multi(cmds).exec(fn || noop);
+  this.writeIndex(db, id, key, nlpObj, fn);
 
   return this;
 };
@@ -331,14 +411,7 @@ Search.prototype.remove = function(id, fn){
   var key = this.key;
   var db = this.client;
   
-  db.zrevrangebyscore(key + ':object:' + id, '+inf', 0, function(err, constants){
-    if (err) return fn(err);
-    var multi = db.multi().del(key + ':object:' + id);
-    constants.forEach(function(c){
-      multi.zrem(key + ':word:' + c, id);
-    });
-    multi.exec(fn);
-  });
+  this.removeIndex(db, id, key, fn);
   
   return this;
 };
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "reds",
-  "version": "0.2.5",
+  "version": "1.0.0",
   "description": "Redis search for node.js",
   "keywords": [
     "redis",
@@ -11,10 +11,12 @@
   ],
   "author": "TJ Holowaychuk <tj@vision-media.ca>",
   "dependencies": {
-    "natural": "^0.5.0",
-    "redis": "^0.12.1"
+    "natural": "0.5.0",
+    "redis": "^2.7.1"
   },
   "devDependencies": {
+    "yargs": "^7.0.2",
+    "async": "^2.3.0",
     "matcha": "^0.6.0",
     "should": "^3.3.2",
     "superagent": "^0.21.0"
diff --git a/test/index.js b/test/index.js
