fix: sharding and query for windows interop (#6)

* fix: sharding and query for windows interop

* chore: update CI and deps

* another circle CI

* Update package.json

* chore(lint): new Buffer is deprecated

* chore(lint): new Buffer is deprecated, forget one file

Author: daviddias

.appveyor.yml

@@ -0,0 +1,23 @@
+environment:
+  matrix:
+    - nodejs_version: "6"
+    - nodejs_version: "8"
+
+# cache:
+#  - node_modules
+
+platform:
+  - x64
+
+install:
+  - ps: Install-Product node $env:nodejs_version $env:platform
+  - npm install
+
+test_script:
+  - node --version
+  - npm --version
+  - npm test
+
+build: off
+
+version: "{build}"

.flowconfig

@@ -2,6 +2,7 @@
 .*/radium/.*
 .*/standard-changelog/.*
 .*/conventional-changelog-core/.*
+.*/node_modules/.*
 
 [include]
 

.travis.yml

@@ -3,24 +3,26 @@ language: node_js
 
 matrix:
   include:
-  - node_js: 4
-    env: CXX=g++-4.8
   - node_js: 6
-    env:
-      - SAUCE=true
-      - CXX=g++-4.8
-  - node_js: stable
     env: CXX=g++-4.8
+  - node_js: 8
+    env: CXX=g++-4.8
+    #  - node_js: stable
+    #    env: CXX=g++-4.8
 
 script:
   - npm run lint
-  - npm run flow
-  - npm test
+  - npm run test
+  - npm run coverage
+  - make test
 
 before_script:
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
 
+after_success:
+  - npm run coverage-publish
+
 addons:
   firefox: 'latest'
   apt:

circle.yml

@@ -2,11 +2,17 @@ machine:
   node:
     version: stable
 
+test:
+  post:
+    - npm run coverage -- --upload
+
 dependencies:
   pre:
     - google-chrome --version
-    - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
-    - sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
+    - curl -L -o google-chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+    - sudo dpkg -i google-chrome.deb || true
     - sudo apt-get update
-    - sudo apt-get --only-upgrade install google-chrome-stable
+    - sudo apt-get install -f
+    - sudo apt-get install --only-upgrade lsb-base
+    - sudo dpkg -i google-chrome.deb
     - google-chrome --version

package.json

@@ -4,18 +4,18 @@
   "description": "Wrapper implmentation for interface-datastore",
   "main": "src/index.js",
   "scripts": {
-    "lint": "aegir-lint",
-    "build": "aegir-build",
-    "test": "aegir-test",
+    "lint": "aegir lint",
+    "build": "aegir build",
+    "test": "aegir test",
     "flow": "flow",
-    "test:node": "aegir-test --env node",
-    "test:browser": "aegir-test --env browser",
-    "release": "aegir-release --docs",
-    "release-minor": "aegir-release --type minor --docs",
-    "release-major": "aegir-release --type major --docs",
-    "coverage": "aegir-coverage",
-    "coverage-publish": "aegir-coverage publish",
-    "docs": "aegir-docs"
+    "test:node": "aegir test --target node",
+    "test:browser": "aegir test --target browser",
+    "release": "aegir release --docs",
+    "release-minor": "aegir release --type minor --docs",
+    "release-major": "aegir release --type major --docs",
+    "coverage": "aegir coverage",
+    "coverage-publish": "aegir coverage --provider coveralls",
+    "docs": "aegir docs"
   },
   "repository": {
     "type": "git",
@@ -34,24 +34,24 @@
   },
   "homepage": "https://github.com/ipfs/js-datastore-core#readme",
   "devDependencies": {
-    "aegir": "^11.0.2",
-    "chai": "^4.1.0",
+    "aegir": "^12.1.3",
+    "chai": "^4.1.2",
     "dirty-chai": "^2.0.1",
-    "flow-bin": "^0.51.0"
+    "flow-bin": "^0.58.0"
   },
   "dependencies": {
     "async": "^2.5.0",
-    "interface-datastore": "^0.3.0",
+    "interface-datastore": "~0.4.0",
     "left-pad": "^1.1.3",
     "pull-many": "^1.0.8",
-    "pull-stream": "^3.6.0"
+    "pull-stream": "^3.6.1"
   },
   "engines": {
-    "node": ">=4.0.0",
+    "node": ">=6.0.0",
     "npm": ">=3.0.0"
   },
   "contributors": [
     "David Dias <[email protected]>",
     "Friedel Ziegelmayer <[email protected]>"
   ]
-}
+}

src/sharding.js

@@ -3,7 +3,6 @@
 
 const waterfall = require('async/waterfall')
 const parallel = require('async/parallel')
-const pull = require('pull-stream')
 const Key = require('interface-datastore').Key
 
 const sh = require('./shard')
@@ -86,8 +85,8 @@ class ShardingDatastore {
       if (!exists) {
         const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store)
         return parallel([
-          (cb) => put(shardKey, new Buffer(shard.toString() + '\n'), cb),
-          (cb) => put(shardReadmeKey, new Buffer(sh.readme), cb)
+          (cb) => put(shardKey, Buffer.from(shard.toString() + '\n'), cb),
+          (cb) => put(shardReadmeKey, Buffer.from(sh.readme), cb)
         ], err => callback(err))
       }
 
@@ -129,51 +128,44 @@ class ShardingDatastore {
 
   query (q /* : Query<Buffer> */) /* : QueryResult<Buffer> */ {
     const tq/* : Query<Buffer> */ = {
-      keysOnly: q.keysOnly
+      keysOnly: q.keysOnly,
+      offset: q.offset,
+      limit: q.limit,
+      filters: [
+        (e, cb) => cb(null, e.key.toString() !== shardKey.toString()),
+        (e, cb) => cb(null, e.key.toString() !== shardReadmeKey.toString())
+      ]
     }
 
     if (q.prefix != null) {
-      // TODO: transform
-      tq.prefix = q.prefix
+      tq.filters.push((e, cb) => {
+        cb(null, this._invertKey(e.key).toString().startsWith(q.prefix))
+      })
     }
 
     if (q.filters != null) {
-      tq.filters = q.filters.map((f) => (e, cb) => {
+      const filters = q.filters.map((f) => (e, cb) => {
         f(Object.assign({}, e, {
           key: this._invertKey(e.key)
         }), cb)
       })
+      tq.filters = tq.filters.concat(filters)
     }
 
     if (q.orders != null) {
       tq.orders = q.orders.map((o) => (res, cb) => {
-        const m = res.map((e) => {
-          return Object.assign({}, e, {
-            key: this._invertKey(e.key)
-          })
+        res.forEach((e) => { e.key = this._invertKey(e.key) })
+        o(res, (err, ordered) => {
+          if (err) {
+            return cb(err)
+          }
+          ordered.forEach((e) => { e.key = this._convertKey(e.key) })
+          cb(null, ordered)
         })
-        o(m, cb)
       })
     }
 
-    if (q.offset != null) {
-      tq.offset = q.offset + 2
-    }
-
-    if (q.limit != null) {
-      tq.limit = q.limit + 2
-    }
-
-    return pull(
-      this.child.query(tq),
-      pull.filter((e) => {
-        if (e.key.toString() === shardKey.toString() ||
-            e.key.toString() === shardReadmeKey.toString()) {
-          return false
-        }
-        return true
-      })
-    )
+    return this.child.query(tq)
   }
 
   close (callback /* : Callback<void> */) /* : void */ {

test/keytransform.spec.js

@@ -44,7 +44,7 @@ describe('KeyTransformDatastore', () => {
 
     series([
       (cb) => each(keys, (k, cb) => {
-        kStore.put(k, new Buffer(k.toString()), cb)
+        kStore.put(k, Buffer.from(k.toString()), cb)
       }, cb),
       (cb) => parallel([
         (cb) => map(keys, (k, cb) => {

test/mount.spec.js

@@ -18,7 +18,7 @@ describe('MountStore', () => {
   it('put - no mount', (done) => {
     const m = new MountStore([])
 
-    m.put(new Key('hello'), new Buffer('foo'), (err) => {
+    m.put(new Key('hello'), Buffer.from('foo'), (err) => {
       expect(err).to.be.an('Error')
       done()
     })
@@ -30,7 +30,7 @@ describe('MountStore', () => {
       prefix: new Key('cool')
     }])
 
-    m.put(new Key('/fail/hello'), new Buffer('foo'), (err) => {
+    m.put(new Key('/fail/hello'), Buffer.from('foo'), (err) => {
       expect(err).to.be.an('Error')
       done()
     })
@@ -43,7 +43,7 @@ describe('MountStore', () => {
       prefix: new Key('cool')
     }])
 
-    const val = new Buffer('hello')
+    const val = Buffer.from('hello')
     series([
       (cb) => m.put(new Key('/cool/hello'), val, cb),
       (cb) => mds.get(new Key('/hello'), (err, res) => {
@@ -61,7 +61,7 @@ describe('MountStore', () => {
       prefix: new Key('cool')
     }])
 
-    const val = new Buffer('hello')
+    const val = Buffer.from('hello')
     series([
       (cb) => mds.put(new Key('/hello'), val, cb),
       (cb) => m.get(new Key('/cool/hello'), (err, res) => {
@@ -79,7 +79,7 @@ describe('MountStore', () => {
       prefix: new Key('cool')
     }])
 
-    const val = new Buffer('hello')
+    const val = Buffer.from('hello')
     series([
       (cb) => mds.put(new Key('/hello'), val, cb),
       (cb) => m.has(new Key('/cool/hello'), (err, exists) => {
@@ -97,7 +97,7 @@ describe('MountStore', () => {
       prefix: new Key('cool')
     }])
 
-    const val = new Buffer('hello')
+    const val = Buffer.from('hello')
     series([
       (cb) => m.put(new Key('/cool/hello'), val, cb),
       (cb) => m.delete(new Key('/cool/hello'), cb),
@@ -121,7 +121,7 @@ describe('MountStore', () => {
       prefix: new Key('cool')
     }])
 
-    const val = new Buffer('hello')
+    const val = Buffer.from('hello')
     series([
       (cb) => m.put(new Key('/cool/hello'), val, cb),
       (cb) => {

test/namespace.spec.js

@@ -36,7 +36,7 @@ describe('KeyTransformDatastore', () => {
 
     series([
       (cb) => each(keys, (k, cb) => {
-        store.put(k, new Buffer(k.toString()), cb)
+        store.put(k, Buffer.from(k.toString()), cb)
       }, cb),
       (cb) => parallel([
         (cb) => map(keys, (k, cb) => {

test/sharding.spec.js

@@ -74,18 +74,17 @@ describe('ShardingStore', () => {
       const store = ss
 
       series([
-        (cb) => store.put(new Key('hello'), new Buffer('test'), cb),
+        (cb) => store.put(new Key('hello'), Buffer.from('test'), cb),
         (cb) => ms.get(new Key('ll').child(new Key('hello')), (err, res) => {
           expect(err).to.not.exist()
-          expect(res).to.eql(new Buffer('test'))
+          expect(res).to.eql(Buffer.from('test'))
           cb()
         })
       ], done)
     })
   })
 
-  // TODO: fix query prefix and orders
-  describe.skip('interface-datastore', () => {
+  describe('interface-datastore', () => {
     require('interface-datastore/src/tests')({
       setup (callback) {
         const shard = new sh.NextToLast(2)

test/tiered.spec.js

@@ -26,7 +26,7 @@ describe('Tiered', () => {
 
     it('put', (done) => {
       const k = new Key('hello')
-      const v = new Buffer('world')
+      const v = Buffer.from('world')
       series([
         (cb) => store.put(k, v, cb),
         (cb) => parallel([
@@ -44,7 +44,7 @@ describe('Tiered', () => {
 
     it('get and has, where available', (done) => {
       const k = new Key('hello')
-      const v = new Buffer('world')
+      const v = Buffer.from('world')
 
       series([
         (cb) => ms[1].put(k, v, cb),
@@ -63,7 +63,7 @@ describe('Tiered', () => {
 
     it('has and delete', (done) => {
       const k = new Key('hello')
-      const v = new Buffer('world')
+      const v = Buffer.from('world')
       series([
         (cb) => store.put(k, v, cb),
         (cb) => parallel([