feat: sync with go-ipfs 0.5 (#3013)

New features:

- `ipfs.dht.get` accepts string or buffer
- http-client `ipfs.dht.get` returns a buffer to match core api

Breaking changes:

- ipfs.ls remove all options in core api, but keep `long` in http api and cli same as go-ipfs
  - remove sort from the cli and ignore go-ipfs `U` option
  - enable http api mfs tests
- key.gen defaults, to rsa and 2048
- pin.list only supports streaming responses
- -U argument to `files.ls` has been removed
- dht put uses body to send data instead of query string 

Co-authored-by: Alex Potsides <[email protected]>

Author: hugomrdias

package.json

@@ -70,7 +70,7 @@
   "devDependencies": {
     "aegir": "^22.0.0",
     "cross-env": "^7.0.0",
-    "go-ipfs-dep": "0.4.23-3",
+    "go-ipfs-dep": "^0.5.1",
     "interface-ipfs-core": "^0.135.1",
     "ipfsd-ctl": "^4.1.1",
     "it-all": "^1.0.1",

src/dht/get.js

@@ -1,30 +1,25 @@
 'use strict'
 
 const { Buffer } = require('buffer')
-const encodeBufferURIComponent = require('../lib/encode-buffer-uri-component')
 const configure = require('../lib/configure')
 const toUrlSearchParams = require('../lib/to-url-search-params')
 const { Value } = require('./response-types')
 
 module.exports = configure(api => {
   return async function get (key, options = {}) {
-    if (!Buffer.isBuffer(key)) {
-      throw new Error('invalid key')
-    }
-
     const res = await api.post('dht/get', {
       timeout: options.timeout,
       signal: options.signal,
       searchParams: toUrlSearchParams({
-        key: encodeBufferURIComponent(key),
+        arg: Buffer.isBuffer(key) ? key.toString() : key,
         ...options
       }),
       headers: options.headers
     })
 
     for await (const message of res.ndjson()) {
       if (message.Type === Value) {
-        return message.Extra
+        return Buffer.from(message.Extra, 'base64')
       }
     }
 

src/dht/put.js

@@ -5,20 +5,20 @@ const multiaddr = require('multiaddr')
 const toCamel = require('../lib/object-to-camel')
 const configure = require('../lib/configure')
 const toUrlSearchParams = require('../lib/to-url-search-params')
+const multipartRequest = require('../lib/multipart-request')
 
 module.exports = configure(api => {
   return async function * put (key, value, options = {}) {
     const res = await api.post('dht/put', {
       timeout: options.timeout,
       signal: options.signal,
       searchParams: toUrlSearchParams({
-        arg: [
-          key,
-          value
-        ],
+        arg: key,
         ...options
       }),
-      headers: options.headers
+      ...(
+        await multipartRequest(value, options.headers)
+      )
     })
 
     for await (let message of res.ndjson()) {

src/files/ls.js

@@ -17,12 +17,8 @@ module.exports = configure(api => {
       signal: options.signal,
       searchParams: toUrlSearchParams({
         arg: CID.isCID(path) ? `/ipfs/${path}` : path,
-
-        // TODO the args below are not in the go-ipfs or interface core docs
-        long: options.long == null ? true : options.long,
-
-        // TODO: remove after go-ipfs 0.5 is released
-        l: options.long == null ? true : options.long,
+        // default long to true, diverges from go-ipfs where its false by default
+        long: true,
         ...options,
         stream: true
       }),

src/lib/encode-buffer-uri-component.js

@@ -7,7 +7,7 @@ const modeToString = require('../lib/mode-to-string')
 const mtimeToObject = require('../lib/mtime-to-object')
 const merge = require('merge-options').bind({ ignoreUndefined: true })
 
-async function multipartRequest (source, abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) {
+async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) {
   async function * streamFiles (source) {
     try {
       let index = 0

src/lib/multipart-request.js

@@ -25,14 +25,7 @@ module.exports = configure(api => {
     })
 
     for await (const pin of res.ndjson()) {
-      if (pin.Keys) { // non-streaming response
-        // eslint-disable-next-line guard-for-in
-        for (const key in pin.Keys) {
-          yield { cid: new CID(key), type: pin.Keys[key].Type }
-        }
-      } else {
-        yield { cid: new CID(pin.Cid), type: pin.Type }
-      }
+      yield { cid: new CID(pin.Cid), type: pin.Type }
     }
   }
 })

src/pin/ls.js

@@ -1,25 +1,29 @@
 'use strict'
 
-const { Buffer } = require('buffer')
-const encodeBuffer = require('../lib/encode-buffer-uri-component')
 const configure = require('../lib/configure')
 const toUrlSearchParams = require('../lib/to-url-search-params')
+const multipartRequest = require('../lib/multipart-request')
+const anySignal = require('any-signal')
+const AbortController = require('abort-controller')
 
 module.exports = configure(api => {
   return async (topic, data, options = {}) => {
-    data = Buffer.from(data)
-
     const searchParams = toUrlSearchParams({
-      arg: [
-        topic
-      ],
+      arg: topic,
       ...options
     })
 
-    const res = await api.post(`pubsub/pub?${searchParams}&arg=${encodeBuffer(data)}`, {
+    // allow aborting requests on body errors
+    const controller = new AbortController()
+    const signal = anySignal([controller.signal, options.signal])
+
+    const res = await api.post('pubsub/pub', {
       timeout: options.timeout,
-      signal: options.signal,
-      headers: options.headers
+      signal,
+      searchParams,
+      ...(
+        await multipartRequest(data, controller, options.headers)
+      )
     })
 
     await res.text()

src/pubsub/publish.js

@@ -51,7 +51,12 @@ describe('interface-ipfs-core tests', () => {
     ]
   })
 
-  tests.bitswap(commonFactory)
+  tests.bitswap(commonFactory, {
+    skip: [{
+      name: 'should get the wantlist by peer ID for a different node',
+      reason: 'unskip when https://github.com/ipfs/go-bitswap/pull/390 is released in go-ipfs'
+    }]
+  })
 
   tests.block(commonFactory, {
     skip: [{