Skip to content

Commit 8aadde8

Browse files
futpibsindresorhus
futpib
authored and
sindresorhus
committed
Add globby.stream (#113)
1 parent 2dd76d2 commit 8aadde8

7 files changed

+226
-12
lines changed

‎index.d.ts

+21
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,27 @@ declare const globby: {
111111
options?: globby.GlobbyOptions
112112
): string[];
113113

114+
/**
115+
@param patterns - See supported `minimatch` [patterns](https://github.com/isaacs/minimatch#usage).
116+
@param options - See the [`fast-glob` options](https://github.com/mrmlnc/fast-glob#options-1) in addition to the ones in this package.
117+
@returns The stream of matching paths.
118+
119+
@example
120+
```
121+
import globby = require('globby');
122+
123+
(async () => {
124+
for await (const path of globby.stream('*.tmp')) {
125+
console.log(path);
126+
}
127+
})();
128+
```
129+
*/
130+
stream(
131+
patterns: string | readonly string[],
132+
options?: globby.GlobbyOptions
133+
): NodeJS.ReadableStream;
134+
114135
/**
115136
Note that you should avoid running the same tasks multiple times as they contain a file system cache. Instead, run this method each time to ensure file system changes are taken into consideration.
116137

‎index.js

+27-7
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
'use strict';
22
const fs = require('fs');
33
const arrayUnion = require('array-union');
4+
const merge2 = require('merge2');
45
const glob = require('glob');
56
const fastGlob = require('fast-glob');
67
const dirGlob = require('dir-glob');
78
const gitignore = require('./gitignore');
9+
const {FilterStream, UniqueStream} = require('./stream-utils');
810

911
const DEFAULT_FILTER = () => false;
1012

@@ -81,6 +83,12 @@ const globDirs = (task, fn) => {
8183

8284
const getPattern = (task, fn) => task.options.expandDirectories ? globDirs(task, fn) : [task.pattern];
8385

86+
const getFilterSync = options => {
87+
return options && options.gitignore ?
88+
gitignore.sync({cwd: options.cwd, ignore: options.ignore}) :
89+
DEFAULT_FILTER;
90+
};
91+
8492
const globToTask = task => glob => {
8593
const {options} = task;
8694
if (options.ignore && Array.isArray(options.ignore) && options.expandDirectories) {
@@ -120,24 +128,36 @@ module.exports = async (patterns, options) => {
120128
module.exports.sync = (patterns, options) => {
121129
const globTasks = generateGlobTasks(patterns, options);
122130

123-
const getFilter = () => {
124-
return options && options.gitignore ?
125-
gitignore.sync({cwd: options.cwd, ignore: options.ignore}) :
126-
DEFAULT_FILTER;
127-
};
128-
129131
const tasks = globTasks.reduce((tasks, task) => {
130132
const newTask = getPattern(task, dirGlob.sync).map(globToTask(task));
131133
return tasks.concat(newTask);
132134
}, []);
133135

134-
const filter = getFilter();
136+
const filter = getFilterSync(options);
137+
135138
return tasks.reduce(
136139
(matches, task) => arrayUnion(matches, fastGlob.sync(task.pattern, task.options)),
137140
[]
138141
).filter(path_ => !filter(path_));
139142
};
140143

144+
module.exports.stream = (patterns, options) => {
145+
const globTasks = generateGlobTasks(patterns, options);
146+
147+
const tasks = globTasks.reduce((tasks, task) => {
148+
const newTask = getPattern(task, dirGlob.sync).map(globToTask(task));
149+
return tasks.concat(newTask);
150+
}, []);
151+
152+
const filter = getFilterSync(options);
153+
const filterStream = new FilterStream(p => !filter(p));
154+
const uniqueStream = new UniqueStream();
155+
156+
return merge2(tasks.map(task => fastGlob.stream(task.pattern, task.options)))
157+
.pipe(filterStream)
158+
.pipe(uniqueStream);
159+
};
160+
141161
module.exports.generateGlobTasks = generateGlobTasks;
142162

143163
module.exports.hasMagic = (patterns, options) => []

‎index.test-d.ts

+28
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
GlobTask,
55
FilterFunction,
66
sync as globbySync,
7+
stream as globbyStream,
78
generateGlobTasks,
89
hasMagic,
910
gitignore
@@ -45,6 +46,33 @@ expectType<string[]>(
4546
expectType<string[]>(globbySync('*.tmp', {gitignore: true}));
4647
expectType<string[]>(globbySync('*.tmp', {ignore: ['**/b.tmp']}));
4748

49+
// Globby (stream)
50+
expectType<NodeJS.ReadableStream>(globbyStream('*.tmp'));
51+
expectType<NodeJS.ReadableStream>(globbyStream(['a.tmp', '*.tmp', '!{c,d,e}.tmp']));
52+
53+
expectType<NodeJS.ReadableStream>(globbyStream('*.tmp', {expandDirectories: false}));
54+
expectType<NodeJS.ReadableStream>(globbyStream('*.tmp', {expandDirectories: ['a*', 'b*']}));
55+
expectType<NodeJS.ReadableStream>(
56+
globbyStream('*.tmp', {
57+
expandDirectories: {
58+
files: ['a', 'b'],
59+
extensions: ['tmp']
60+
}
61+
})
62+
);
63+
expectType<NodeJS.ReadableStream>(globbyStream('*.tmp', {gitignore: true}));
64+
expectType<NodeJS.ReadableStream>(globbyStream('*.tmp', {ignore: ['**/b.tmp']}));
65+
66+
(async () => {
67+
const streamResult = [];
68+
for await (const path of globbyStream('*.tmp')) {
69+
streamResult.push(path);
70+
}
71+
// `NodeJS.ReadableStream` is not generic, unfortunately,
72+
// so it seems `(string | Buffer)[]` is the best we can get here
73+
expectType<(string | Buffer)[]>(streamResult);
74+
})();
75+
4876
// GenerateGlobTasks
4977
expectType<GlobTask[]>(generateGlobTasks('*.tmp'));
5078
expectType<GlobTask[]>(generateGlobTasks(['a.tmp', '*.tmp', '!{c,d,e}.tmp']));

‎package.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
"files": [
2020
"index.js",
2121
"gitignore.js",
22-
"index.d.ts"
22+
"index.d.ts",
23+
"stream-utils.js"
2324
],
2425
"keywords": [
2526
"all",
@@ -61,10 +62,12 @@
6162
"fast-glob": "^2.2.6",
6263
"glob": "^7.1.3",
6364
"ignore": "^5.1.1",
65+
"merge2": "^1.2.3",
6466
"slash": "^3.0.0"
6567
},
6668
"devDependencies": {
6769
"ava": "^2.1.0",
70+
"get-stream": "^5.1.0",
6871
"glob-stream": "^6.1.0",
6972
"globby": "sindresorhus/globby#master",
7073
"matcha": "^0.7.0",

‎readme.md

+18
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ Default: `true`
6767
If set to `true`, `globby` will automatically glob directories for you. If you define an `Array` it will only glob files that matches the patterns inside the `Array`. You can also define an `object` with `files` and `extensions` like below:
6868

6969
```js
70+
const globby = require('globby');
71+
7072
(async () => {
7173
const paths = await globby('images', {
7274
expandDirectories: {
@@ -93,6 +95,22 @@ Respect ignore patterns in `.gitignore` files that apply to the globbed files.
9395

9496
Returns `string[]` of matching paths.
9597

98+
### globby.stream(patterns, options?)
99+
100+
Returns a [`stream.Readable`](https://nodejs.org/api/stream.html#stream_readable_streams) of matching paths.
101+
102+
Since Node.js 10, [readable streams are iterable](https://nodejs.org/api/stream.html#stream_readable_symbol_asynciterator), so you can loop over glob matches in a [`for await...of` loop](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) like this:
103+
104+
```js
105+
const globby = require('globby');
106+
107+
(async () => {
108+
for await (const path of globby.stream('*.tmp')) {
109+
console.log(path);
110+
}
111+
})();
112+
```
113+
96114
### globby.generateGlobTasks(patterns, options?)
97115

98116
Returns an `object[]` in the format `{pattern: string, options: Object}`, which can be passed as arguments to [`fast-glob`](https://github.com/mrmlnc/fast-glob). This is useful for other globbing-related packages.

‎stream-utils.js

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
'use strict';
2+
const {Transform} = require('stream');
3+
4+
class ObjectTransform extends Transform {
5+
constructor() {
6+
super({
7+
objectMode: true
8+
});
9+
}
10+
}
11+
12+
class FilterStream extends ObjectTransform {
13+
constructor(filter) {
14+
super();
15+
this._filter = filter;
16+
}
17+
18+
_transform(data, encoding, callback) {
19+
if (this._filter(data)) {
20+
this.push(data);
21+
}
22+
23+
callback();
24+
}
25+
}
26+
27+
class UniqueStream extends ObjectTransform {
28+
constructor() {
29+
super();
30+
this._pushed = new Set();
31+
}
32+
33+
_transform(data, encoding, callback) {
34+
if (!this._pushed.has(data)) {
35+
this.push(data);
36+
this._pushed.add(data);
37+
}
38+
39+
callback();
40+
}
41+
}
42+
43+
module.exports = {
44+
FilterStream,
45+
UniqueStream
46+
};

‎test.js

+82-4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import fs from 'fs';
22
import util from 'util';
33
import path from 'path';
44
import test from 'ava';
5+
import getStream from 'get-stream';
56
import globby from '.';
67

78
const cwd = process.cwd();
@@ -72,6 +73,41 @@ test('return [] for all negative patterns - async', async t => {
7273
t.deepEqual(await globby(['!a.tmp', '!b.tmp']), []);
7374
});
7475

76+
test('glob - stream', async t => {
77+
t.deepEqual((await getStream.array(globby.stream('*.tmp'))).sort(), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
78+
});
79+
80+
// Readable streams are iterable since Node.js 10, but this test runs on 6 and 8 too.
81+
// So we define the test only if async iteration is supported.
82+
if (Symbol.asyncIterator) {
83+
// For the reason behind `eslint-disable` below see https://github.com/avajs/eslint-plugin-ava/issues/216
84+
// eslint-disable-next-line ava/no-async-fn-without-await
85+
test('glob - stream async iterator support', async t => {
86+
const results = [];
87+
for await (const path of globby.stream('*.tmp')) {
88+
results.push(path);
89+
}
90+
91+
t.deepEqual(results, ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
92+
});
93+
}
94+
95+
test('glob - stream - multiple file paths', async t => {
96+
t.deepEqual(await getStream.array(globby.stream(['a.tmp', 'b.tmp'])), ['a.tmp', 'b.tmp']);
97+
});
98+
99+
test('glob with multiple patterns - stream', async t => {
100+
t.deepEqual(await getStream.array(globby.stream(['a.tmp', '*.tmp', '!{c,d,e}.tmp'])), ['a.tmp', 'b.tmp']);
101+
});
102+
103+
test('respect patterns order - stream', async t => {
104+
t.deepEqual(await getStream.array(globby.stream(['!*.tmp', 'a.tmp'])), ['a.tmp']);
105+
});
106+
107+
test('return [] for all negative patterns - stream', async t => {
108+
t.deepEqual(await getStream.array(globby.stream(['!a.tmp', '!b.tmp'])), []);
109+
});
110+
75111
test('cwd option', t => {
76112
process.chdir(tmp);
77113
t.deepEqual(globby.sync('*.tmp', {cwd}), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
@@ -89,6 +125,11 @@ test('don\'t mutate the options object - sync', t => {
89125
t.pass();
90126
});
91127

128+
test('don\'t mutate the options object - stream', async t => {
129+
await getStream.array(globby.stream(['*.tmp', '!b.tmp'], Object.freeze({ignore: Object.freeze([])})));
130+
t.pass();
131+
});
132+
92133
test('expose generateGlobTasks', t => {
93134
const tasks = globby.generateGlobTasks(['*.tmp', '!b.tmp'], {ignore: ['c.tmp']});
94135

@@ -180,7 +221,7 @@ test.failing('relative paths and ignores option', t => {
180221
await t.throwsAsync(globby(value), message);
181222
});
182223

183-
test(`throws for invalid patterns input: ${valueString}`, t => {
224+
test(`throws for invalid patterns input: ${valueString} - sync`, t => {
184225
t.throws(() => {
185226
globby.sync(value);
186227
}, TypeError);
@@ -190,6 +231,16 @@ test.failing('relative paths and ignores option', t => {
190231
}, message);
191232
});
192233

234+
test(`throws for invalid patterns input: ${valueString} - stream`, t => {
235+
t.throws(() => {
236+
globby.stream(value);
237+
}, TypeError);
238+
239+
t.throws(() => {
240+
globby.stream(value);
241+
}, message);
242+
});
243+
193244
test(`generateGlobTasks throws for invalid patterns input: ${valueString}`, t => {
194245
t.throws(() => {
195246
globby.generateGlobTasks(value);
@@ -201,7 +252,7 @@ test.failing('relative paths and ignores option', t => {
201252
});
202253
});
203254

204-
test('gitignore option defaults to false', async t => {
255+
test('gitignore option defaults to false - async', async t => {
205256
const actual = await globby('*', {onlyFiles: false});
206257
t.true(actual.includes('node_modules'));
207258
});
@@ -211,7 +262,12 @@ test('gitignore option defaults to false - sync', t => {
211262
t.true(actual.includes('node_modules'));
212263
});
213264

214-
test('respects gitignore option true', async t => {
265+
test('gitignore option defaults to false - stream', async t => {
266+
const actual = await getStream.array(globby.stream('*', {onlyFiles: false}));
267+
t.true(actual.includes('node_modules'));
268+
});
269+
270+
test('respects gitignore option true - async', async t => {
215271
const actual = await globby('*', {gitignore: true, onlyFiles: false});
216272
t.false(actual.includes('node_modules'));
217273
});
@@ -221,7 +277,12 @@ test('respects gitignore option true - sync', t => {
221277
t.false(actual.includes('node_modules'));
222278
});
223279

224-
test('respects gitignore option false', async t => {
280+
test('respects gitignore option true - stream', async t => {
281+
const actual = await getStream.array(globby.stream('*', {gitignore: true, onlyFiles: false}));
282+
t.false(actual.includes('node_modules'));
283+
});
284+
285+
test('respects gitignore option false - async', async t => {
225286
const actual = await globby('*', {gitignore: false, onlyFiles: false});
226287
t.true(actual.includes('node_modules'));
227288
});
@@ -237,6 +298,11 @@ test('gitignore option with stats option', async t => {
237298
t.false(actual.includes('node_modules'));
238299
});
239300

301+
test('respects gitignore option false - stream', async t => {
302+
const actual = await getStream.array(globby.stream('*', {gitignore: false, onlyFiles: false}));
303+
t.true(actual.includes('node_modules'));
304+
});
305+
240306
// https://github.com/sindresorhus/globby/issues/97
241307
test.failing('`{extension: false}` and `expandDirectories.extensions` option', t => {
242308
t.deepEqual(
@@ -284,3 +350,15 @@ test('throws when specifying a file as cwd - sync', t => {
284350
globby.sync('*', {cwd: isFile});
285351
}, 'The `cwd` option must be a path to a directory');
286352
});
353+
354+
test('throws when specifying a file as cwd - stream', t => {
355+
const isFile = path.resolve('fixtures/gitignore/bar.js');
356+
357+
t.throws(() => {
358+
globby.stream('.', {cwd: isFile});
359+
}, 'The `cwd` option must be a path to a directory');
360+
361+
t.throws(() => {
362+
globby.stream('*', {cwd: isFile});
363+
}, 'The `cwd` option must be a path to a directory');
364+
});

0 commit comments

Comments
 (0)