Skip to content

Commit

Permalink
Add globby.stream
Browse files Browse the repository at this point in the history
  • Loading branch information
futpib committed Mar 16, 2019
1 parent 89cee24 commit 958e815
Show file tree
Hide file tree
Showing 5 changed files with 156 additions and 11 deletions.
34 changes: 27 additions & 7 deletions index.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
'use strict';
const fs = require('fs');
const arrayUnion = require('array-union');
const merge2 = require('merge2');
const glob = require('glob');
const fastGlob = require('fast-glob');
const dirGlob = require('dir-glob');
const gitignore = require('./gitignore');
const {FilterStream, UniqueStream} = require('./stream-utils');

const DEFAULT_FILTER = () => false;

Expand Down Expand Up @@ -71,6 +73,12 @@ const globDirs = (task, fn) => {

const getPattern = (task, fn) => task.options.expandDirectories ? globDirs(task, fn) : [task.pattern];

const getFilterSync = options => {
return options && options.gitignore ?
gitignore.sync({cwd: options.cwd, ignore: options.ignore}) :
DEFAULT_FILTER;
};

const globToTask = task => glob => {
const {options} = task;
if (options.ignore && Array.isArray(options.ignore) && options.expandDirectories) {
Expand Down Expand Up @@ -120,24 +128,36 @@ module.exports.default = globby;
module.exports.sync = (patterns, options) => {
const globTasks = generateGlobTasks(patterns, options);

const getFilter = () => {
return options && options.gitignore ?
gitignore.sync({cwd: options.cwd, ignore: options.ignore}) :
DEFAULT_FILTER;
};

const tasks = globTasks.reduce((tasks, task) => {
const newTask = getPattern(task, dirGlob.sync).map(globToTask(task));
return tasks.concat(newTask);
}, []);

const filter = getFilter();
const filter = getFilterSync(options);

return tasks.reduce(
(matches, task) => arrayUnion(matches, fastGlob.sync(task.pattern, task.options)),
[]
).filter(p => !filter(p));
};

module.exports.stream = (patterns, options) => {
const globTasks = generateGlobTasks(patterns, options);

const tasks = globTasks.reduce((tasks, task) => {
const newTask = getPattern(task, dirGlob.sync).map(globToTask(task));
return tasks.concat(newTask);
}, []);

const filter = getFilterSync(options);
const filterStream = new FilterStream(p => !filter(p));
const uniqueStream = new UniqueStream();

return merge2(tasks.map(task => fastGlob.stream(task.pattern, task.options)))
.pipe(filterStream)
.pipe(uniqueStream);
};

module.exports.generateGlobTasks = generateGlobTasks;

module.exports.hasMagic = (patterns, options) => []
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,13 @@
"fast-glob": "^2.2.6",
"glob": "^7.1.3",
"ignore": "^4.0.3",
"merge2": "^1.2.3",
"pify": "^4.0.1",
"slash": "^2.0.0"
},
"devDependencies": {
"ava": "^1.2.1",
"get-stream": "4.1.0",
"glob-stream": "^6.1.0",
"globby": "sindresorhus/globby#master",
"matcha": "^0.7.0",
Expand Down
4 changes: 4 additions & 0 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ Respect ignore patterns in `.gitignore` files that apply to the globbed files.

Returns an `Array` of matching paths.

### globby.stream(patterns, [options])

Returns a `ReadableStream` of matching paths.

### globby.generateGlobTasks(patterns, [options])

Returns an `Array<Object>` in the format `{pattern: string, options: Object}`, which can be passed as arguments to [`fast-glob`](https://github.com/mrmlnc/fast-glob). This is useful for other globbing-related packages.
Expand Down
46 changes: 46 additions & 0 deletions stream-utils.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
'use strict';
const {Transform} = require('stream');

class ObjectTransform extends Transform {
constructor() {
super({
objectMode: true
});
}
}

class FilterStream extends ObjectTransform {
constructor(filter) {
super();
this._filter = filter;
}

_transform(data, encoding, callback) {
if (this._filter(data)) {
this.push(data);
}

callback();
}
}

class UniqueStream extends ObjectTransform {
constructor() {
super();
this._pushed = new Set();
}

_transform(data, encoding, callback) {
if (!this._pushed.has(data)) {
this.push(data);
this._pushed.add(data);
}

callback();
}
}

module.exports = {
FilterStream,
UniqueStream
};
81 changes: 77 additions & 4 deletions test.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import fs from 'fs';
import util from 'util';
import path from 'path';
import test from 'ava';
import getStream from 'get-stream';
import globby from '.';

const cwd = process.cwd();
Expand Down Expand Up @@ -72,6 +73,41 @@ test('return [] for all negative patterns - async', async t => {
t.deepEqual(await globby(['!a.tmp', '!b.tmp']), []);
});

test('glob - stream', async t => {
t.deepEqual((await getStream.array(globby.stream('*.tmp'))).sort(), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
});

// Readable streams are readable since node version 10, but this test runs on 6 and 8 too.
// So we define the test only if async iteration is supported.
if (Symbol.asyncIterator) {
// For the reason behind `eslint-disable` below see https://github.com/avajs/eslint-plugin-ava/issues/216
// eslint-disable-next-line ava/no-async-fn-without-await
test('glob - stream async iterator support', async t => {
const results = [];
for await (const path of globby.stream('*.tmp')) {
results.push(path);
}

t.deepEqual(results, ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
});
}

test('glob - stream - multiple file paths', async t => {
t.deepEqual(await getStream.array(globby.stream(['a.tmp', 'b.tmp'])), ['a.tmp', 'b.tmp']);
});

test('glob with multiple patterns - stream', async t => {
t.deepEqual(await getStream.array(globby.stream(['a.tmp', '*.tmp', '!{c,d,e}.tmp'])), ['a.tmp', 'b.tmp']);
});

test('respect patterns order - stream', async t => {
t.deepEqual(await getStream.array(globby.stream(['!*.tmp', 'a.tmp'])), ['a.tmp']);
});

test('return [] for all negative patterns - stream', async t => {
t.deepEqual(await getStream.array(globby.stream(['!a.tmp', '!b.tmp'])), []);
});

test('cwd option', t => {
process.chdir(tmp);
t.deepEqual(globby.sync('*.tmp', {cwd}), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
Expand All @@ -89,6 +125,11 @@ test('don\'t mutate the options object - sync', t => {
t.pass();
});

test('don\'t mutate the options object - stream', async t => {
await getStream.array(globby.stream(['*.tmp', '!b.tmp'], Object.freeze({ignore: Object.freeze([])})));
t.pass();
});

test('expose generateGlobTasks', t => {
const tasks = globby.generateGlobTasks(['*.tmp', '!b.tmp'], {ignore: ['c.tmp']});

Expand Down Expand Up @@ -180,18 +221,23 @@ test.failing('relative paths and ignores option', t => {
await t.throwsAsync(globby(v), msg);
});

test(`throws for invalid patterns input: ${valstring}`, t => {
test(`throws for invalid patterns input: ${valstring} - sync`, t => {
t.throws(() => globby.sync(v), TypeError);
t.throws(() => globby.sync(v), msg);
});

test(`throws for invalid patterns input: ${valstring} - stream`, t => {
t.throws(() => globby.stream(v), TypeError);
t.throws(() => globby.stream(v), msg);
});

test(`generateGlobTasks throws for invalid patterns input: ${valstring}`, t => {
t.throws(() => globby.generateGlobTasks(v), TypeError);
t.throws(() => globby.generateGlobTasks(v), msg);
});
});

test('gitignore option defaults to false', async t => {
test('gitignore option defaults to false - async', async t => {
const actual = await globby('*', {onlyFiles: false});
t.true(actual.indexOf('node_modules') > -1);
});
Expand All @@ -201,7 +247,12 @@ test('gitignore option defaults to false - sync', t => {
t.true(actual.indexOf('node_modules') > -1);
});

test('respects gitignore option true', async t => {
test('gitignore option defaults to false - stream', async t => {
const actual = await getStream.array(globby.stream('*', {onlyFiles: false}));
t.true(actual.indexOf('node_modules') > -1);
});

test('respects gitignore option true - async', async t => {
const actual = await globby('*', {gitignore: true, onlyFiles: false});
t.false(actual.indexOf('node_modules') > -1);
});
Expand All @@ -211,7 +262,12 @@ test('respects gitignore option true - sync', t => {
t.false(actual.indexOf('node_modules') > -1);
});

test('respects gitignore option false', async t => {
test('respects gitignore option true - stream', async t => {
const actual = await getStream.array(globby.stream('*', {gitignore: true, onlyFiles: false}));
t.false(actual.indexOf('node_modules') > -1);
});

test('respects gitignore option false - async', async t => {
const actual = await globby('*', {gitignore: false, onlyFiles: false});
t.true(actual.indexOf('node_modules') > -1);
});
Expand All @@ -221,6 +277,11 @@ test('respects gitignore option false - sync', t => {
t.true(actual.indexOf('node_modules') > -1);
});

test('respects gitignore option false - stream', async t => {
const actual = await getStream.array(globby.stream('*', {gitignore: false, onlyFiles: false}));
t.true(actual.indexOf('node_modules') > -1);
});

// https://github.com/sindresorhus/globby/issues/97
test.failing('`{extension: false}` and `expandDirectories.extensions` option', t => {
t.deepEqual(
Expand Down Expand Up @@ -268,3 +329,15 @@ test('throws when specifying a file as cwd - sync', t => {
globby.sync('*', {cwd: isFile});
}, 'The `cwd` option must be a path to a directory');
});

test('throws when specifying a file as cwd - stream', t => {
const isFile = path.resolve('fixtures/gitignore/bar.js');

t.throws(() => {
globby.stream('.', {cwd: isFile});
}, 'The `cwd` option must be a path to a directory');

t.throws(() => {
globby.stream('*', {cwd: isFile});
}, 'The `cwd` option must be a path to a directory');
});

0 comments on commit 958e815

Please sign in to comment.