Skip to content

Commit d56cf2a

Browse files
author
Spencer
authored
[6.5] [dev/build] scan node_modules rather than lots of deleteAll() calls (#24692) (#24775)
Backports the following commits to 6.5: - [dev/build] scan node_modules rather than lots of deleteAll() calls (#24692)
1 parent 81b8d4e commit d56cf2a

File tree

7 files changed

+273
-92
lines changed

7 files changed

+273
-92
lines changed

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@
249249
"@types/classnames": "^2.2.3",
250250
"@types/d3": "^5.0.0",
251251
"@types/dedent": "^0.7.0",
252+
"@types/del": "^3.0.1",
252253
"@types/elasticsearch": "^5.0.26",
253254
"@types/enzyme": "^3.1.12",
254255
"@types/eslint": "^4.16.2",

‎src/dev/build/lib/fs.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ const readFileAsync = promisify(fs.readFile);
3939
const readdirAsync = promisify(fs.readdir);
4040
const utimesAsync = promisify(fs.utimes);
4141

42-
function assertAbsolute(path) {
42+
export function assertAbsolute(path) {
4343
if (!isAbsolute(path)) {
4444
throw new TypeError(
4545
'Please use absolute paths to keep things explicit. You probably want to use `build.resolvePath()` or `config.resolveFromRepo()`.'

‎src/dev/build/lib/index.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,4 @@ export {
3131
untar,
3232
deleteAll,
3333
} from './fs';
34+
export { scanDelete } from './scan_delete';

‎src/dev/build/lib/scan_delete.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import { readdirSync } from 'fs';
21+
import { relative, resolve } from 'path';
22+
23+
import del from 'del';
24+
25+
// @ts-ignore
26+
import { mkdirp, write } from './fs';
27+
import { scanDelete } from './scan_delete';
28+
29+
const TMP = resolve(__dirname, '__tests__/__tmp__');
30+
31+
// clean and recreate TMP directory
32+
beforeEach(async () => {
33+
await del(TMP);
34+
await mkdirp(resolve(TMP, 'foo/bar/baz'));
35+
await mkdirp(resolve(TMP, 'foo/bar/box'));
36+
await mkdirp(resolve(TMP, 'a/b/c/d/e'));
37+
await write(resolve(TMP, 'a/bar'), 'foo');
38+
});
39+
40+
// cleanup TMP directory
41+
afterAll(async () => {
42+
await del(TMP);
43+
});
44+
45+
it('requires an absolute directory', async () => {
46+
await expect(
47+
scanDelete({
48+
directory: relative(process.cwd(), TMP),
49+
regularExpressions: [],
50+
})
51+
).rejects.toMatchInlineSnapshot(
52+
`[TypeError: Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`.]`
53+
);
54+
});
55+
56+
it('deletes files/folders matching regular expression', async () => {
57+
await scanDelete({
58+
directory: TMP,
59+
regularExpressions: [/^.*[\/\\](bar|c)([\/\\]|$)/],
60+
});
61+
expect(readdirSync(resolve(TMP, 'foo'))).toEqual([]);
62+
expect(readdirSync(resolve(TMP, 'a'))).toEqual(['b']);
63+
expect(readdirSync(resolve(TMP, 'a/b'))).toEqual([]);
64+
});

‎src/dev/build/lib/scan_delete.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import Fs from 'fs';
21+
22+
import del from 'del';
23+
import { join } from 'path';
24+
import * as Rx from 'rxjs';
25+
import { count, map, mergeAll, mergeMap } from 'rxjs/operators';
26+
27+
// @ts-ignore
28+
import { assertAbsolute } from './fs';
29+
30+
const getStat$ = Rx.bindNodeCallback(Fs.stat);
31+
const getReadDir$ = Rx.bindNodeCallback(Fs.readdir);
32+
33+
interface Options {
34+
directory: string;
35+
regularExpressions: RegExp[];
36+
concurrency?: 20;
37+
}
38+
39+
/**
40+
* Scan the files in a directory and delete the directories/files that
41+
* are matched by an array of regular expressions.
42+
*
43+
* @param options.directory the directory to scan, all files including dot files will be checked
44+
* @param options.regularExpressions an array of regular expressions, if any matches the file/directory will be deleted
45+
* @param options.concurrency optional concurrency to run deletes, defaults to 20
46+
*/
47+
export async function scanDelete(options: Options) {
48+
const { directory, regularExpressions, concurrency = 20 } = options;
49+
50+
assertAbsolute(directory);
51+
52+
// get an observable of absolute paths within a directory
53+
const getChildPath$ = (path: string) =>
54+
getReadDir$(path).pipe(
55+
mergeAll(),
56+
map(name => join(path, name))
57+
);
58+
59+
// get an observable of all paths to be deleted, by starting with the arg
60+
// and recursively iterating through all children, unless a child matches
61+
// one of the supplied regular expressions
62+
const getPathsToDelete$ = (path: string): Rx.Observable<string> => {
63+
if (regularExpressions.some(re => re.test(path))) {
64+
return Rx.of(path);
65+
}
66+
67+
return getStat$(path).pipe(
68+
mergeMap(stat => (stat.isDirectory() ? getChildPath$(path) : Rx.EMPTY)),
69+
mergeMap(getPathsToDelete$)
70+
);
71+
};
72+
73+
return await Rx.of(directory)
74+
.pipe(
75+
mergeMap(getPathsToDelete$),
76+
mergeMap(async path => await del(path), concurrency),
77+
count()
78+
)
79+
.toPromise();
80+
}

‎src/dev/build/tasks/clean_tasks.js‎

Lines changed: 110 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
* under the License.
1818
*/
1919

20-
import { deleteAll } from '../lib';
20+
import minimatch from 'minimatch';
21+
22+
import { deleteAll, scanDelete } from '../lib';
2123

2224
export const CleanTask = {
2325
global: true,
@@ -49,105 +51,122 @@ export const CleanTypescriptTask = {
4951
'Cleaning typescript source files that have been transpiled to JS',
5052

5153
async run(config, log, build) {
52-
await deleteAll(log, [
53-
build.resolvePath('**/*.{ts,tsx,d.ts}'),
54-
build.resolvePath('**/tsconfig*.json'),
55-
]);
54+
log.info('Deleted %d files', await scanDelete({
55+
directory: build.resolvePath(),
56+
regularExpressions: [
57+
/\.(ts|tsx|d\.ts)$/,
58+
/tsconfig.*\.json$/
59+
]
60+
}));
5661
},
5762
};
5863

5964
export const CleanExtraFilesFromModulesTask = {
6065
description: 'Cleaning tests, examples, docs, etc. from node_modules',
6166

6267
async run(config, log, build) {
63-
const deleteFromNodeModules = globs => {
64-
return deleteAll(
65-
log,
66-
globs.map(p => build.resolvePath(`node_modules/**/${p}`))
68+
const makeRegexps = patterns =>
69+
patterns.map(pattern =>
70+
minimatch.makeRe(pattern, { nocase: true })
6771
);
68-
};
6972

70-
const tests = [
71-
'test',
72-
'tests',
73-
'__tests__',
74-
'mocha.opts',
75-
'*.test.js',
76-
'*.snap',
77-
'coverage',
78-
];
79-
const docs = [
80-
'doc',
81-
'docs',
82-
'CONTRIBUTING.md',
83-
'Contributing.md',
84-
'contributing.md',
85-
'History.md',
86-
'HISTORY.md',
87-
'history.md',
88-
'CHANGELOG.md',
89-
'Changelog.md',
90-
'changelog.md',
91-
];
92-
const examples = ['example', 'examples', 'demo', 'samples'];
93-
const bins = ['.bin'];
94-
const linters = [
95-
'.eslintrc',
96-
'.eslintrc.js',
97-
'.eslintrc.yml',
98-
'.prettierrc',
99-
'.jshintrc',
100-
'.babelrc',
101-
'.jscs.json',
102-
'.lint',
103-
];
104-
const hints = ['*.flow', '*.webidl', '*.map', '@types'];
105-
const scripts = [
106-
'*.sh',
107-
'*.bat',
108-
'*.exe',
109-
'Gruntfile.js',
110-
'gulpfile.js',
111-
'Makefile',
112-
];
113-
const untranspiledSources = ['*.coffee', '*.scss', '*.sass', '.ts', '.tsx'];
114-
const editors = ['.editorconfig', '.vscode'];
115-
const git = [
116-
'.gitattributes',
117-
'.gitkeep',
118-
'.gitempty',
119-
'.gitmodules',
120-
'.keep',
121-
'.empty',
122-
];
123-
const ci = [
124-
'.travis.yml',
125-
'.coveralls.yml',
126-
'.instanbul.yml',
127-
'appveyor.yml',
128-
'.zuul.yml',
129-
];
130-
const meta = [
131-
'package-lock.json',
132-
'component.json',
133-
'bower.json',
134-
'yarn.lock',
135-
];
136-
const misc = ['.*ignore', '.DS_Store', 'Dockerfile', 'docker-compose.yml'];
137-
138-
await deleteFromNodeModules(tests);
139-
await deleteFromNodeModules(docs);
140-
await deleteFromNodeModules(examples);
141-
await deleteFromNodeModules(bins);
142-
await deleteFromNodeModules(linters);
143-
await deleteFromNodeModules(hints);
144-
await deleteFromNodeModules(scripts);
145-
await deleteFromNodeModules(untranspiledSources);
146-
await deleteFromNodeModules(editors);
147-
await deleteFromNodeModules(git);
148-
await deleteFromNodeModules(ci);
149-
await deleteFromNodeModules(meta);
150-
await deleteFromNodeModules(misc);
73+
log.info('Deleted %d files', await scanDelete({
74+
directory: build.resolvePath('node_modules'),
75+
regularExpressions: makeRegexps([
76+
// tests
77+
'**/test',
78+
'**/tests',
79+
'**/__tests__',
80+
'**/mocha.opts',
81+
'**/*.test.js',
82+
'**/*.snap',
83+
'**/coverage',
84+
85+
// docs
86+
'**/doc',
87+
'**/docs',
88+
'**/CONTRIBUTING.md',
89+
'**/Contributing.md',
90+
'**/contributing.md',
91+
'**/History.md',
92+
'**/HISTORY.md',
93+
'**/history.md',
94+
'**/CHANGELOG.md',
95+
'**/Changelog.md',
96+
'**/changelog.md',
97+
98+
// examples
99+
'**/example',
100+
'**/examples',
101+
'**/demo',
102+
'**/samples',
103+
104+
// bins
105+
'**/.bin',
106+
107+
// linters
108+
'**/.eslintrc',
109+
'**/.eslintrc.js',
110+
'**/.eslintrc.yml',
111+
'**/.prettierrc',
112+
'**/.jshintrc',
113+
'**/.babelrc',
114+
'**/.jscs.json',
115+
'**/.lint',
116+
117+
// hints
118+
'**/*.flow',
119+
'**/*.webidl',
120+
'**/*.map',
121+
'**/@types',
122+
123+
// scripts
124+
'**/*.sh',
125+
'**/*.bat',
126+
'**/*.exe',
127+
'**/Gruntfile.js',
128+
'**/gulpfile.js',
129+
'**/Makefile',
130+
131+
// untranspiled sources
132+
'**/*.coffee',
133+
'**/*.scss',
134+
'**/*.sass',
135+
'**/.ts',
136+
'**/.tsx',
137+
138+
// editors
139+
'**/.editorconfig',
140+
'**/.vscode',
141+
142+
// git
143+
'**/.gitattributes',
144+
'**/.gitkeep',
145+
'**/.gitempty',
146+
'**/.gitmodules',
147+
'**/.keep',
148+
'**/.empty',
149+
150+
// ci
151+
'**/.travis.yml',
152+
'**/.coveralls.yml',
153+
'**/.instanbul.yml',
154+
'**/appveyor.yml',
155+
'**/.zuul.yml',
156+
157+
// metadata
158+
'**/package-lock.json',
159+
'**/component.json',
160+
'**/bower.json',
161+
'**/yarn.lock',
162+
163+
// misc
164+
'**/.*ignore',
165+
'**/.DS_Store',
166+
'**/Dockerfile',
167+
'**/docker-compose.yml'
168+
])
169+
}));
151170
},
152171
};
153172

0 commit comments

Comments
 (0)