Skip to content

Commit

Permalink
Delete local ripgrep stuff (with test placeholder) - #47058
Browse files Browse the repository at this point in the history
  • Loading branch information
roblourens committed Apr 20, 2018
1 parent 9383f2f commit 1fa023b
Show file tree
Hide file tree
Showing 8 changed files with 388 additions and 808 deletions.
28 changes: 28 additions & 0 deletions extensions/search-rg/src/test/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

//
// PLEASE DO NOT MODIFY / DELETE UNLESS YOU KNOW WHAT YOU ARE DOING
//
// This file is providing the test runner to use when running extension tests.
// By default the test runner in use is Mocha based.
//
// You can provide your own test runner if you want to override it by exporting
// a function run(testRoot: string, clb: (error:Error) => void) that the extension
// host can call to run the tests. The test runner is expected to use console.log
// to report the results back to the caller. When the tests are finished, return
// a possible error to the callback or null if none.

const testRunner = require('vscode/lib/testrunner');

// You can directly control Mocha options by uncommenting the following lines
// See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info
testRunner.configure({
ui: 'tdd', // the TDD UI is being used in extension.test.ts (suite, test, etc.)
useColors: process.platform !== 'win32', // colored output from test results (only windows cannot handle)
timeout: 60000
});

export = testRunner;
220 changes: 220 additions & 0 deletions extensions/search-rg/src/test/ripgrepTextSearch.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

'use strict';

import * as path from 'path';
import * as assert from 'assert';
import 'mocha';

// import * as arrays from 'vs/base/common/arrays';
// import * as platform from 'vs/base/common/platform';

// import { RipgrepParser, getAbsoluteGlob, fixDriveC } from 'vs/workbench/services/search/node/ripgrepTextSearch';
// import { ISerializedFileMatch } from 'vs/workbench/services/search/node/search';


suite('RipgrepParser', () => {
test('stuff', done => {
done();
});

// const rootFolder = '/workspace';
// const fileSectionEnd = '\n';

// function getFileLine(relativePath: string): string {
// return `\u001b\[0m${relativePath}\u001b\[0m`;
// }

// function getMatchLine(lineNum: number, matchParts: string[]): string {
// let matchLine = `\u001b\[0m${lineNum}\u001b\[0m:` +
// `${matchParts.shift()}${RipgrepParser.MATCH_START_MARKER}${matchParts.shift()}${RipgrepParser.MATCH_END_MARKER}${matchParts.shift()}`;

// while (matchParts.length) {
// matchLine += `${RipgrepParser.MATCH_START_MARKER}${matchParts.shift()}${RipgrepParser.MATCH_END_MARKER}${matchParts.shift() || ''}`;
// }

// return matchLine;
// }

// function parseInputStrings(inputChunks: string[]): ISerializedFileMatch[] {
// return parseInput(inputChunks.map(chunk => Buffer.from(chunk)));
// }

// function parseInput(inputChunks: Buffer[]): ISerializedFileMatch[] {
// const matches: ISerializedFileMatch[] = [];
// const rgp = new RipgrepParser(1e6, rootFolder);
// rgp.on('result', (match: ISerializedFileMatch) => {
// matches.push(match);
// });

// inputChunks.forEach(chunk => rgp.handleData(chunk));
// rgp.flush();

// return matches;
// }

// function halve(str: string) {
// const halfIdx = Math.floor(str.length / 2);
// return [str.substr(0, halfIdx), str.substr(halfIdx)];
// }

// function arrayOfChars(str: string) {
// const chars = [];
// for (let char of str) {
// chars.push(char);
// }

// return chars;
// }

// test('Parses one chunk', () => {
// const input = [
// [getFileLine('a.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n')
// ];

// const results = parseInputStrings(input);
// assert.equal(results.length, 1);
// assert.deepEqual(results[0],
// <ISerializedFileMatch>{
// numMatches: 2,
// path: path.join(rootFolder, 'a.txt'),
// lineMatches: [
// {
// lineNumber: 0,
// preview: 'beforematchafter',
// offsetAndLengths: [[6, 5]]
// },
// {
// lineNumber: 1,
// preview: 'beforematchafter',
// offsetAndLengths: [[6, 5]]
// }
// ]
// });
// });

// test('Parses multiple chunks broken at file sections', () => {
// const input = [
// [getFileLine('a.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n'),
// [getFileLine('b.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n'),
// [getFileLine('c.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n')
// ];

// const results = parseInputStrings(input);
// assert.equal(results.length, 3);
// results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
// });

// const singleLineChunks = [
// getFileLine('a.txt'),
// getMatchLine(1, ['before', 'match', 'after']),
// getMatchLine(2, ['before', 'match', 'after']),
// fileSectionEnd,
// getFileLine('b.txt'),
// getMatchLine(1, ['before', 'match', 'after']),
// getMatchLine(2, ['before', 'match', 'after']),
// fileSectionEnd,
// getFileLine('c.txt'),
// getMatchLine(1, ['before', 'match', 'after']),
// getMatchLine(2, ['before', 'match', 'after']),
// fileSectionEnd
// ];

// test('Parses multiple chunks broken at each line', () => {
// const input = singleLineChunks.map(chunk => chunk + '\n');

// const results = parseInputStrings(input);
// assert.equal(results.length, 3);
// results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
// });

// test('Parses multiple chunks broken in the middle of each line', () => {
// const input = arrays.flatten(singleLineChunks
// .map(chunk => chunk + '\n')
// .map(halve));

// const results = parseInputStrings(input);
// assert.equal(results.length, 3);
// results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
// });

// test('Parses multiple chunks broken at each character', () => {
// const input = arrays.flatten(singleLineChunks
// .map(chunk => chunk + '\n')
// .map(arrayOfChars));

// const results = parseInputStrings(input);
// assert.equal(results.length, 3);
// results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
// });

// test('Parses chunks broken before newline', () => {
// const input = singleLineChunks
// .map(chunk => '\n' + chunk);

// const results = parseInputStrings(input);
// assert.equal(results.length, 3);
// results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
// });

// test('Parses chunks broken in the middle of a multibyte character', () => {
// const text = getFileLine('foo/bar') + '\n' + getMatchLine(0, ['before漢', 'match', 'after']) + '\n';
// const buf = new Buffer(text);

// // Split the buffer at every possible position - it should still be parsed correctly
// for (let i = 0; i < buf.length; i++) {
// const inputBufs = [
// buf.slice(0, i),
// buf.slice(i)
// ];

// const results = parseInput(inputBufs);
// assert.equal(results.length, 1);
// assert.equal(results[0].lineMatches.length, 1);
// assert.deepEqual(results[0].lineMatches[0].offsetAndLengths, [[7, 5]]);
// }
// });
// });

// suite('RipgrepParser - etc', () => {
// function testGetAbsGlob(params: string[]): void {
// const [folder, glob, expectedResult] = params;
// assert.equal(fixDriveC(getAbsoluteGlob(folder, glob)), expectedResult, JSON.stringify(params));
// }

// test('getAbsoluteGlob_win', () => {
// if (!platform.isWindows) {
// return;
// }

// [
// ['C:/foo/bar', 'glob/**', '/foo\\bar\\glob\\**'],
// ['c:/', 'glob/**', '/glob\\**'],
// ['C:\\foo\\bar', 'glob\\**', '/foo\\bar\\glob\\**'],
// ['c:\\foo\\bar', 'glob\\**', '/foo\\bar\\glob\\**'],
// ['c:\\', 'glob\\**', '/glob\\**'],
// ['\\\\localhost\\c$\\foo\\bar', 'glob/**', '\\\\localhost\\c$\\foo\\bar\\glob\\**'],

// // absolute paths are not resolved further
// ['c:/foo/bar', '/path/something', '/path/something'],
// ['c:/foo/bar', 'c:\\project\\folder', '/project\\folder']
// ].forEach(testGetAbsGlob);
// });

// test('getAbsoluteGlob_posix', () => {
// if (platform.isWindows) {
// return;
// }

// [
// ['/foo/bar', 'glob/**', '/foo/bar/glob/**'],
// ['/', 'glob/**', '/glob/**'],

// // absolute paths are not resolved further
// ['/', '/project/folder', '/project/folder'],
// ].forEach(testGetAbsGlob);
// });
});
27 changes: 26 additions & 1 deletion src/vs/workbench/services/search/node/fileSearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import * as extfs from 'vs/base/node/extfs';
import * as flow from 'vs/base/node/flow';
import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine, IFolderSearch } from './search';
import { spawnRipgrepCmd } from './ripgrepFileSearch';
import { rgErrorMsgForDisplay } from './ripgrepTextSearch';

enum Traversal {
Node = 1,
Expand Down Expand Up @@ -820,4 +819,30 @@ class AbsoluteAndRelativeParsedExpression {

return pathTerms;
}
}

/**
* Read the first line of stderr and return an error for display or undefined, based on a whitelist.
* Ripgrep produces stderr output which is not from a fatal error, and we only want the search to be
* "failed" when a fatal error was produced.
*/
function rgErrorMsgForDisplay(msg: string): string | undefined {
const firstLine = msg.split('\n')[0];

if (strings.startsWith(firstLine, 'Error parsing regex')) {
return firstLine;
}

if (strings.startsWith(firstLine, 'error parsing glob') ||
strings.startsWith(firstLine, 'unsupported encoding')) {
// Uppercase first letter
return firstLine.charAt(0).toUpperCase() + firstLine.substr(1);
}

if (strings.startsWith(firstLine, 'Literal ')) {
// e.g. "Literal \n not allowed"
return firstLine;
}

return undefined;
}
26 changes: 0 additions & 26 deletions src/vs/workbench/services/search/node/rawSearchService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import * as strings from 'vs/base/common/strings';
import { PPromise, TPromise } from 'vs/base/common/winjs.base';
import { FileWalker, Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch';
import { MAX_FILE_SIZE } from 'vs/platform/files/node/files';
import { RipgrepEngine } from 'vs/workbench/services/search/node/ripgrepTextSearch';
import { Engine as TextSearchEngine } from 'vs/workbench/services/search/node/textSearch';
import { TextSearchWorkerProvider } from 'vs/workbench/services/search/node/textSearchWorkerProvider';
import { IRawSearchService, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine, IFileSearchProgressItem, ITelemetryEvent } from './search';
Expand All @@ -44,31 +43,6 @@ export class SearchService implements IRawSearchService {
this.legacyTextSearch(config);
}

public ripgrepTextSearch(config: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem> {
config.maxFilesize = MAX_FILE_SIZE;
let engine = new RipgrepEngine(config);

return new PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>((c, e, p) => {
// Use BatchedCollector to get new results to the frontend every 2s at least, until 50 results have been returned
const collector = new BatchedCollector<ISerializedFileMatch>(SearchService.BATCH_SIZE, p);
engine.search((match) => {
collector.addItem(match, match.numMatches);
}, (message) => {
p(message);
}, (error, stats) => {
collector.flush();

if (error) {
e(error);
} else {
c(stats);
}
});
}, () => {
engine.cancel();
});
}

public legacyTextSearch(config: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem> {
if (!this.textSearchWorkerProvider) {
this.textSearchWorkerProvider = new TextSearchWorkerProvider();
Expand Down
Loading

0 comments on commit 1fa023b

Please sign in to comment.