diff --git a/src/functional_test_runner/lib/config/schema.js b/src/functional_test_runner/lib/config/schema.js index 96ac32729b015..92f75d107445c 100644 --- a/src/functional_test_runner/lib/config/schema.js +++ b/src/functional_test_runner/lib/config/schema.js @@ -73,6 +73,7 @@ export const schema = Joi.object().keys({ timeouts: Joi.object().keys({ find: Joi.number().default(10000), try: Joi.number().default(40000), + waitFor: Joi.number().default(20000), esRequestTimeout: Joi.number().default(30000), kibanaStabilize: Joi.number().default(15000), navigateStatusPageCheck: Joi.number().default(250), diff --git a/test/common/services/retry.js b/test/common/services/retry.js deleted file mode 100644 index aacc2e87f6c88..0000000000000 --- a/test/common/services/retry.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import bluebird from 'bluebird'; - -export function RetryProvider({ getService }) { - const config = getService('config'); - const log = getService('log'); - - class Retry { - tryForTime(timeout, block) { - const start = Date.now(); - const retryDelay = 502; - let lastTry = 0; - let finalMessage; - let prevMessage; - - function attempt() { - lastTry = Date.now(); - - if (lastTry - start > timeout) { - throw new Error('tryForTime timeout: ' + finalMessage); - } - - return bluebird - .try(block) - .catch(function tryForTimeCatch(err) { - if (err.message === prevMessage) { - log.debug('--- tryForTime errored again with the same message ...'); - } else { - prevMessage = err.message; - log.debug('--- tryForTime error: ' + prevMessage); - } - finalMessage = err.stack || err.message; - return bluebird.delay(retryDelay).then(attempt); - }); - } - - return bluebird.try(attempt); - } - - try(block) { - return this.tryForTime(config.get('timeouts.try'), block); - } - - tryMethod(object, method, ...args) { - return this.try(() => object[method](...args)); - } - } - - return new Retry(); -} diff --git a/test/common/services/retry/index.js b/test/common/services/retry/index.js new file mode 100644 index 0000000000000..298867a7a6b74 --- /dev/null +++ b/test/common/services/retry/index.js @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { RetryProvider } from './retry'; diff --git a/test/common/services/retry/retry.js b/test/common/services/retry/retry.js new file mode 100644 index 0000000000000..897bfb4535445 --- /dev/null +++ b/test/common/services/retry/retry.js @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { retryForTruthy } from './retry_for_truthy'; +import { retryForSuccess } from './retry_for_success'; + +export function RetryProvider({ getService }) { + const config = getService('config'); + const log = getService('log'); + + return new class Retry { + async tryForTime(timeout, block) { + return await retryForSuccess(log, { + timeout, + methodName: 'retry.tryForTime', + block + }); + } + + async try(block) { + return await retryForSuccess(log, { + timeout: config.get('timeouts.try'), + methodName: 'retry.try', + block + }); + } + + async tryMethod(object, method, ...args) { + return await retryForSuccess(log, { + timeout: config.get('timeouts.try'), + methodName: 'retry.tryMethod', + block: async () => ( + await object[method](...args) + ) + }); + } + + async waitForWithTimeout(description, timeout, block) { + await retryForTruthy(log, { + timeout, + methodName: 'retry.waitForWithTimeout', + description, + block + }); + } + + async waitFor(description, block) { + await retryForTruthy(log, { + timeout: config.get('timeouts.waitFor'), + methodName: 'retry.waitFor', + description, + block + }); + } + }; +} diff --git a/test/common/services/retry/retry_for_success.js b/test/common/services/retry/retry_for_success.js new file mode 100644 index 0000000000000..2571650a9f6ec --- /dev/null +++ b/test/common/services/retry/retry_for_success.js @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { inspect } from 'util'; + +const delay = ms => new Promise(resolve => ( + setTimeout(resolve, ms) +)); + +const returnTrue = () => true; + +const defaultOnFailure = (methodName) => (lastError) => { + throw new Error(`${methodName} timeout: ${lastError.stack || lastError.message}`); +}; + +/** + * Run a function and return either an error or result + * @param {Function} block + */ +async function runAttempt(block) { + try { + return { + result: await block() + }; + } catch (error) { + return { + // we rely on error being truthy and throwing falsy values is *allowed* + // so we cast falsy values to errors + error: error || new Error(`${inspect(error)} thrown`), + }; + } +} + +export async function retryForSuccess(log, { + timeout, + methodName, + block, + onFailure = defaultOnFailure(methodName), + accept = returnTrue +}) { + const start = Date.now(); + const retryDelay = 502; + let lastError; + + while (true) { + if (Date.now() - start > timeout) { + await onFailure(lastError); + throw new Error('expected onFailure() option to throw an error'); + } + + const { result, error } = await runAttempt(block); + + if (!error && accept(result)) { + return result; + } + + if (error) { + if (lastError && lastError.message === error.message) { + log.debug(`--- ${methodName} failed again with the same message...`); + } else { + log.debug(`--- ${methodName} error: ${error.message}`); + } + + lastError = error; + } + + await delay(retryDelay); + } +} diff --git a/test/common/services/retry/retry_for_truthy.js b/test/common/services/retry/retry_for_truthy.js new file mode 100644 index 0000000000000..65ebef1e30420 --- /dev/null +++ b/test/common/services/retry/retry_for_truthy.js @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { retryForSuccess } from './retry_for_success'; + +export async function retryForTruthy(log, { + timeout, + methodName, + description, + block +}) { + log.debug(`Waiting up to ${timeout}ms for ${description}...`); + + const accept = result => Boolean(result); + + const onFailure = lastError => { + let msg = `timed out waiting for ${description}`; + + if (lastError) { + msg = `${msg} -- last error: ${lastError.stack || lastError.message}`; + } + + throw new Error(msg); + }; + + await retryForSuccess(log, { + timeout, + methodName, + block, + onFailure, + accept + }); +}