Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DDW-435] Improve cardano-node process handling logic #1119

Merged
merged 8 commits into from
Oct 12, 2018
Merged

[DDW-435] Improve cardano-node process handling logic #1119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9faa3c4
[DDW-435] harden cardano-ndoe process handling logic
DominikGuzei Oct 11, 2018
6c8d1ba
[DDW-435] harden cardano-node restart logic
DominikGuzei Oct 11, 2018
4bf25e0
[DDW-435] small cleanup
DominikGuzei Oct 11, 2018
b547f61
Merge branch 'develop' into fix/ddw-435-improve-cardano-node-stopping…
nikolaglumac Oct 11, 2018
ba52437
[DDW-436] harden cardano-node killing logic
DominikGuzei Oct 12, 2018
13ffcfe
Merge branch 'fix/ddw-435-improve-cardano-node-stopping-procedure' of…
DominikGuzei Oct 12, 2018
b39a976
Merge branch 'develop' into fix/ddw-435-improve-cardano-node-stopping…
DominikGuzei Oct 12, 2018
52ae963
[DDW-435] Merge latest develop and fix conflicts
nikolaglumac Oct 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion source/common/types/cardanoNode.types.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@ export const NetworkNameOptions = {

export type CardanoNodeState = (
'stopped' | 'starting' | 'running' | 'stopping' | 'updating' |
'updated' | 'crashed' | 'errored'
'updated' | 'crashed' | 'errored' | 'exiting'
);

export const CardanoNodeStates: {
STARTING: CardanoNodeState,
RUNNING: CardanoNodeState;
EXITING: CardanoNodeState;
STOPPING: CardanoNodeState;
STOPPED: CardanoNodeState;
UPDATING: CardanoNodeState;
Expand All @@ -38,6 +39,7 @@ export const CardanoNodeStates: {
} = {
STARTING: 'starting',
RUNNING: 'running',
EXITING: 'exiting',
STOPPING: 'stopping',
STOPPED: 'stopped',
UPDATING: 'updating',
Expand Down
232 changes: 131 additions & 101 deletions source/main/cardano/CardanoNode.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
// @flow
import Store from 'electron-store';
import type { spawn, ChildProcess } from 'child_process';
import type { ChildProcess, spawn } from 'child_process';
import type { WriteStream } from 'fs';
import psList from 'ps-list';
import { isObject, toInteger } from 'lodash';
import { toInteger } from 'lodash';
import environment from '../../common/environment';
import type { CardanoNodeState, TlsConfig } from '../../common/types/cardanoNode.types';
import { promisedCondition, deriveStorageKeys, deriveProcessNames } from './utils';
import { CardanoNodeStates } from '../../common/types/cardanoNode.types';
import type { Process } from '../utils/processes';
import { deriveProcessNames, deriveStorageKeys, getProcess, promisedCondition } from './utils';

type Logger = {
debug: (string) => void,
Expand Down Expand Up @@ -163,7 +161,6 @@ export class CardanoNode {
this._log = log;
this._actions = actions;
this._transitionListeners = transitions;
this._resetTlsConfig();
}

/**
Expand Down Expand Up @@ -195,7 +192,7 @@ export class CardanoNode {

this._startupTries++;
this._changeToState(CardanoNodeStates.STARTING);
_log.info(`CardanoNode: trying to start cardano-node for the ${this._startupTries}. time.`);
_log.info(`CardanoNode#start: trying to start cardano-node for the ${this._startupTries}. time.`);

return new Promise((resolve, reject) => {
const logFile = createWriteStream(config.logFilePath, { flags: 'a' });
Expand All @@ -214,10 +211,10 @@ export class CardanoNode {
node.on('error', this._handleCardanoNodeError);
// Request cardano-node to reply with port
node.send({ QueryPort: [] });
_log.info(`CardanoNode: cardano-node child process spawned with PID ${node.pid}`);
_log.info(`CardanoNode#start: cardano-node child process spawned with PID ${node.pid}`);
resolve();
} catch (_) {
reject('CardanoNode: Error while spawning cardano-node.');
reject('CardanoNode#start: Error while spawning cardano-node.');
}
});
});
Expand All @@ -230,27 +227,28 @@ export class CardanoNode {
*
* @returns {Promise<void>} resolves if the node could be stopped, rejects with error otherwise.
*/
stop(): Promise<void> {
async stop(): Promise<void> {
const { _node, _log, _config } = this;
return new Promise(async (resolve, reject) => {
if (!_node || await this._isNodeProcessNotRunningAnymore()) return resolve();
_log.info('CardanoNode: disconnecting from cardano-node process.');
if (await this._isDead()) {
_log.info('CardanoNode#stop: process is not running anymore.');
return Promise.resolve();
}
_log.info('CardanoNode#stop: disconnecting from cardano-node process.');
try {
if (_node) _node.disconnect();
this._changeToState(CardanoNodeStates.STOPPING);
await this._waitForNodeProcessToExit(_config.shutdownTimeout);
await this._storeProcessStates();
this._reset();
return Promise.resolve();
} catch (error) {
_log.info(`CardanoNode#stop: cardano-node did not stop correctly: ${error}`);
try {
_node.disconnect();
this._changeToState(CardanoNodeStates.STOPPING);
await this._waitForNodeProcessToExit(_config.shutdownTimeout);
await this._storeProcessStates();
this._reset();
resolve();
} catch (error) {
_log.info(`CardanoNode: cardano-node did not shut itself down correctly: ${error}`);
try {
await this.kill();
} catch (killError) {
reject(killError);
}
await this.kill();
} catch (killError) {
return Promise.reject(killError);
}
});
}
}

/**
Expand All @@ -260,18 +258,21 @@ export class CardanoNode {
* @returns {Promise<void>} resolves if the node could be killed, rejects with error otherwise.
*/
kill(): Promise<void> {
const { _node, _log, _config } = this;
const { _node, _log } = this;
return new Promise(async (resolve, reject) => {
if (!_node || await this._isNodeProcessNotRunningAnymore()) return resolve();
if (await this._isDead()) {
_log.info('CardanoNode#kill: process is already dead.');
return Promise.resolve();
}
try {
_log.info('CardanoNode: killing cardano-node process.');
_node.kill();
await this._waitForNodeProcessToExit(_config.killTimeout);
_log.info('CardanoNode#kill: killing cardano-node process.');
if (_node) _node.kill();
await this._waitForCardanoToExitOrKillIt();
await this._storeProcessStates();
this._reset();
resolve();
} catch (_) {
_log.info('CardanoNode: could not kill cardano-node.');
_log.info('CardanoNode#kill: could not kill cardano-node.');
await this._storeProcessStates();
this._reset();
reject('Could not kill cardano-node.');
Expand All @@ -287,14 +288,18 @@ export class CardanoNode {
* @returns {Promise<void>} resolves if the node could be restarted, rejects with error otherwise.
*/
async restart(isForced: boolean = false): Promise<void> {
const { _log } = this;
const { _log, _config } = this;
try {
_log.info('CardanoNode#restart: stopping current node.');
await this.stop();
// Stop cardano nicely if it is still awake
if (await this._isConnected()) {
_log.info('CardanoNode#restart: stopping current node.');
await this.stop();
}
_log.info(`CardanoNode#restart: restarting node with previous config (isForced: ${isForced.toString()}).`);
await this.start(this._config, isForced);
await this._waitForCardanoToExitOrKillIt();
await this.start(_config, isForced);
} catch (error) {
_log.info(`CardanoNode: Could not restart cardano-node "${error}"`);
_log.info(`CardanoNode#restart: Could not restart cardano-node "${error}"`);
return Promise.reject(error);
}
}
Expand Down Expand Up @@ -384,32 +389,39 @@ export class CardanoNode {
};

_handleCardanoNodeExit = async (code: number, signal: string) => {
// console.log(`ON EXIT: ${JSON.stringify(this._tlsConfig.ca)}`);
const { _log, _config } = this;
_log.info(`CardanoNode: cardano-node exited with: ${code}, ${signal}`);
const { _log, _config, _node } = this;
_log.info(`CardanoNode: says it exited with [${code}, ${signal}]`);
// We don't know yet what happened but we can be sure cardano-node is exiting
if (this._state === CardanoNodeStates.RUNNING) {
this._changeToState(CardanoNodeStates.EXITING);
}
try {
// Before proceeding with exit procedures, wait until the node is really dead.
await this._waitForNodeProcessToExit(_config.shutdownTimeout);
} catch (_) {
_log.error(`CardanoNode: sent exit code ${code} but was still running after ${_config.shutdownTimeout}ms. Killing it now.`);
try {
if (_node) await this._ensureProcessIsNotRunning(_node.pid, CARDANO_PROCESS_NAME);
} catch (e) {
_log.info('CardanoNode: did not exit correctly.');
}
}
_log.info(`CardanoNode: process really exited with [${code}, ${signal}]}`);
// Handle various exit scenarios
if (this._state === CardanoNodeStates.STOPPING) {
this._changeToState(CardanoNodeStates.STOPPED);
} else if (this._state === CardanoNodeStates.UPDATING && code === CARDANO_UPDATE_EXIT_CODE) {
try {
await this._waitForNodeProcessToExit(_config.shutdownTimeout);
this._changeToState(CardanoNodeStates.UPDATED);
} catch (error) {
_log.error('CardanoNode: cardano-node process did not exit as expected during update.');
}
} else if (this._state !== CardanoNodeStates.UPDATED) {
this._changeToState(CardanoNodeStates.UPDATED);
} else {
this._changeToState(CardanoNodeStates.CRASHED, code, signal);
}
this._reset();
};

_resetTlsConfig = () => this._tlsConfig = null;

_reset = () => {
if (this._cardanoLogFile) this._cardanoLogFile.end();
if (this._node) {
this._node.removeAllListeners();
}
this._resetTlsConfig();
if (this._node) this._node.removeAllListeners();
this._tlsConfig = null;
};

_changeToState(state: CardanoNodeState, ...args: Array<any>) {
Expand All @@ -430,16 +442,17 @@ export class CardanoNode {
}

/**
* Checks if cardano-node child_process has been created, and is connected, and is stateful
* Checks if cardano-node child_process is connected and can be interacted with
* @returns {boolean}
*/
_isAwake = (): boolean => (
this._node != null && this._node.connected && (
this._state === CardanoNodeStates.STARTING ||
this._state === CardanoNodeStates.RUNNING ||
this._state === CardanoNodeStates.STOPPING ||
this._state === CardanoNodeStates.UPDATING
)
_isConnected = (): boolean => this._node != null && this._node.connected;

/**
* Checks if cardano-node child_process is not running anymore
* @returns {boolean}
*/
_isDead = async (): Promise<boolean> => (
!this._isConnected() && await this._isNodeProcessNotRunningAnymore()
);

/**
Expand All @@ -451,51 +464,56 @@ export class CardanoNode {
* @private
*/
_canBeStarted = async (): Promise<boolean> => {
if (this._isAwake()) { return false; }
await this._ensurePreviousCardanoNodeIsNotRunning();
return true;
if (this._isConnected()) { return false; }
try {
await this._ensurePreviousCardanoNodeIsNotRunning();
return true;
} catch (error) {
return false;
}
};

_ensurePreviousCardanoNodeIsNotRunning = async (): Promise<void> => {
this._log.info(
'CardanoNode: checking for previous instance of cardano-node still running on last known PID'
);
const previousPID: ?number = await this._retrieveData(PREVIOUS_CARDANO_PID);

if (previousPID == null) { return; }

const processIsCardanoNode = (
previousPID ? await this._isProcessRunning(previousPID, CARDANO_PROCESS_NAME) : false
);

if (processIsCardanoNode) {
this._log.info(`CardanoNode: attempting to kill previous cardano-node process with PID: ${previousPID}`);
// kill previous process
await this._killProcessWithName(previousPID, CARDANO_PROCESS_NAME);
return;
_ensureProcessIsNotRunning = async (pid: number, name: string) => {
const { _log } = this;
_log.info(`CardanoNode: checking if ${name} process (PID: ${pid}) is still running`);
if (await this._isProcessRunning(pid, name)) {
_log.info(`CardanoNode: killing ${name} process (PID: ${pid})`);
try {
await this._killProcessWithName(pid, name);
return Promise.resolve();
} catch (error) {
_log.info(`CardanoNode: could not kill ${name} process (PID: ${pid})`);
return Promise.reject();
}
}
this._log.info(`No ${name} process (PID: ${pid}) is running.`);
};

this._log.info('Previous instance of cardano-node does not exist');
_ensureCurrentCardanoNodeIsNotRunning = async (): Promise<void> => {
const { _log, _node } = this;
_log.info('CardanoNode: checking if current cardano-node process is still running');
if (_node == null) { return Promise.resolve(); }
return await this._ensureProcessIsNotRunning(_node.pid, CARDANO_PROCESS_NAME);
};

_ensurePreviousCardanoNodeIsNotRunning = async (): Promise<void> => {
const { _log } = this;
_log.info('CardanoNode: checking if previous cardano-node process is still running');
const previousPID: ?number = await this._retrieveData(PREVIOUS_CARDANO_PID);
if (previousPID == null) { return Promise.resolve(); }
return await this._ensureProcessIsNotRunning(previousPID, CARDANO_PROCESS_NAME);
};

_isProcessRunning = async (previousPID: number, processName: string): Promise<boolean> => {
const { _log } = this;
try {
// retrieves all running processes
const runningProcesses: Array<Process> = await psList();
// filters running processes against previous PID
const matchingProcesses: Array<Process> =
runningProcesses.filter(({ pid }) => previousPID === pid);
// return false if no processes exist with a matching PID
if (!matchingProcesses.length) {
const previousProcess = await getProcess(previousPID, processName);
if (!previousProcess) {
_log.debug(`CardanoNode: No previous ${processName} process is running anymore.`);
return false;
}
// pull first result
const previousProcess: Process = matchingProcesses[0];
// check name of process to identify cardano-node or Daedalus
_log.debug(`CardanoNode: previous ${processName} process found: ${JSON.stringify(previousProcess)} - expecting to match against process name: ${processName} (${(previousProcess.name === processName) ? 'MATCH' : 'NO-MATCH'})`);
return (isObject(previousProcess) && previousProcess.name === processName);
_log.debug(`CardanoNode: previous ${processName} process found: ${JSON.stringify(previousProcess)}`);
return true;
} catch (error) {
return false;
}
Expand All @@ -507,12 +525,14 @@ export class CardanoNode {
try {
process.kill(pid);
await promisedCondition(() => !this._isProcessRunning(pid, name), _config.killTimeout);
this._log.info(`CardanoNode: successfuly killed process with pid ${pid}`);
this._log.info(`CardanoNode: successfuly killed ${name} process (PID: ${pid})`);
return Promise.resolve();
} catch (error) {
this._log.info(
`CardanoNode: _killPreviousProcess returned an error attempting to kill
process with pid ${pid}. Error: ${JSON.stringify(error)}`
`CardanoNode: _killPreviousProcess returned an error attempting to kill ${name}
process (PID: ${pid}). Error: ${JSON.stringify(error)}`
);
return Promise.reject(error);
}
};

Expand Down Expand Up @@ -561,14 +581,24 @@ export class CardanoNode {
})
);

_isNodeProcessStillRunning = async () => (
this._node && await this._isProcessRunning(this._node.pid, CARDANO_PROCESS_NAME)
_isNodeProcessStillRunning = async (): Promise<boolean> => (
this._node != null && await this._isProcessRunning(this._node.pid, CARDANO_PROCESS_NAME)
);

_isNodeProcessNotRunningAnymore = async () => await this._isNodeProcessStillRunning() === false
_isNodeProcessNotRunningAnymore = async () => await this._isNodeProcessStillRunning() === false;

_waitForNodeProcessToExit = async (timeout: number) => (
await promisedCondition(this._isNodeProcessNotRunningAnymore, timeout)
);

_waitForCardanoToExitOrKillIt = async () => {
const { _config } = this;
if (this._isNodeProcessNotRunningAnymore()) return Promise.resolve();
try {
await this._waitForNodeProcessToExit(_config.shutdownTimeout);
} catch (_) {
await this._ensureCurrentCardanoNodeIsNotRunning();
}
}

}
Loading