Skip to content
This repository was archived by the owner on Sep 18, 2024. It is now read-only.

add reusable for win remote #3500

Merged
merged 8 commits into from
Apr 7, 2021
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions ts/nni_manager/training_service/common/containerJobData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,10 @@ else
# Install nni
python3 -m pip install --user --upgrade nni
fi`;

export const CONTAINER_INSTALL_NNI_SHELL_FORMAT_FOR_WIN: string =
`python -c "import nni" 2>$error
if ($error -ne ''){
python -m pip install --user --upgrade nni
}
exit`;
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ class ShellExecutor {
private readonly sshClient: Client;
private readonly log: Logger;
private tempPath: string = "";
private isWindows: boolean = false;
private channelDefaultOutputs: string[] = [];
private pythonPath: string | undefined;

public isWindows: boolean = false;

constructor() {
this.log = getLogger();
this.sshClient = new Client();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { getExperimentId } from '../../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../../common/log';
import { EnvironmentInformation, EnvironmentService } from '../environment';
import {
getExperimentRootDir,
getExperimentRootDir, getLogLevel,
} from '../../../common/utils';
import { TrialConfig } from '../../common/trialConfig';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
Expand Down Expand Up @@ -218,6 +218,27 @@ export class RemoteEnvironmentService extends EnvironmentService {
this.remoteMachineMetaOccupiedMap.set(remoteEnvironment.rmMachineMeta, false);
}

private async getScript(environment: EnvironmentInformation): Promise<string> {
const executor = await this.getExecutor(environment.id);
const logLevel = getLogLevel();
let script: string = environment.command;

if (executor.isWindows) {
const prepare = `mkdir envs\\${environment.id} && cd envs\\${environment.id}`;
const startrun = `powershell ..\\install_nni.ps1 && python -m nni.tools.trial_tool.trial_runner`;
const developingScript = "IF EXIST nni_trial_tool (ECHO \"nni_trial_tool exists already\") ELSE (mkdir nni_trial_tool && tar -xof ../nni_trial_tool.tar.gz -C ./nni_trial_tool) && pip3 install websockets";

script = logLevel == "debug" ? `${prepare} && ${developingScript} && ${startrun}` : `${prepare} && ${startrun}`;
}

script = `cd ${environment.runnerWorkingFolder} && \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this script value will override line 221.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It only works for windows "isdebug" mode or not, Default(Linux) is not affected.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FIxed.

${script} --job_pid_file ${environment.runnerWorkingFolder}/pid \
1>${environment.runnerWorkingFolder}/trialrunner_stdout 2>${environment.runnerWorkingFolder}/trialrunner_stderr \
&& echo $? \`date +%s%3N\` >${environment.runnerWorkingFolder}/code`;

return script;
}

public async startEnvironment(environment: EnvironmentInformation): Promise<void> {
if (this.sshConnectionPromises.length > 0) {
await Promise.all(this.sshConnectionPromises);
Expand Down Expand Up @@ -268,11 +289,8 @@ export class RemoteEnvironmentService extends EnvironmentService {
} else {
this.remoteExperimentRootDir = executor.getRemoteExperimentRootDir(getExperimentId());
}
environment.runnerWorkingFolder = executor.joinPath(this.remoteExperimentRootDir, 'envs', environment.id);
environment.command = `cd ${this.remoteExperimentRootDir} && \
${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
1>${environment.runnerWorkingFolder}/trialrunner_stdout 2>${environment.runnerWorkingFolder}/trialrunner_stderr \
&& echo $? \`date +%s%3N\` >${environment.runnerWorkingFolder}/code`;

environment.command = await this.getScript(environment);
return Promise.resolve(true);
}
}
Expand Down
6 changes: 4 additions & 2 deletions ts/nni_manager/training_service/reusable/trialDispatcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { delay, getExperimentRootDir, getIPV4Address, getLogLevel, getVersion, m
import { GPU_INFO, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, REPORT_METRIC_DATA, SEND_TRIAL_JOB_PARAMETER, STDOUT, TRIAL_END, VERSION_CHECK } from '../../core/commands';
import { ScheduleResultType } from '../../training_service/common/gpuData';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT_FOR_WIN } from '../common/containerJobData';
import { TrialConfig } from '../common/trialConfig';
import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey';
import { validateCodeDir } from '../common/util';
Expand All @@ -32,7 +33,6 @@ import { NFSSharedStorageService } from './shared_storages/nfsStorageService'
import { AzureBlobSharedStorageService } from './shared_storages/azureblobStorageService'
import { TrialDetail } from './trial';


/**
* It uses to manage jobs on training platforms
* and expose trial as trial job to upper level.
Expand Down Expand Up @@ -225,8 +225,10 @@ class TrialDispatcher implements TrainingService {
const codeFileName = await storageService.copyDirectory(codeDir, envDir, true);
storageService.rename(codeFileName, "nni-code.tar.gz");

const installFileName = storageService.joinPath(envDir, 'install_nni.sh');
const installFileName = storageService.joinPath(envDir, `install_nni.sh`);
const installFileNameForWin = storageService.joinPath(envDir, `install_nni.ps1`);
await storageService.save(CONTAINER_INSTALL_NNI_SHELL_FORMAT, installFileName);
await storageService.save(CONTAINER_INSTALL_NNI_SHELL_FORMAT_FOR_WIN, installFileNameForWin);

const runnerSettingsConfig = storageService.joinPath(envDir, "settings.json");
await storageService.save(JSON.stringify(runnerSettings), runnerSettingsConfig);
Expand Down