Skip to content

Commit

Permalink
feat: workers remove (#480)
Browse files Browse the repository at this point in the history
* feat: workers remove

* up

* one test

* Apply automatic changes

* fix

* Apply automatic changes

* fix

* fix

* up

* fix

* up

* fix

* check

* check

* fix

* Apply automatic changes

* areAllWorkersRemoved function

* Apply automatic changes

* fix

* fix

* up

* up

* fix

* fix

* fix

* export

* fix

* fix

* all tests

* fix: npm not being found when installed with yarn [fixes DXJ-479] (#484)

* fix: npm not being found when installed with yarn [fixes DXJ-479]

* display both errors

* improve error

* check local npm exists

* improve formatting

* improve error message

* chore(main): release fluence-cli 0.8.8 (#495)

* chore(main): release fluence-cli 0.8.8

* Apply automatic changes

---------

Co-authored-by: fluencebot <[email protected]>
Co-authored-by: shamsartem <[email protected]>

* up

* improve

---------

Co-authored-by: shamsartem <[email protected]>
Co-authored-by: fluencebot <[email protected]>
Co-authored-by: fluencebot <[email protected]>
  • Loading branch information
4 people authored Sep 26, 2023
1 parent 8321f24 commit 71acd94
Show file tree
Hide file tree
Showing 8 changed files with 398 additions and 128 deletions.
35 changes: 35 additions & 0 deletions docs/commands/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
* [`fluence update [CHANNEL]`](#fluence-update-channel)
* [`fluence workers deploy [WORKER-NAMES]`](#fluence-workers-deploy-worker-names)
* [`fluence workers logs [WORKER-NAMES]`](#fluence-workers-logs-worker-names)
* [`fluence workers remove [WORKER-NAMES]`](#fluence-workers-remove-worker-names)
* [`fluence workers upload [WORKER-NAMES]`](#fluence-workers-upload-worker-names)

## `fluence air beautify [PATH]`
Expand Down Expand Up @@ -974,6 +975,40 @@ EXAMPLES

_See code: [dist/commands/workers/logs.ts](https://github.com/fluencelabs/cli/blob/v0.8.8/dist/commands/workers/logs.ts)_

## `fluence workers remove [WORKER-NAMES]`

Remove workers from hosts, described in 'hosts' property in workers.yaml

```
USAGE
$ fluence workers remove [WORKER-NAMES] [--no-input] [-k <value>] [--off-aqua-logs] [--priv-key <value>] [--relay
<value>] [--ttl <value>] [--dial-timeout <value>] [--particle-id] [--tracing]
ARGUMENTS
WORKER-NAMES Comma separated names of workers to remove. Example: "worker1,worker2" (by default all workers from
'hosts' property in workers.yaml are removed)
FLAGS
-k, --key-pair-name=<name> Key pair name
--dial-timeout=<milliseconds> [default: 60000] Timeout for Fluence js-client to connect to relay peer
--no-input Don't interactively ask for any input from the user
--off-aqua-logs Turns off logs from Console.print in aqua and from IPFS service
--particle-id Print particle ids when running Fluence js-client
--priv-key=<private-key> !WARNING! for debug purposes only. Passing private keys through flags is unsecure
--relay=<multiaddress> Relay for Fluence js-client to connect to
--tracing Compile aqua in tracing mode (for debugging purposes)
--ttl=<milliseconds> [default: 120000] Particle Time To Live since 'now'. After that, particle is expired
and not processed.
DESCRIPTION
Remove workers from hosts, described in 'hosts' property in workers.yaml
EXAMPLES
$ fluence workers remove
```

_See code: [dist/commands/workers/remove.ts](https://github.com/fluencelabs/cli/blob/v0.8.8/dist/commands/workers/remove.ts)_

## `fluence workers upload [WORKER-NAMES]`

Upload workers to hosts, described in 'hosts' property in fluence.yaml
Expand Down
24 changes: 12 additions & 12 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
"on-each-commit": "yarn check && yarn gen-config-docs && cd docs/commands && oclif readme --no-aliases",
"gen-config-docs": "shx rm -rf schemas && shx rm -rf docs/configs && ts-node ./src/genConfigDocs.ts",
"unused-exports": "ts-unused-exports tsconfig.json",
"upgrade-dependencies": "npm-check-updates -u",
"up-deps": "npm-check-updates -u",
"oclif-pack": "yarn clean && oclif pack tarballs -t 'linux-x64,darwin-x64,darwin-arm64' --no-xz",
"oclif-upload": "yarn oclif-pack && oclif upload tarballs -t 'linux-x64,darwin-x64,darwin-arm64'"
},
Expand All @@ -58,7 +58,7 @@
"@fluencelabs/fluence-network-environment": "1.1.2",
"@fluencelabs/js-client": "0.1.7",
"@iarna/toml": "^2.2.5",
"@mswjs/interceptors": "^0.25.1",
"@mswjs/interceptors": "^0.25.4",
"@multiformats/multiaddr": "^12.1.7",
"@oclif/color": "^1.0.10",
"@oclif/core": "^2.15.0",
Expand All @@ -75,7 +75,7 @@
"dotenv": "^16.3.1",
"ethers": "6.6.0",
"filenamify": "^6.0.0",
"get-random-values": "^2.1.0",
"get-random-values": "^3.0.0",
"inquirer": "^9.2.11",
"ipfs-http-client": "^60.0.1",
"lodash-es": "^4.17.21",
Expand All @@ -89,21 +89,21 @@
"yaml-diff-patch": "^2.0.0"
},
"devDependencies": {
"@fluencelabs/installation-spell": "0.5.21",
"@fluencelabs/installation-spell": "0.5.22",
"@total-typescript/ts-reset": "^0.5.1",
"@tsconfig/node18-strictest-esm": "^1.0.1",
"@types/debug": "^4.1.8",
"@types/decompress": "^4.2.4",
"@types/debug": "^4.1.9",
"@types/decompress": "^4.2.5",
"@types/iarna__toml": "^2.0.2",
"@types/inquirer": "^9.0.3",
"@types/jest": "^29.5.5",
"@types/lodash-es": "^4.17.9",
"@types/node": "^18.17.18",
"@types/node": "^18.17.19",
"@types/platform": "^1.3.4",
"@types/semver": "^7.5.2",
"@typescript-eslint/eslint-plugin": "^6.7.0",
"@typescript-eslint/parser": "^6.7.0",
"eslint": "^8.49.0",
"@typescript-eslint/eslint-plugin": "^6.7.2",
"@typescript-eslint/parser": "^6.7.2",
"eslint": "^8.50.0",
"eslint-config-prettier": "^9.0.0",
"eslint-plugin-import": "^2.28.1",
"eslint-plugin-license-header": "^0.6.0",
Expand All @@ -113,7 +113,7 @@
"jest": "^29.7.0",
"madge": "^6.1.0",
"npm-check-updates": "^16.14.4",
"oclif": "^3.16.0",
"oclif": "^3.17.1",
"prettier": "^3.0.3",
"shx": "^0.3.4",
"ts-jest": "^29.1.1",
Expand All @@ -122,7 +122,7 @@
"ts-unused-exports": "^10.0.1",
"tslib": "^2.6.2",
"typescript": "^5.2.2",
"undici": "^5.24.0"
"undici": "^5.25.2"
},
"oclif": {
"bin": "fluence",
Expand Down
2 changes: 1 addition & 1 deletion src/beforeBuild.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ const INSTALLATION_SPELL_AQUA_DIR_PATH = join(

const compileInstallationSpellAqua = async (tracing = false) => {
return Promise.all(
["upload", "cli", "deal_spell", "files"].map(async (fileName) => {
["upload", "cli", "deal_spell", "files", "deploy"].map(async (fileName) => {
const compilationResult = await compileFromPath({
filePath: join(
INSTALLATION_SPELL_AQUA_DIR_PATH,
Expand Down
163 changes: 163 additions & 0 deletions src/commands/workers/remove.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
/**
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { color } from "@oclif/color";
import { Args } from "@oclif/core";

import { BaseCommand, baseFlags } from "../../baseCommand.js";
import { commandObj } from "../../lib/commandObj.js";
import type { RemoveArgWorkers } from "../../lib/compiled-aqua/installation-spell/deploy.js";
import { initNewWorkersConfig } from "../../lib/configs/project/workers.js";
import {
KEY_PAIR_FLAG,
PRIV_KEY_FLAG,
OFF_AQUA_LOGS_FLAG,
FLUENCE_CONFIG_FULL_FILE_NAME,
FLUENCE_CLIENT_FLAGS,
TRACING_FLAG,
WORKERS_CONFIG_FULL_FILE_NAME,
} from "../../lib/const.js";
import { parseWorkers } from "../../lib/deployWorkers.js";
import {
disconnectFluenceClient,
initFluenceClient,
} from "../../lib/jsClient.js";
import { initCli } from "../../lib/lifeCycle.js";

export default class Remove extends BaseCommand<typeof Remove> {
static override description = `Remove workers from hosts, described in 'hosts' property in ${WORKERS_CONFIG_FULL_FILE_NAME}`;
static override examples = ["<%= config.bin %> <%= command.id %>"];
static override flags = {
...baseFlags,
...KEY_PAIR_FLAG,
...OFF_AQUA_LOGS_FLAG,
...PRIV_KEY_FLAG,
...FLUENCE_CLIENT_FLAGS,
...TRACING_FLAG,
};
static override args = {
"WORKER-NAMES": Args.string({
description: `Comma separated names of workers to remove. Example: "worker1,worker2" (by default all workers from 'hosts' property in ${WORKERS_CONFIG_FULL_FILE_NAME} are removed)`,
}),
};
async run(): Promise<void> {
const { flags, fluenceConfig, args } = await initCli(
this,
await this.parse(Remove),
true,
);

const workersConfig = await initNewWorkersConfig();

const { ensureAquaFileWithWorkerInfo } = await import(
"../../lib/deployWorkers.js"
);

await initFluenceClient(flags, fluenceConfig);
const { Fluence } = await import("@fluencelabs/js-client");
const relayId = (await Fluence.getClient()).getRelayPeerId();

if (workersConfig.hosts === undefined) {
return commandObj.error(
`There are no workers in ${FLUENCE_CONFIG_FULL_FILE_NAME}`,
);
}

const workersToRemove =
args["WORKER-NAMES"] === undefined
? Object.keys(workersConfig.hosts)
: parseWorkers(args["WORKER-NAMES"]);

const removeArg: RemoveArgWorkers = {
workers: Object.entries(workersConfig.hosts)
.filter(([workerName]) => {
return workersToRemove.includes(workerName);
})
.map(([name, worker]): RemoveArgWorkers["workers"][number] => {
return {
definition: worker.definition,
name,
dummy_deal_id: worker.dummyDealId,
installation_spells: worker.installation_spells,
};
}),
};

const removeResult = await remove(flags.tracing, removeArg);

const newHosts = Object.fromEntries(
Object.entries(workersConfig.hosts)
.map(([name, { installation_spells: prevInstSp, ...rest }]) => {
const currentWorkerResult = removeResult.find((r) => {
return r.name === name;
});

const removedWorkerIds = currentWorkerResult?.worker_ids ?? [];

const notRemovedWorkers = prevInstSp.filter(({ worker_id }) => {
const workerRemovedSuccessfully =
removedWorkerIds.includes(worker_id);

return !workerRemovedSuccessfully;
});

return [
name,
{ installation_spells: notRemovedWorkers, ...rest },
] as const;
})
.filter(([, { installation_spells }]) => {
return installation_spells.length > 0;
}),
);

workersConfig.hosts = newHosts;

if (Object.keys(workersConfig.hosts).length === 0) {
delete workersConfig.hosts;
}

await workersConfig.$commit();
await ensureAquaFileWithWorkerInfo(workersConfig, fluenceConfig);
const { yamlDiffPatch } = await import("yaml-diff-patch");

commandObj.log(
`\n\n${color.yellow("Success!")}\n\nrelay: ${relayId}\n\n${yamlDiffPatch(
"",
{},
{ removed: removeResult },
)}`,
);

await disconnectFluenceClient();
}
}

async function remove(tracing: boolean, removeArg: RemoveArgWorkers) {
if (tracing) {
const { remove } = await import(
"../../lib/compiled-aqua-with-tracing/installation-spell/deploy.js"
);

return remove(removeArg);
}

const { remove } = await import(
"../../lib/compiled-aqua/installation-spell/deploy.js"
);

return remove(removeArg);
}
4 changes: 2 additions & 2 deletions src/lib/localServices/ipfs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ const upload = async (
content: Parameters<IPFSHTTPClient["add"]>[0],
log: (msg: unknown) => void,
) => {
const ipfsClient = await createIPFSClient(multiaddr);

try {
const ipfsClient = await createIPFSClient(multiaddr);

const { cid } = await ipfsClient.add(content, {
pin: true,
cidVersion: 1,
Expand Down
22 changes: 21 additions & 1 deletion test/aqua/runDeployedWorkers.aqua
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
aqua Main

import "@fluencelabs/aqua-lib/builtin.aqua"
import "@fluencelabs/aqua-lib/workers.aqua"

use "hosts.aqua"
import "services.aqua"

export runDeployedServices
export runDeployedServices, areAllWorkersRemoved

data Answer:
answer: string
Expand All @@ -21,3 +22,22 @@ func runDeployedServices() -> *Answer:
answers <<- Answer(answer=answer, peer=s.hostId)

<- answers

func areAllWorkersRemoved() -> bool:
hosts <- Hosts.get()
installationSpells = hosts.defaultWorker!.installationSpells
dummyDealId = hosts.defaultWorker!.dummyDealId

allWorkersRemoved: ?bool

for s <- installationSpells:
on s.hostId:
res <- DealWorker.get_worker_id(?[dummyDealId])

if res != nil:
allWorkersRemoved <<- false

if allWorkersRemoved == nil:
allWorkersRemoved <<- true

<- allWorkersRemoved!
Loading

0 comments on commit 71acd94

Please sign in to comment.