From 0029993a6d2061b4a6b45961f43868f7cd11dd98 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Thu, 28 Dec 2023 19:09:15 -0800 Subject: [PATCH 01/41] controller workflow APIs and simplified FedAvg and Fed Kaplan-Meier examples --- examples/hello-world/hello-fedavg/README.md | 175 +++++++++++ .../fedavg/app/config/config_fed_client.conf | 116 ++++++++ .../fedavg/app/config/config_fed_server.conf | 28 ++ .../jobs/fedavg/app/custom/cifar10.py | 137 +++++++++ .../jobs/fedavg/app/custom/fedavg.py | 219 ++++++++++++++ .../jobs/fedavg/app/custom/fedavg_pt.py | 42 +++ .../jobs/fedavg/app/custom/net.py | 37 +++ .../hello-fedavg/jobs/fedavg/meta.conf | 7 + .../hello-world/hello-fedavg/requirements.txt | 0 examples/hello-world/hello-km/README.md | 164 +++++++++++ examples/hello-world/hello-km/demo/km.ipynb | 101 +++++++ examples/hello-world/hello-km/demo/km.json | 172 +++++++++++ .../app/config/config_fed_client.conf | 116 ++++++++ .../app/config/config_fed_server.conf | 24 ++ .../kaplan-meier/app/custom/kaplan_meier.py | 110 +++++++ .../kaplan-meier/app/custom/km_analysis.py | 46 +++ .../jobs/kaplan-meier/app/custom/km_train.py | 71 +++++ .../hello-km/jobs/kaplan-meier/meta.conf | 7 + .../hello-km/km_survival_curve.png | Bin 0 -> 61673 bytes .../hello-world/hello-km/requirements.txt | 1 + nvflare/apis/dxo.py | 1 + nvflare/app_common/utils/fl_model_utils.py | 3 + nvflare/app_common/utils/math_utils.py | 89 ++++++ .../app_common/workflows/wf_comm/__init__.py | 13 + .../workflows/wf_comm/wf_comm_api.py | 144 +++++++++ .../workflows/wf_comm/wf_comm_api_spec.py | 62 ++++ .../app_common/workflows/wf_comm/wf_queue.py | 58 ++++ .../app_common/workflows/wf_comm/wf_spec.py | 21 ++ nvflare/app_common/workflows/wf_controller.py | 275 ++++++++++++++++++ nvflare/app_opt/pt/wf_controller.py | 32 ++ 30 files changed, 2271 insertions(+) create mode 100644 examples/hello-world/hello-fedavg/README.md create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf create mode 100644 examples/hello-world/hello-fedavg/requirements.txt create mode 100644 examples/hello-world/hello-km/README.md create mode 100644 examples/hello-world/hello-km/demo/km.ipynb create mode 100644 examples/hello-world/hello-km/demo/km.json create mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf create mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf create mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py create mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py create mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py create mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf create mode 100644 examples/hello-world/hello-km/km_survival_curve.png create mode 100644 examples/hello-world/hello-km/requirements.txt create mode 100644 nvflare/app_common/utils/math_utils.py create mode 100644 nvflare/app_common/workflows/wf_comm/__init__.py create mode 100644 nvflare/app_common/workflows/wf_comm/wf_comm_api.py create mode 100644 nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py create mode 100644 nvflare/app_common/workflows/wf_comm/wf_queue.py create mode 100644 nvflare/app_common/workflows/wf_comm/wf_spec.py create mode 100644 nvflare/app_common/workflows/wf_controller.py create mode 100644 nvflare/app_opt/pt/wf_controller.py diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md new file mode 100644 index 0000000000..d6309bcaa9 --- /dev/null +++ b/examples/hello-world/hello-fedavg/README.md @@ -0,0 +1,175 @@ +# FedAvg: simplified + +This example illustrates two features: +* How to use the new Flare Communicator API to contract a workflow: no need to write a controller. + +## FLARE Workflow Communicator API + +The Flare workflow Communicator API only has small set methods + +``` +class WFCommAPISpec(ABC): + @abstractmethod + def broadcast_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def broadcast(self, msg_payload): + pass + + @abstractmethod + def send(self, msg_payload: Dict): + pass + + @abstractmethod + def send_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def get_site_names(self): + pass + + @abstractmethod + def wait(self, min_responses): + pass +``` + + +## Writing a new Workflow + +With this new API writing the new workflow is really simple: + +* Workflow (Server) + +``` + +class FedAvg(WF): + def __init__(self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + early_stop_metrics: dict = None, + model_format: str = None + ): + super(FedAvg, self).__init__() + self.logger = logging.getLogger(self.__class__.__name__) + + + + # (1) init flare_comm + self.flare_comm = WFComm(result_check_interval=10) + self.flare_comm.init(self) + + + def run(self): + + self.logger.info("start Fed Avg Workflow\n \n") + + net = Net() + model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) + + start = self.start_round + end = self.start_round + self.num_rounds + + for current_round in range(start, end): + if self.should_early_stop(model.metrics, self.early_stop_metrics): + break + + self.current_round = current_round + + self.logger.info(f"Round {current_round}/{self.num_rounds} started.") + + sag_results = self.scatter_and_gather(model, current_round) + + aggr_result = self.aggr_fn(sag_results) + + self.logger.info(f"aggregate metrics = {aggr_result.metrics}") + + model = update_model(model, aggr_result) + + self.select_best_model(model) + + self.save_model(self.best_model, self.output_path) +``` +Scatter and Gather (SAG): + +SAG is simply ask WFController to broadcast the model to all clients + +``` + def scatter_and_gather(self, model: FLModel, current_round): + msg_payload = {"min_responses": self.min_clients, + "current_round": current_round, + "num_round": self.num_rounds, + "start_round": self.start_round, + "data": model} + + # (2) broadcast and wait + results = self.flare_comm.broadcast_and_wait(msg_payload) + return results +``` + +The base class ```WF``` is define as + +``` +class WF(ABC): + + @abstractmethod + def run(self): + raise NotImplemented +``` +is mainly make sure user define ```run()``` method + +## Configurations + +### client-side configuration + +This is the same as FLARE Client API configuration + +### server-side configuration + + Server side controller is really simple, all we need is to user WFController with newly defined workflow class +```KM``` + +``` +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "fed_avg" + path = "nvflare.app_common.workflows.wf_controller.WFController" + args { + task_name = "train" + wf_class_path = "fedavg_wf.FedAvg", + wf_args { + min_clients = 2 + num_rounds = 10 + output_path = "/tmp/nvflare/fedavg/mode.pth" + model_format = "torch" + early_stop_metrics { + accuracy = 55 + } + + } + } + } + ] + + components = [] + +} + +``` + + +## Run the job + +assume current working directory is at ```hello-fedavg``` directory + +``` +nvflare simulator job -w /tmp/nvflare/km/job -n 2 -t 2 +``` diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf new file mode 100644 index 0000000000..aaba629957 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf @@ -0,0 +1,116 @@ +{ + # version of the configuration + format_version = 2 + + # This is the application script which will be invoked. Client can replace this script with user's own training script. + app_script = "cifar10.py" + + # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. + app_config = "" + + # Client Computing Executors. + executors = [ + { + # tasks the executors are defined to handle + tasks = ["train"] + + # This particular executor + executor { + + # This is an executor for Client API. The underline data exchange is using Pipe. + path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" + + args { + # launcher_id is used to locate the Launcher object in "components" + launcher_id = "launcher" + + # pipe_id is used to locate the Pipe object in "components" + pipe_id = "pipe" + + # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. + # Please refer to the class docstring for all available arguments + heartbeat_timeout = 60 + + # format of the exchange parameters + params_exchange_format = "pytorch" + + # if the transfer_type is FULL, then it will be sent directly + # if the transfer_type is DIFF, then we will calculate the + # difference VS received parameters and send the difference + params_transfer_type = "DIFF" + + # if train_with_evaluation is true, the executor will expect + # the custom code need to send back both the trained parameters and the evaluation metric + # otherwise only trained parameters are expected + train_with_evaluation = true + } + } + } + ], + + # this defined an array of task data filters. If provided, it will control the data from server controller to client executor + task_data_filters = [] + + # this defined an array of task result filters. If provided, it will control the result from client executor to server controller + task_result_filters = [] + + components = [ + { + # component id is "launcher" + id = "launcher" + + # the class path of this component + path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" + + args { + # the launcher will invoke the script + script = "python3 custom/{app_script} {app_config} " + # if launch_once is true, the SubprocessLauncher will launch once for the whole job + # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server + launch_once = true + } + } + { + id = "pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + } + { + id = "metrics_pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + }, + { + id = "metric_relay" + path = "nvflare.app_common.widgets.metric_relay.MetricRelay" + args { + pipe_id = "metrics_pipe" + event_type = "fed.analytix_log_stats" + # how fast should it read from the peer + read_interval = 0.1 + } + }, + { + # we use this component so the client api `flare.init()` can get required information + id = "config_preparer" + path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" + args { + component_ids = ["metric_relay"] + } + } + ] +} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf new file mode 100644 index 0000000000..65a23423b3 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf @@ -0,0 +1,28 @@ +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "fed_avg" + path = "nvflare.app_opt.pt.wf_controller.PTWFController" + args { + comm_msg_pull_interval = 5 + task_name = "train" + wf_class_path = "fedavg_pt.PTFedAvg", + wf_args { + min_clients = 2 + num_rounds = 10 + output_path = "/tmp/nvflare/fedavg/mode.pth" + stop_cond = "accuracy >= 55" + model_selection_rule = "accuracy >=" + } + } + } + ] + + components = [] + +} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py new file mode 100644 index 0000000000..9e8cfd1c39 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py @@ -0,0 +1,137 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.optim as optim +import torchvision +import torchvision.transforms as transforms +from net import Net + +# (1) import nvflare client API +import nvflare.client as flare + +# (optional) metrics +from nvflare.client.tracking import SummaryWriter + +# (optional) set a fix place so we don't need to download everytime +DATASET_PATH = "/tmp/nvflare/data" +# (optional) We change to use GPU to speed things up. +# if you want to use CPU, change DEVICE="cpu" +DEVICE = "cuda:0" + + +def main(): + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + + batch_size = 4 + epochs = 2 + + trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) + trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) + + testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) + testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) + + net = Net() + + # (2) initializes NVFlare client API + flare.init() + + summary_writer = SummaryWriter() + while flare.is_running(): + # (3) receives FLModel from NVFlare + input_model = flare.receive() + print(f"current_round={input_model.current_round}") + + # (4) loads model from NVFlare + net.load_state_dict(input_model.params) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) + + # (optional) use GPU to speed things up + net.to(DEVICE) + # (optional) calculate total steps + steps = epochs * len(trainloader) + for epoch in range(epochs): # loop over the dataset multiple times + + running_loss = 0.0 + for i, data in enumerate(trainloader, 0): + # get the inputs; data is a list of [inputs, labels] + # (optional) use GPU to speed things up + inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) + + # zero the parameter gradients + optimizer.zero_grad() + + # forward + backward + optimize + outputs = net(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # print statistics + running_loss += loss.item() + if i % 2000 == 1999: # print every 2000 mini-batches + print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") + global_step = input_model.current_round * steps + epoch * len(trainloader) + i + + summary_writer.add_scalar(tag="loss_for_each_batch", scalar=running_loss, global_step=global_step) + running_loss = 0.0 + + print("Finished Training") + + PATH = "./cifar_net.pth" + torch.save(net.state_dict(), PATH) + + # (5) wraps evaluation logic into a method to re-use for + # evaluation on both trained and received model + def evaluate(input_weights): + net = Net() + net.load_state_dict(input_weights) + # (optional) use GPU to speed things up + net.to(DEVICE) + + correct = 0 + total = 0 + # since we're not training, we don't need to calculate the gradients for our outputs + with torch.no_grad(): + for data in testloader: + # (optional) use GPU to speed things up + images, labels = data[0].to(DEVICE), data[1].to(DEVICE) + # calculate outputs by running images through the network + outputs = net(images) + # the class with the highest energy is what we choose as prediction + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") + return 100 * correct // total + + # (6) evaluate on received model for model selection + accuracy = evaluate(input_model.params) + # (7) construct trained FL model + output_model = flare.FLModel( + params=net.cpu().state_dict(), + metrics={"accuracy": accuracy}, + meta={"NUM_STEPS_CURRENT_ROUND": steps}, + ) + # (8) send model back to NVFlare + flare.send(output_model) + + +if __name__ == "__main__": + main() diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py new file mode 100644 index 0000000000..8f65913c4c --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -0,0 +1,219 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import traceback +from typing import Callable, Dict, Optional + +from net import Net + +from nvflare.app_common.abstract.fl_model import FLModel, ParamsType +from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper +from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.utils.math_utils import parse_compare_criteria, parse_compare_operator +from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( + CURRENT_ROUND, + DATA, + MIN_RESPONSES, + NUM_ROUNDS, + START_ROUND, +) +from nvflare.app_common.workflows.wf_comm.wf_spec import WF +from nvflare.security.logging import secure_format_exception + +update_model = FLModelUtils.update_model + + +# FedAvg Workflow + + +class FedAvg(WF): + def __init__( + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + model_selection_rule: str = None, + ): + super(FedAvg, self).__init__() + self.logger = logging.getLogger(self.__class__.__name__) + + self.output_path = output_path + self.min_clients = min_clients + self.num_rounds = num_rounds + self.start_round = start_round + self.current_round = start_round + self.best_model: Optional[FLModel] = None + if stop_cond: + self.stop_criteria = parse_compare_criteria(stop_cond) + else: + self.stop_criteria = None + + if model_selection_rule: + self.metric_comp_rule = parse_compare_operator(model_selection_rule) + else: + self.metric_comp_rule = None + + # (1) init flare_comm + self.flare_comm = WFCommAPI() + + def run(self): + try: + self.logger.info("start Fed Avg Workflow\n \n") + + start = self.start_round + end = self.start_round + self.num_rounds + + model = self.init_model() + for current_round in range(start, end): + + self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") + self.current_round = current_round + + if self.should_stop(model.metrics, self.stop_criteria): + self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") + break + + sag_results = self.scatter_and_gather(model, current_round) + + aggr_result = self.aggr_fn(sag_results) + + self.logger.info(f"aggregate metrics = {aggr_result.metrics}") + + model = update_model(model, aggr_result) + + self.select_best_model(model) + + self.save_model(self.best_model, self.output_path) + + self.logger.info("end Fed Avg Workflow\n \n") + except Exception as e: + print(f"\n\n =============== {traceback.format_exc()} ") + raise e + + def init_model(self): + net = Net() + model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) + return model + + def scatter_and_gather(self, model: FLModel, current_round): + msg_payload = { + MIN_RESPONSES: self.min_clients, + CURRENT_ROUND: current_round, + NUM_ROUNDS: self.num_rounds, + START_ROUND: self.start_round, + DATA: model, + } + + # (2) broadcast and wait + results = self.flare_comm.broadcast_and_wait(msg_payload) + return results + + def aggr_fn(self, sag_result: Dict[str, Dict[str, FLModel]]) -> FLModel: + + self.logger.info("fed avg aggregate \n") + + if not sag_result: + raise RuntimeError("input is None or empty") + + task_name, task_result = next(iter(sag_result.items())) + + if not task_result: + raise RuntimeError("task_result is None or empty ") + + self.logger.info(f"aggregating {len(task_result)} update(s) at round {self.current_round}") + + try: + aggr_params_helper = WeightedAggregationHelper() + aggr_metrics_helper = WeightedAggregationHelper() + params_type = None + for site, fl_model in task_result.items(): + if params_type is None: + params_type = fl_model.params_type + + aggr_params_helper.add( + data=fl_model.params, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) + + self.logger.info(f"site={site} {fl_model.metrics=}") + + aggr_metrics_helper.add( + data=fl_model.metrics, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) + + aggr_params = aggr_params_helper.get_result() + aggr_metrics = aggr_metrics_helper.get_result() + + self.logger.info(f"{aggr_metrics=}") + + aggr_result = FLModel( + params=aggr_params, + params_type=params_type, + metrics=aggr_metrics, + meta={"num_rounds_aggregated": len(task_result), "current_round": self.current_round}, + ) + return aggr_result + except Exception as e: + traceback_str = traceback.format_exc() + raise RuntimeError(f"Exception in aggregate call: {secure_format_exception(e, traceback_str)}") + + def select_best_model(self, curr_model: FLModel): + if self.best_model is None: + self.best_model = curr_model + return + + if self.metric_comp_rule is None: + return + metric, op_fn = self.metric_comp_rule + + self.logger.info("compare models") + if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): + self.best_model = curr_model + + def save_model(self, model: FLModel, file_path: str): + pass + + def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[str] = None): + self.logger.info(f"stop_criteria, metrics = {stop_criteria=}, {metrics=}") + if stop_criteria is None or metrics is None: + return False + + key, target, op_fn = stop_criteria + value = metrics.get(key, None) + + if value is None: + raise RuntimeError(f"stop criteria key '{key}' doesn't exists in metrics") + + return op_fn(value, target) + + def is_curr_mode_better( + self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable + ) -> bool: + curr_metrics = curr_model.metrics + if curr_metrics is None: + return False + if target_metric not in curr_metrics: + return False + + best_metrics = best_model.metrics + return op_fn(curr_metrics.get(target_metric), best_metrics.get(target_metric)) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py new file mode 100644 index 0000000000..beea8e7e68 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import torch +from fedavg import FedAvg + +from nvflare.app_common.abstract.fl_model import FLModel + + +class PTFedAvg(FedAvg): + def __init__( + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + model_selection_rule: str = None, + ): + super().__init__(min_clients, num_rounds, output_path, start_round, stop_cond, model_selection_rule) + + def save_model(self, model: FLModel, file_path: str): + if not file_path: + raise ValueError("invalid file path") + + dir_name = os.path.dirname(file_path) + os.makedirs(dir_name, exist_ok=True) + + self.logger.info(f"save best model to {file_path} \n") + torch.save(model.params, file_path) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py new file mode 100644 index 0000000000..031f84f432 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Net(nn.Module): + def __init__(self): + super().__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = torch.flatten(x, 1) # flatten all dimensions except batch + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf new file mode 100644 index 0000000000..1c27c4e99c --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf @@ -0,0 +1,7 @@ +{ + name = "fedavg" + deploy_map { + app = ["@ALL"] + } + min_clients = 2 +} diff --git a/examples/hello-world/hello-fedavg/requirements.txt b/examples/hello-world/hello-fedavg/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/hello-world/hello-km/README.md b/examples/hello-world/hello-km/README.md new file mode 100644 index 0000000000..b2c5b94415 --- /dev/null +++ b/examples/hello-world/hello-km/README.md @@ -0,0 +1,164 @@ +# Kaplan-Meier Analysis + +This example illustrates two features: +* How to perform Kaplan-Meirer Survival Analysis in federated setting +* How to use the new Flare Communicator API to contract a workflow: no need to write a controller. + +## FLARE Workflow Communicator API + +The Flare workflow Communicator API only has small set methods + +``` +class WFCommAPISpec(ABC): + @abstractmethod + def broadcast_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def broadcast(self, msg_payload): + pass + + @abstractmethod + def send(self, msg_payload: Dict): + pass + + @abstractmethod + def send_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def get_site_names(self): + pass + + @abstractmethod + def wait(self, min_responses): + pass +``` + + +## Writing a new Workflow + +With this new API writing the new workflow is really simple: + +For example for Kaplan-Meier Analysis, we could write a new workflow like this: + +``` + +class KM(WF): + def __init__(self, min_clients: int, output_path: str): + super(KM, self).__init__() + self.logger = logging.getLogger(self.__class__.__name__) + self.output_path = output_path + self.min_clients = min_clients + self.num_rounds = 1 + self.flare_comm = WFCommAPI() + + def run(self): + results = self.start_km_analysis() + global_res = self.aggr_km_result(results) + self.save(global_res, self.output_path) + +``` + +The base class ```WF``` is define as + +``` +class WF(ABC): + + @abstractmethod + def run(self): + raise NotImplemented +``` +is mainly make sure user define ```run()``` method + +for Kaplan-Meier analysis, it literal involves + +* start the analysis --> ask all clients to perform local KM analysis, then wait for results +* then aggregate the result to obtain gloabl results +* save the result + +We only need to one_round trip from server --> client, client --> server + +Let's define the start_km_analysis() + +``` + + def start_km_analysis(self): + self.logger.info("send kaplan-meier analysis command to all sites \n") + + msg_payload = { + MIN_RESPONSES: self.min_clients, + CURRENT_ROUND: 1, + NUM_ROUNDS: self.num_rounds, + START_ROUND: 1, + DATA: {}, + } + + results = self.flare_comm.broadcast_and_wait(msg_payload) + return results + +``` + +looks like to simply call send broadcast command, then just get the results. + +## Configurations + +### client-side configuration + +This is the same as FLARE Client API configuration + +### server-side configuration + + Server side controller is really simple, all we need is to user WFController with newly defined workflow class +```KM``` + +``` +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "km" + path = "nvflare.app_common.workflows.wf_controller.WFController" + args { + task_name = "train" + wf_class_path = "km_wf.KM", + wf_args { + min_clients = 2 + output_path = "/tmp/nvflare/km/km.json" + } + } + } + ] + + components = [] + +} + +``` + + +## Run the job + +assume current working directory is at ```hello-km``` directory + +``` +nvflare simulator job -w /tmp/nvflare/km/job -n 2 -t 2 +``` + + +## Display Result + +Once finished the results will be written to the output_path defined about. +We can copy the result to the demo directory and start notebook + +``` +cp /tmp/nvflare/km/km.json demo/. + +jupyter lab demo/km.ipynb + +``` +![KM survival curl](km_survival_curve.png) diff --git a/examples/hello-world/hello-km/demo/km.ipynb b/examples/hello-world/hello-km/demo/km.ipynb new file mode 100644 index 0000000000..a76483151b --- /dev/null +++ b/examples/hello-world/hello-km/demo/km.ipynb @@ -0,0 +1,101 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "16c35505-cb9e-4517-bd85-6262cf881b3a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "41bd1b35-a8bb-4fe4-9619-c369b2201bcb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "with open(\"km.json\", 'r') as json_file:\n", + " data = json.load(json_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e81f00ed-24c5-434e-942d-9b91a53eb232", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABm9UlEQVR4nO3deVwU9f8H8NdwLsutXIoInqmFiBeiKSQoeJBHKiIpoFmWmkaWV4rmlaampWlqlpUomkeWeRDhkfeBWmqeGJYcInLItRzz+4Mv+3PlcBcXBpfX8/HYx2PnM5+Zz3uGhX3z+XxmRhBFUQQRERGRjtCTOgAiIiIibWJyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckMkoTt37kAQBHz77bdSh1Itvv32WwiCgDt37kgdimRCQ0Ph4uJSrW0IgoA5c+ZUaxtEzxMmN1TnlX4Bnz17VqU8IyMDnTt3hkwmw/79+yWKrnoJggBBEPDGG2+Uu37mzJnKOqmpqTUcneb+/PNPDBkyBM7OzpDJZHB0dESvXr3wxRdfSB1arZGZmYm5c+fCzc0NZmZmMDExwUsvvYSpU6fi3r17UodHpBVMbojKkZmZid69e+PSpUvYtWsX/P39pQ6p2shkMuzYsQMKhaLMui1btkAmk1V53yNHjkRubi6cnZ2fJUS1HD9+HB07dsTFixcxduxYrFq1Cm+88Qb09PSwcuXKam+/IuvXr8e1a9cka/9xt2/fRrt27TBv3jy0adMGixcvxueff45XXnkFX3/9Nby9vaUOkUgrDKQOgKi2ycrKgp+fHy5cuICdO3eiT58+UodUrfz9/bFnzx7s27cPAwYMUJYfP34c8fHxeO2117Bjx44q7VtfXx/6+vraChXZ2dkwNTUtd92CBQtgaWmJM2fOwMrKSmVdSkpKjcRQHkNDQ621/SwKCwsxePBgJCcn49ChQ3j55ZdV1i9YsACLFy/WSlt5eXkwMjKCnh7/fyZp8JNH9JhHjx7B398f58+fx44dO9CvXz+V9T/99BP69euHhg0bwtjYGM2aNcO8efNQVFSkUs/b2xsvvfQSzp07h65du8LExARNmjTB2rVrnxrDpUuXEBoaiqZNm0Imk8HBwQGjR4/GgwcPVOrNmTMHgiDg5s2bCA0NhZWVFSwtLREWFoacnBy1j9nR0RE9evRAZGSkSvnmzZvh6uqKl156qdztTp06BX9/f1haWkIul8PLywvHjh1TqVPRnJt9+/ahe/fuMDU1hbm5Ofr164fLly+r1AkNDYWZmRlu3bqFvn37wtzcHMHBwRUex61bt/Diiy+WSWwAwM7OTvm+snlOT85dKT3HV65cwYgRI2BtbY2XX34ZS5cuhSAI+Oeff8rsY/r06TAyMsLDhw+Vx1E656agoAD16tVDWFhYme0yMzMhk8kwZcoUAIBCocDs2bPRoUMHWFpawtTUFN27d0dsbGyF56AyO3bswMWLFzFz5swyiQ0AWFhYYMGCBcplFxcXhIaGlqnn7e2t0sNz6NAhCIKArVu34qOPPoKjoyPkcjnOnz8PQRCwadOmMvs4cOAABEHAL7/8oiz777//MHr0aNjb28PY2BgvvvgiNm7cWKVjJWJyQ/Q/2dnZ6NOnD86cOYPt27ejf//+Zep8++23MDMzQ3h4OFauXIkOHTpg9uzZmDZtWpm6Dx8+RN++fdGhQwcsWbIEjRo1wttvv/3UP9jR0dG4ffs2wsLC8MUXX2D48OHYunUr+vbtC1EUy9QfNmwYsrKysGjRIgwbNgzffvst5s6dq9GxjxgxAj///DMePXoEoOS//O3bt2PEiBHl1v/999/Ro0cPZGZmIiIiAgsXLkR6ejp69uyJ06dPV9rW999/j379+sHMzAyLFy/GrFmzcOXKFbz88stlkqDCwkL4+fnBzs4OS5cuxWuvvVbhfp2dnXHu3Dn89ddfGh27OoYOHYqcnBwsXLgQY8eOxbBhwyAIArZt21am7rZt29C7d29YW1uXWWdoaIhBgwZh9+7dZYYBd+/ejfz8fAwfPhxASbKzYcMGeHt7Y/HixZgzZw7u37+v7FXU1J49ewCUDBVWh3nz5mHv3r2YMmUKFi5ciDZt2qBp06blnqOoqChYW1vDz88PAJCcnIwuXbrgt99+w4QJE7By5Uo0b94cY8aMwYoVK6olXtJxIlEd980334gARGdnZ9HQ0FDcvXt3hXVzcnLKlL311luiXC4X8/LylGVeXl4iAHHZsmXKsvz8fLFdu3ainZ2dqFAoRFEUxfj4eBGA+M0331TaxpYtW0QA4pEjR5RlERERIgBx9OjRKnUHDRok1q9f/+kHLooiAHH8+PFiWlqaaGRkJH7//feiKIri3r17RUEQxDt37ijbuX//viiKolhcXCy2aNFC9PPzE4uLi1XibtKkidirVy9lWem5jY+PF0VRFLOyskQrKytx7NixKnEkJSWJlpaWKuUhISEiAHHatGlqHcvBgwdFfX19UV9fX/T09BQ//PBD8cCBA8pzXaq8c/74+YiIiFAulx57UFBQmbqenp5ihw4dVMpOnz4tAhC/++47leNwdnZWLh84cEAEIP78888q2/bt21ds2rSpcrmwsFDMz89XqfPw4UPR3t6+zM/8ybjL4+7uLlpaWlZa53HOzs5iSEhImXIvLy/Ry8tLuRwbGysCEJs2bVrmszt9+nTR0NBQTEtLU5bl5+eLVlZWKscwZswYsUGDBmJqaqrK9sOHDxctLS3L/Z0gqgx7boj+Jzk5GTKZDE5OThXWMTExUb7PyspCamoqunfvjpycHPz9998qdQ0MDPDWW28pl42MjPDWW28hJSUF586dU6uNvLw8pKamokuXLgCA8+fPl6k/btw4leXu3bvjwYMHyMzMrLCNJ1lbW8Pf3x9btmwBAERGRqJr167lTgS+cOECbty4gREjRuDBgwdITU1FamoqsrOz4ePjgyNHjqC4uLjcdqKjo5Geno6goCDldqmpqdDX14eHh0e5Qy5vv/22WsfQq1cvnDhxAq+++iouXryIJUuWwM/PD46Ojspei6p68hwDQGBgIM6dO4dbt24py6KiomBsbKwyd+lJPXv2hI2NDaKiopRlDx8+RHR0NAIDA5Vl+vr6MDIyAgAUFxcjLS0NhYWF6NixY7mfg6fJzMyEubm5xtupKyQkROWzC5Sco4KCAuzcuVNZdvDgQaSnpyuPVRRF7NixAwEBARBFUeVz4efnh4yMjCodL9VtTG6I/uerr76CkZER/P39K7y65fLlyxg0aBAsLS1hYWEBW1tbvP766wBKLh1/XMOGDctMPG3ZsiUAVHrfl7S0NEyaNAn29vYwMTGBra0tmjRpUm4bANC4cWOV5dLhkNI5H2lpaUhKSlK+ytsHUDI0FR0djYSEBOzevbvCIakbN24AKPkys7W1VXlt2LAB+fn5FbZRum3Pnj3LbHvw4MEyE38NDAzQqFGjcvdVnk6dOmHnzp14+PAhTp8+jenTpyMrKwtDhgzBlStX1N7Pk0rP/+OGDh0KPT09ZZIiiiK2b9+OPn36wMLCosJ9GRgY4LXXXsNPP/2E/Px8AMDOnTtRUFCgktwAwKZNm9C2bVvIZDLUr18ftra22Lt3b4XntzIWFhbIysrSeDt1lXeO3Nzc0KpVK5VELioqCjY2NujZsycA4P79+0hPT8e6devKfCZK5yZpc0I41Q28Worof9q0aYNff/0VPj4+6NWrF44dO6bSi5Oeng4vLy9YWFjg448/RrNmzSCTyXD+/HlMnTq1wt4KTQ0bNgzHjx/HBx98gHbt2sHMzAzFxcXw9/cvt42KrkYS/zc/Z/DgwTh8+LCyPCQkpNzJtK+++iqMjY0REhKC/Px8DBs2rNz9lsbw6aefol27duXWMTMzq3Tb77//Hg4ODmXWGxio/kkyNjau0hU3RkZG6NSpEzp16oSWLVsiLCwM27dvR0REBARBKHebJyeFP+7JHgmgJHnt3r07tm3bhhkzZuDkyZNISEhQ64qj4cOH46uvvsK+ffswcOBAbNu2Da1atYKbm5uyzg8//IDQ0FAMHDgQH3zwAezs7KCvr49Fixap9Bapq1WrVoiLi8Pdu3cr7Z0sVdl5Ku8zV945Akp6bxYsWIDU1FSYm5tjz549CAoKUv6sSz8Tr7/+OkJCQsrdR9u2bZ8aL9HjmNwQPaZz587YvXs3+vXrh169euHo0aOwtbUFUHJVyIMHD7Bz50706NFDuU18fHy5+7p3716Zy4avX78OABXesfbhw4eIiYnB3LlzMXv2bGV5aY9HVSxbtkzZiwOUfCmXx8TEBAMHDsQPP/yAPn36wMbGptx6zZo1A1DSE+Dr66tRLKXb2tnZabxtVXXs2BEAkJiYCOD/e7bS09NV6pV35dPTBAYG4p133sG1a9cQFRUFuVyOgICAp27Xo0cPNGjQAFFRUXj55Zfx+++/Y+bMmSp1fvzxRzRt2hQ7d+5USTQiIiI0jhMAAgICsGXLFvzwww+YPn36U+tbW1uXOUdAyXlq2rSp2u0GBgZi7ty52LFjB+zt7ZGZmamcNA0Atra2MDc3R1FRUY19Jkj3cViK6Ak+Pj7YsmULbt68CX9/f+XcldL/VsXHrlhSKBT48ssvy91PYWEhvvrqK5W6X331FWxtbdGhQ4dytymvDQDPdMVIhw4d4Ovrq3y1adOmwrpTpkxBREQEZs2aVen+mjVrhqVLlyqvrnrc/fv3K9zWz88PFhYWWLhwIQoKCjTa9mliY2PLvZrs119/BQC88MILAEqSMhsbGxw5ckSlXkU/x8q89tpr0NfXx5YtW5RX2KlzDxw9PT0MGTIEP//8M77//nsUFhaWGZIq77Nw6tQpnDhxQuM4AWDIkCFwdXXFggULyt1HVlaWSoLVrFkznDx5UuWqrl9++QV3797VqN3WrVvD1dUVUVFRiIqKQoMGDVT+OdDX11feS6m8K92e5TNBdRd7bojKMWjQIKxfvx6jR4/Gq6++iv3796Nr166wtrZGSEgI3n33XQiCgO+//77cL1SgpIdk8eLFuHPnDlq2bImoqChcuHAB69atq/DGbhYWFujRoweWLFmCgoICODo64uDBgxX2Dmmbm5ubytBIefT09LBhwwb06dMHL774IsLCwuDo6Ij//vsPsbGxsLCwwM8//1zuthYWFlizZg1GjhyJ9u3bY/jw4bC1tUVCQgL27t2Lbt26YdWqVVWKfeLEicjJycGgQYPQqlUrKBQKHD9+HFFRUXBxcVG5t8wbb7yBTz75BG+88QY6duyII0eOKHvVNGFnZ4dXXnkFy5cvR1ZWVpkEpTKBgYH44osvEBERAVdXV7Ru3Vplff/+/bFz504MGjQI/fr1Q3x8PNauXYs2bdqUm1Q+jaGhIXbu3AlfX1/06NEDw4YNQ7du3WBoaIjLly8jMjIS1tbWynvdvPHGG/jxxx/h7++PYcOG4datW/jhhx+UvW+aCAwMxOzZsyGTyTBmzJgyQ42ffPIJYmNj4eHhgbFjx6JNmzZIS0vD+fPn8dtvvyEtLU3jNqmOk+w6LaJaovRy5TNnzpRZt3TpUhGA2L9/f7GgoEA8duyY2KVLF9HExERs2LCh8nJjAGJsbKxyOy8vL/HFF18Uz549K3p6eooymUx0dnYWV61apbL/8i5L/vfff8VBgwaJVlZWoqWlpTh06FDx3r17FV6mXHqJ9pPHU3r5dWXwv0vBK1NRO3FxceLgwYPF+vXri8bGxqKzs7M4bNgwMSYm5qmxxMbGin5+fqKlpaUok8nEZs2aiaGhoeLZs2eVdUJCQkRTU9OnHkOpffv2iaNHjxZbtWolmpmZiUZGRmLz5s3FiRMnisnJySp1c3JyxDFjxoiWlpaiubm5OGzYMDElJUXtc/y49evXiwBEc3NzMTc3t8z6Jy8FL1VcXCw6OTmJAMT58+eXu37hwoWis7OzaGxsLLq7u4u//PJLuft7Mu7KPHz4UJw9e7bo6uoqyuVyUSaTiS+99JI4ffp0MTExUaXusmXLREdHR9HY2Fjs1q2bePbs2QovBd++fXuFbd64cUMEIAIQ//jjj3LrJCcni+PHjxednJxEQ0ND0cHBQfTx8RHXrVun1nERPU4QxQr+7SSiKvP29kZqamq13FCOiIgqxzk3REREpFOY3BAREZFOYXJDREREOoVzboiIiEinsOeGiIiIdAqTGyIiItIpde4mfsXFxbh37x7Mzc0rfHYKERER1S6iKCIrKwsNGzZ86jPn6lxyc+/ePbUeGkdERES1z927d9GoUaNK69S55Mbc3BxAycmxsLCQOBoiIiJSR2ZmJpycnJTf45Wpc8lN6VCUhYUFkxsiIqLnjDpTSjihmIiIiHQKkxsiIiLSKUxuiIiISKfUuTk3RERSKioqQkFBgdRhENVKRkZGT73MWx1MboiIaoAoikhKSkJ6errUoRDVWnp6emjSpAmMjIyeaT9MboiIakBpYmNnZwe5XM6biBI9ofQmu4mJiWjcuPEz/Y4wuSEiqmZFRUXKxKZ+/fpSh0NUa9na2uLevXsoLCyEoaFhlffDCcVERNWsdI6NXC6XOBKi2q10OKqoqOiZ9sPkhoiohnAoiqhy2vodYXJDREREOkXS5ObIkSMICAhAw4YNIQgCdu/e/dRtDh06hPbt28PY2BjNmzfHt99+W+1xEhFRWaGhoRg4cKDUYdRa1XF+XFxcsGLFCq3uUxdJmtxkZ2fDzc0Nq1evVqt+fHw8+vXrh1deeQUXLlzA5MmT8cYbb+DAgQPVHCkRET1p5cqVKv9gent7Y/Lkyc+834KCAkydOhWurq4wNTVFw4YNMWrUKNy7d0/jfR0+fBg9e/ZEvXr1IJfL0aJFC4SEhEChUDxznE/z5PmpKZmZmZg5cyZatWoFmUwGBwcH+Pr6YufOnRBFscbjkYKkV0v16dMHffr0Ubv+2rVr0aRJEyxbtgwA0Lp1a/zxxx/47LPP4OfnV11hqqW4qAgPs+5LGgMAyGT1IGjhBkhVZWKoz3kFRHWEpaVltew3JycH58+fx6xZs+Dm5oaHDx9i0qRJePXVV3H27Fm193PlyhX4+/tj4sSJ+Pzzz2FiYoIbN25gx44dzzRhVaFQqHUfluo6P5VJT0/Hyy+/jIyMDMyfPx+dOnWCgYEBDh8+jA8//BA9e/aElZVVlfZdUFDwTFcw1aTn6lLwEydOwNfXV6XMz8+v0v8U8vPzkZ+fr1zOzMysltgeZt2H90+9qmXfmmieKyDuzgJI1SnX0dka28d5MsEh0hE//vgj5s6di5s3b0Iul8Pd3R0//fQTTE1NERoaivT0dOzevRuhoaE4fPgwDh8+jJUrVwIo6W13cXHBX3/9hQ8++ABHjx6Fqakpevfujc8++ww2NjbltmlpaYno6GiVslWrVqFz585ISEhA48aN1Yr94MGDcHBwwJIlS5RlzZo1g7+/v3J5zpw52L17Ny5cuKAsW7FiBVasWIE7d+4AgPI4O3XqhNWrV8PY2BhBQUGIiYnBqVOnVNp0c3PDa6+9htmzZ6ucn3Xr1mHOnDn4999/Ve7AO2DAANSvXx8bN27ErVu3EB4ejpMnTyI7OxutW7fGokWLynzvVWbGjBm4c+cOrl+/joYNGyrLW7ZsiaCgIMhkMgAlE3d37dqlMmxmZWWFFStWIDQ0FHfu3EGTJk2wdetWfPnllzh16hSWLFmCqVOnYufOnSodE7t27cKoUaOQnJwMuVyOu3fv4v3338fBgwehp6eH7t27Y+XKlXBxcVH7OJ7VczWhOCkpCfb29ipl9vb2yMzMRG5ubrnbLFq0CJaWlsqXk5NTTYQqmZsmIkyER5K1f/afh8gteLZL+IjqAlEUkaMolOSl7tBEYmIigoKCMHr0aFy9ehWHDh3C4MGDy91+5cqV8PT0xNixY5GYmIjExEQ4OTkhPT0dPXv2hLu7O86ePYv9+/cjOTkZw4YN0+h8ZWRkQBAElV4Hb29vhIaGVriNg4MDEhMTceTIEY3aKk9MTAyuXbuG6Oho/PLLLwgODsbp06dx69YtZZ3Lly/j0qVLGDFiRJnthw4digcPHiA2NlZZlpaWhv379yM4OBgA8OjRI/Tt2xcxMTGIi4uDv78/AgICkJCQoFaMxcXF2Lp1K4KDg1USm1JmZmYwMNCsT2PatGmYNGkSrl69iqFDh6J///6IjIxUqbN582YMHDgQcrkcBQUF8PPzg7m5OY4ePYpjx47BzMwM/v7+NTIUWOq56rmpiunTpyM8PFy5nJmZWS0JjrW5LQ4NiH56xWqSm5eGPgcCAQB/TH0FJvLy/yOqLjmKInSc/1uNtkn0PMstKEKb2dLMF7zysR/kRk//85+YmIjCwkIMHjwYzs7OAABXV9dy61paWsLIyAhyuRwODg7K8lWrVsHd3R0LFy5Ulm3cuBFOTk64fv06WrZs+dQ48vLyMHXqVAQFBcHCwkJZ3rhxYzRo0KDC7YYOHYoDBw7Ay8sLDg4O6NKlC3x8fDBq1CiV/ajD1NQUGzZsUBmOcnNzQ2RkJGbNmgWg5Evew8MDzZs3L7O9tbU1+vTpg8jISPj4+AAo6RWzsbHBK6+8otyfm5ubcpt58+Zh165d2LNnDyZMmPDUGFNTU/Hw4UO0atVKo2OrzOTJkzF48GDlcnBwMEaOHImcnBzI5XJkZmZi79692LVrFwAgKioKxcXF2LBhg7IH/5tvvoGVlRUOHTqE3r17ay22yjxXyY2DgwOSk5NVypKTk2FhYQETE5NytzE2NoaxsXG1x6anr4/6Vg5Pr1hNcnL+/0dpYmSg1h8uIqLKuLm5wcfHB66urvDz80Pv3r0xZMgQWFtbq72PixcvIjY2FmZmZmXW3bp1C2fOnMFbb72lLNu3bx+6d++uXC4oKMCwYcMgiiLWrFmjsv13331Xadv6+vr45ptvMH/+fPz+++84deoUFi5ciMWLF+P06dOVJkZPcnV1LTPPJjg4GBs3bsSsWbMgiiK2bNmi8s/0k4KDgzF27Fh8+eWXMDY2xubNmzF8+HDlMNWjR48wZ84c7N27V5lY5ubmqt1zUx2ThTt27Kiy3LdvXxgaGmLPnj0YPnw4duzYAQsLC+XQ2cWLF3Hz5k2Ym5urbJeXl6fSy1XdnqtvQE9PT/z6668qZdHR0fD09JQoIiKiqjEx1MeVj6W5EMLEUF+tevr6+oiOjsbx48dx8OBBfPHFF5g5cyZOnTqFJk2aqLWPR48eISAgAIsXLy6zrkGDBiguLoaHh4eyzNHRUfm+NLH5559/8Pvvv2vc2/L4PkeOHImRI0di3rx5aNmyJdauXYu5c+dCT0+vTFJQ3lPbTU1Ny5QFBQVh6tSpOH/+PHJzc3H37l0EBgZWGEdAQABEUcTevXvRqVMnHD16FJ999ply/ZQpUxAdHY2lS5eiefPmMDExwZAhQ9QezrG1tYWVlRX+/vvvp9YVBKFKx21kZIQhQ4YgMjISw4cPR2RkJAIDA5XDXY8ePUKHDh2wefPmcuOrKZImN48ePcLNmzeVy/Hx8bhw4QLq1auHxo0bY/r06fjvv/+U2fm4ceOwatUqfPjhhxg9ejR+//13bNu2DXv37pXqEIiIqkQQhOeih1UQBHTr1g3dunXD7Nmz4ezsjF27dpXbQ2FkZFTmKqT27dtjx44dcHFxqXC+x5P/5QP/n9jcuHEDsbGxWnsml7W1NRo0aIDs7GwAJV+4SUlJEEVROYzy+OTiyjRq1AheXl7YvHkzcnNz0atXL9jZ2VVYXyaTYfDgwdi8eTNu3ryJF154Ae3bt1euP3bsGEJDQzFo0CAAJd+RpZOa1aGnp4fhw4fj+++/R0RERJl5N48ePYJMJoOBgQFsbW2RmJioXHfjxg3k5OSo1U5wcDB69eqFy5cv4/fff8f8+fOV69q3b4+oqCjY2dlVORnVBkknFJ89exbu7u5wd3cHAISHh8Pd3R2zZ88GUDLe+3h3XJMmTbB3715ER0fDzc0Ny5Ytw4YNGyS/DJyISBeVDuOcPXsWCQkJ2LlzJ+7fv4/WrVuXW9/FxQWnTp3CnTt3kJqaiuLiYowfPx5paWkICgrCmTNncOvWLRw4cABhYWEVXo5dUFCAIUOG4OzZs9i8eTOKioqQlJSEpKQklV6MUaNGYfr06RXG/9VXX+Htt9/GwYMHcevWLVy+fBlTp07F5cuXERAQAKBkUvL9+/exZMkS3Lp1C6tXr8a+ffvUPkfBwcHYunUrtm/frpwY/LT6e/fuxcaNG8vUb9GiBXbu3IkLFy7g4sWLGDFiBIqLi9WOBQAWLFgAJycneHh44LvvvsOVK1dw48YNbNy4Ee7u7nj0qOSCk549e2LVqlWIi4vD2bNnMW7cOLUv8+7RowccHBwQHByMJk2aqPS8BQcHw8bGBgMGDMDRo0cRHx+PQ4cO4d1338W///6r0bE8C0mTG29vb4iiWOZVetOjb7/9FocOHSqzTVxcHPLz83Hr1q1KZ8oTEVHVWVhY4MiRI+jbty9atmyJjz76CMuWLavw/mRTpkyBvr4+2rRpA1tbWyQkJKBhw4Y4duwYioqK0Lt3b7i6umLy5MmwsrJSuST6cf/99x/27NmDf//9F+3atUODBg2Ur+PHjyvrJSQkqPQ+PKlz58549OgRxo0bhxdffBFeXl44efIkdu/eDS8vLwAl90v78ssvsXr1ari5ueH06dOYMmWK2udoyJAhePDgAXJyctS6G3HpDQWvXbtW5qqq5cuXw9raGl27dkVAQAD8/PxUenbUUa9ePZw8eRKvv/465s+fD3d3d3Tv3h1btmzBp59+qrz3zrJly+Dk5ITu3btjxIgRmDJlitoPdhUEAUFBQbh48WKZBE0ul+PIkSNo3LgxBg8ejNatW2PMmDHIy8ur0Z4cQawrtyv8n8zMTFhaWiIjI0PSLjNty8lJhcf2khn3p4bGQl7jV0sVKq/8UPdKDKK6Ii8vD/Hx8WjSpInyPiNEVFZlvyuafH8/V/e5ISIiInoaJjdERESkU5jcEBERkU5hckNEREQ6hckNERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3RERUJaGhoWo9cqCumjNnDtq1a6fVfXp7e2Py5Mla3acuYnJDRERVsnLlSuWzAAHtfvHu3LkTvXv3Rv369SEIgtpP6n7SxYsX8eqrr8LOzg4ymQwuLi4IDAxESkqKVuKszJQpUxATE1Pt7TxJoVBgyZIlcHNzg1wuh42NDbp164ZvvvkGBQUFNR6PFPgAICIiqpLShzBWh+zsbLz88ssYNmwYxo4dW6V93L9/Hz4+Pujfvz8OHDgAKysr3LlzB3v27EF2dnaVY1MoFDAyMnpqPTMzM5iZmVW5napQKBTw8/PDxYsXMW/ePHTr1g0WFhY4efIkli5dCnd39yr3JhUUFKj95HCpseeGiIgq9OOPP8LV1RUmJiaoX78+fH19lYnB48NSoaGhOHz4MFauXAlBECAIAu7cuQMA+Ouvv9CnTx+YmZnB3t4eI0eORGpqaqXtjhw5ErNnz4avr2+VYz927BgyMjKwYcMGuLu7o0mTJnjllVfw2WefoUmTJgCAb7/9FlZWVirb7d69G4IgKJdLh5c2bNigfKDjunXr0LBhQxQXF6tsO2DAAIwePVplOwA4ePAgZDIZ0tPTVepPmjQJPXv2BAA8ePAAQUFBcHR0hFwuh6urK7Zs2aLRMa9YsQJHjhxBTEwMxo8fj3bt2qFp06YYMWIETp06hRYtWgAAXFxcsGLFCpVt27Vrhzlz5iiXBUHAmjVr8Oqrr8LU1BTz5s1Do0aNsGbNGpXt4uLioKenh3/++QcAkJ6ejjfeeAO2trawsLBAz549cfHiRY2O41kxuSEikoIoAopsaV6iqFaIiYmJCAoKwujRo3H16lUcOnQIgwcPhljO9itXroSnpyfGjh2LxMREJCYmwsnJCenp6ejZsyfc3d1x9uxZ7N+/H8nJyRg2bNgzn8LQ0FB4e3tXuN7BwQGFhYXYtWtXuTFr4ubNm9ixYwd27tyJCxcuYOjQoXjw4AFiY2OVddLS0rB//34EBweX2d7HxwdWVlbYsWOHsqyoqAhRUVHK+nl5eejQoQP27t2Lv/76C2+++SZGjhyJ06dPqx3n5s2b4evrC3d39zLrDA0NYWpqqslhY86cORg0aBD+/PNPvPHGGwgKCkJkZGSZNrt16wZnZ2cAwNChQ5GSkoJ9+/bh3LlzaN++PXx8fJCWlqZR28+Cw1JERFIoyAEWNpSm7Rn3AKOnf8klJiaisLAQgwcPVn5xubq6llvX0tISRkZGkMvlcHBwUJavWrUK7u7uWLhwobJs48aNcHJywvXr19GyZcsqH0aDBg3K9Jw8rkuXLpgxYwZGjBiBcePGoXPnzujZsydGjRoFe3t7jdpSKBT47rvvYGtrqyzr06cPIiMj4ePjA6Ckl8vGxgavvPJKme319fUxfPhwREZGYsyYMQCAmJgYpKen47XXXgMAODo6YsqUKcptJk6ciAMHDmDbtm3o3LmzWnHeuHGj0oRPUyNGjEBYWJhyOTg4GMuWLUNCQgIaN26M4uJibN26FR999BEA4I8//sDp06eRkpICY2NjAMDSpUuxe/du/Pjjj3jzzTe1Fltl2HNDRETlcnNzg4+PD1xdXTF06FCsX78eDx8+1GgfFy9eRGxsrHL+iZmZGVq1agUAuHXrFjZv3qyy7ujRo2rve9GiRfjuu+8qrbNgwQIkJSVh7dq1ePHFF7F27Vq0atUKf/75p0bH4ezsrJLYACVf9Dt27EB+fj6Akh6M4cOHQ0+v/K/W4OBgHDp0CPfu3VPW79evn3JYrKioCPPmzYOrqyvq1asHMzMzHDhwAAkJCWrH+aw9VE/q2LGjynK7du3QunVrZe/N4cOHkZKSgqFDhwIo+Xk/evQI9evXV/m5xsfH49atW1qNrTLsudFBuYW5Jf8V1mibRYCgAMTnY7IZkeQM5SU9KFK1rQZ9fX1ER0fj+PHjOHjwIL744gvMnDkTp06dUs5ZeZpHjx4hICAAixcvLrOutOfFw8NDWebo6KjeMWigfv36GDp0KIYOHYqFCxfC3d0dS5cuxaZNm6Cnp1cmISjviqLyhnMCAgIgiiL27t2LTp064ejRo/jss88qjKNTp05o1qwZtm7dirfffhu7du1Sudrs008/xcqVK7FixQq4urrC1NQUkydPhkKhUPtYW7Zsib///vup9Z7luIODgxEZGYlp06YhMjIS/v7+qF+/PoCSn3eDBg1w6NChMts9ObepOjG50UHeu/pK0q55K6Awxxmi6CdJ+0TPFUFQa2hIaoIgoFu3bujWrRtmz54NZ2dn7Nq1C+Hh4WXqGhkZoaioSKWsffv22LFjB1xcXGBgUP5Xjrm5ebXEXh4jIyM0a9ZMOSna1tYWWVlZyM7OVn6Rq3vZuUwmw+DBg7F582bcvHkTL7zwAtq3b1/pNsHBwdi8eTMaNWoEPT099OvXT7nu2LFjGDBgAF5//XUAQHFxMa5fv442bdqofXwjRozAjBkzEBcXV2beTUFBARQKBUxNTWFra4vExETluszMTMTHx6vdxkcffYRz587hxx9/xNq1a5Xr2rdvj6SkJBgYGMDFxUXtuLWNw1I6wkRfBve8PKnDgIH8H+TlPqj1EyWJ6OlOnTqFhQsX4uzZs0hISMDOnTtx//59tG7dutz6Li4uOHXqFO7cuYPU1FQUFxdj/PjxSEtLQ1BQEM6cOYNbt27hwIEDCAsLK5MIPS4tLQ0XLlzAlStXAADXrl3DhQsXkJSUpKwzffp0jBo1qsJ9/PLLL3j99dfxyy+/4Pr167h27RqWLl2KX3/9FQMGDAAAeHh4QC6XY8aMGbh16xYiIyNVelOeJjg4GHv37sXGjRvLnUhcXv3z589jwYIFGDJkiHJeCgC0aNFC2VN29epVvPXWW0hOTlY7FgCYPHkyunXrBh8fH6xevRoXL17E7du3sW3bNnTp0gU3btwAAPTs2RPff/89jh49ij///BMhISHQ19dXqw0XFxd07doVY8aMQVFREV599VXlOl9fX3h6emLgwIE4ePAg7ty5g+PHj2PmzJk4e/asRsfyLNhzoyMEQcCmxBTkPnb5Yk3KFQR4OzcCAJisbCVdkuHUBRi9v+S/YiJ6JhYWFjhy5AhWrFiBzMxMODs7Y9myZejTp0+59adMmYKQkBC0adMGubm5iI+Ph4uLC44dO4apU6eid+/eyM/Ph7OzM/z9/SucmwIAe/bsUZnIOnz4cABARESE8nLlxMTESuejtGnTBnK5HO+//z7u3r0LY2NjtGjRAhs2bMDIkSMBAPXq1cMPP/yADz74AOvXr4ePjw/mzJmj9sTXnj17ol69erh27RpGjBjx1PrNmzdH586dcfr06TKXYn/00Ue4ffs2/Pz8IJfL8eabb2LgwIHIyMhQKxYAMDY2RnR0ND777DN89dVXmDJlCuRyOVq3bo13330XL730EoCSxDA+Ph79+/eHpaUl5s2bp3bPDVCSpL3zzjsYNWoUTExMlOWCIODXX3/FzJkzERYWhvv378PBwQE9evTQeBL3sxBEbc8+quUyMzNhaWmJjIwMWFhYSB2O9ogisNEfuHtSkuZzBAEeLk4AgFN37kIu5cdKzStBiGpKXl4e4uPjlfdIIaLyVfa7osn3N3tudIUglPRY1PBE4lK5OVnAT71K3k/6G3J5zY2hAwAUOcDS5jXbJhER1UpMbnSJlBMUCx+714SRKXtOiIhIMpxQTERERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBHRM3FxcSnzKIHKzJkzB+3atXvmdgVBwO7du595PzUlNDQUAwcO1Oo+NT33dQWTGyIiqjMOHz6sfB6UXC5HixYtEBISAoVCUe1tr1y5UqOHcmpLZmYmZs6ciVatWkEmk8HBwQG+vr7YuXMndPUJTLxDMRER1QlXrlyBv78/Jk6ciM8//xwmJia4ceMGduzYUekTyp9GoVDAyMjoqfUsLS2r3EZVpaen4+WXX0ZGRgbmz5+PTp06wcDAAIcPH8aHH36Inj17wsrKqkr7LigogKGhoXYD1hL23BARUYWysrIQHBwMU1NTNGjQAJ999hm8vb0xefLkCrdJSEjAgAEDYGZmBgsLCwwbNgzJycll6n311VdwcnKCXC7HsGHDVJ5+febMGfTq1Qs2NjawtLSEl5cXzp8//0zHcvDgQTg4OGDJkiV46aWX0KxZM/j7+2P9+vXKJ1uXN2S2YsUKuLi4KJdLh5cWLFiAhg0b4oUXXsCMGTPg4eFRpk03Nzd8/PHHKtsBwLp169CwYUMUFxer1B8wYABGjx4NALh16xYGDBgAe3t7mJmZoVOnTvjtt980OuYZM2bgzp07OHXqlPKJ7S1btsTYsWNx4cIFmJmZASh/iM/KykrZ03Tnzh0IgoCoqCh4eXlBJpNhzZo1MDExwb59+1S227VrF8zNzZGTU/Ksw7t372LYsGGwsrJCvXr1MGDAANy5c0ej49AUkxsiIgmIooicghxJXpoMRYSHh+PYsWPYs2cPoqOjcfTo0UqTjOLiYgwYMABpaWk4fPgwoqOjcfv2bQQGBqrUu3nzJrZt24aff/4Z+/fvR1xcHN555x3l+qysLISEhOCPP/7AyZMn0aJFC/Tt2xdZWVkVtu3t7Y3Q0NAK1zs4OCAxMRFHjhxR+/grEhMTg2vXriE6Ohq//PILgoODcfr0ady6dUtZ5/Lly7h06RJGjBhRZvuhQ4fiwYMHiI2NVZalpaVh//79CA4OBgA8evQIffv2RUxMDOLi4uDv74+AgAAkJCSoFWNxcTG2bt2K4OBgNGzYsMx6MzMzGBhoNoAzbdo0TJo0CVevXsXQoUPRv39/REZGqtTZvHkzBg4cCLlcjoKCAvj5+cHc3BxHjx7FsWPHYGZmBn9//2odCuSwFBGRBHILc+ERWfY//ZpwasQpyA3lT62XlZWFTZs2ITIyEj4+PgCAb775ptwvylIxMTH4888/ER8fDycnJwDAd999hxdffBFnzpxBp06dAAB5eXn47rvv4OjoCAD44osv0K9fPyxbtgwODg7o2bOnyn7XrVsHKysrHD58GP379y+37caNG6NBgwYVxjZ06FAcOHAAXl5ecHBwQJcuXeDj44NRo0bBwsLiqefjcaamptiwYYPKcJSbmxsiIyMxa9YsACVf8h4eHmjevHmZ7a2trdGnTx+Vc/vjjz/CxsYGr7zyinJ/bm5uym3mzZuHXbt2Yc+ePZgwYcJTY0xNTcXDhw/RqlUrjY6tMpMnT8bgwYOVy8HBwRg5ciRycnIgl8uRmZmJvXv3YteuXQCAqKgoFBcXY8OGDRAEAUDJZ8jKygqHDh1C7969tRbb49hzQ0RE5bp9+zYKCgrQuXNnZZmlpSVeeOGFCre5evUqnJyclIkNALRp0wZWVla4evWqsqxx48bKxAYAPD09UVxcjGvXrgEAkpOTMXbsWLRo0QKWlpawsLDAo0ePKu21+O6777Bo0aIK1+vr6+Obb77Bv//+iyVLlsDR0RELFy7Eiy++iMTExMpPxhNcXV3LzLMJDg5W9mKIoogtW7Yoe2HKExwcjB07diA/Px9ASTI0fPhw6OmVfDU/evQIU6ZMQevWrWFlZQUzMzNcvXpV7Z6b6pgs3LFjR5Xlvn37wtDQEHv27AEA7NixAxYWFvD19QUAXLx4ETdv3oS5uTnMzMxgZmaGevXqIS8vT6WXS9vYc0NEJAETAxOcGnFKsrZru5CQEDx48AArV66Es7MzjI2N4enpqZWhDEdHR4wcORIjR47EvHnz0LJlS6xduxZz586Fnp5emaSgoKCgzD5MTU3LlAUFBWHq1Kk4f/48cnNzcffu3TLDcY8LCAiAKIrYu3cvOnXqhKNHj+Kzzz5Trp8yZQqio6OxdOlSNG/eHCYmJhgyZIja58DW1hZWVlb4+++/n1pXEIQqHbeRkRGGDBmCyMhIDB8+HJGRkQgMDFQOdz169AgdOnTA5s2by42vujC5Ia3LURTBxKCwZhtVFKK0kz1HUQightuvRUwM9ZXdv1R7CYKg1tCQlJo2bQpDQ0OcOXMGjRs3BgBkZGTg+vXr6NGjR7nbtG7dGnfv3sXdu3eVvTdXrlxBeno62rRpo6yXkJCAe/fuKYe4Tp48CT09PWWv0LFjx/Dll1+ib9++AEompaampmr9GK2trdGgQQNkZ2cDKPnCTUpKgiiKyt+jCxcuqLWvRo0awcvLC5s3b0Zubi569eoFOzu7CuvLZDIMHjwYmzdvxs2bN/HCCy+gffv2yvXHjh1DaGgoBg0aBKAkUdBkIq6enh6GDx+O77//HhEREWWGEx89egSZTAYDAwPY2tqq9F7duHFDOSH4aYKDg9GrVy9cvnwZv//+O+bPn69c1759e0RFRcHOzk7job9nweSGtK774lhAfPplkdpkgjxclZW87zD/N+RCVqPt1yYdna2xfZwnExx6Zubm5ggJCcEHH3yAevXqwc7ODhEREdDT06vw8+Xr6wtXV1cEBwdjxYoVKCwsxDvvvAMvLy+VIQ2ZTIaQkBAsXboUmZmZePfddzFs2DA4ODgAAFq0aIHvv/8eHTt2RGZmJj744APlFU0VGTVqFBwdHSscmvrqq69w4cIFDBo0CM2aNVPO+7l8+TK++OILACWTku/fv48lS5ZgyJAh2L9/P/bt26f2F3NwcDAiIiKgUChUemEqq9+/f39cvnwZr7/+usq6Fi1aYOfOnQgICIAgCJg1a1aZq6ueZsGCBTh06BA8PDywYMECdOzYEYaGhjh69CgWLVqEM2fOwMrKCj179sSqVavg6emJoqIiTJ06Ve3LvHv06AEHBwcEBwejSZMmKleNBQcH49NPP8WAAQPw8ccfo1GjRvjnn3+wc+dOfPjhh2jUqJFGx6MuzrkhrZAZ8KNUW5z95yFyC6p+zw6ixy1fvhyenp7o378/fH190a1bN7Ru3RoyWfn/QAiCgJ9++gnW1tbo0aMHfH190bRpU0RFRanUa968OQYPHoy+ffuid+/eaNu2Lb788kvl+q+//hoPHz5E+/btMXLkSLz77ruV9oIAJb1Blc2d6dy5Mx49eoRx48bhxRdfhJeXF06ePIndu3fDy8sLQEnP05dffonVq1fDzc0Np0+fxpQpU9Q9XRgyZAgePHiAnJwcte5GXHpDwWvXrpW5qmr58uWwtrZG165dERAQAD8/P5WeHXXUq1cPJ0+exOuvv4758+fD3d0d3bt3x5YtW/Dpp58q772zbNkyODk5oXv37hgxYgSmTJkCuVy9nkVBEBAUFISLFy+WmWMkl8tx5MgRNG7cGIMHD0br1q0xZswY5OXlVWtPjiDq6u0JK5CZmQlLS0tkZGTUaBeZrsspyFFe+XFo6PGaH9NXZEO+tKTbPGdKAmBUdjxc1+UoitBxfsk9MK587Ae5ETtma4u8vDzEx8ejSZMmFSYFz4vs7Gw4Ojpi2bJlGDNmjNThkI6p7HdFk+9v/vUj7RMUgKBfs23qFSBHEGAiiiVf6vxiJ9KKuLg4/P333+jcuTMyMjKUN6QbMGCAxJERVYzfAKR13tu8pWnYxQnueXnYJIrgbBMi7Vm6dCmuXbsGIyMjdOjQAUePHoWNjY3UYRFViMkNaYWJgQnc7dwRlxInaRxxMhlyi/Igh5mkcRDpCnd3d5w7d07qMIg0wuSGtEIQBGzy34TcwlxJ2s/NfQDvXX0laZuIiGoXJjekNZLet6NAvfsxEEmpjl2/QaQxbf2O8PpdIqJqVnq/EHVvikZUV5XefVlf/9kuSmHPDRFRNdPX14eVlRVSUlIAlNz7gzdZJFJVXFyM+/fvQy6Xa/y08icxuSHdo8gBDLKljqLmKQphgjzkwljqSKgcpXfeLU1wiKgsPT09NG7c+JmTfyY3pHtWtgXq4NwGOYCrMuBMcUtA9JM6HHqCIAho0KAB7Ozsyn0gIRGVPIiz9Knoz4LJDemG5+ApxzWlk9515BTkAMaWUodC5dDX13/m+QREVDkmN6QbHu/C/OBmnUx2crIzIV/ZSuowiIgkx+SGdI+hvORV1ygKpY6AiKhW4KXgREREpFMkT25Wr14NFxcXyGQyeHh44PTp05XWX7FiBV544QWYmJjAyckJ7733HvLy8mooWiIiIqrtJE1uoqKiEB4ejoiICJw/fx5ubm7w8/Or8FLJyMhITJs2DREREbh69Sq+/vprREVFYcaMGTUcOREREdVWkiY3y5cvx9ixYxEWFoY2bdpg7dq1kMvl2LhxY7n1jx8/jm7dumHEiBFwcXFB7969ERQU9NTeHiIiIqo7JEtuFAoFzp07B19f3/8PRk8Pvr6+OHHiRLnbdO3aFefOnVMmM7dv38avv/6Kvn0rfmBifn4+MjMzVV5ERESkuyS7Wio1NRVFRUWwt7dXKbe3t8fff/9d7jYjRoxAamoqXn75ZYiiiMLCQowbN67SYalFixZh7ty5Wo2diIiIai/JJxRr4tChQ1i4cCG+/PJLnD9/Hjt37sTevXsxb968CreZPn06MjIylK+7d+/WYMRERERU0yTrubGxsYG+vj6Sk5NVypOTk5XPYHnSrFmzMHLkSLzxxhsAAFdXV2RnZ+PNN9/EzJkzy71ls7GxMYyN+awdIiKiukKynhsjIyN06NABMTExyrLi4mLExMTA09Oz3G1ycnLKJDCltzEX6+CzhIiIiKgsSe9QHB4ejpCQEHTs2BGdO3fGihUrkJ2djbCwMADAqFGj4OjoiEWLFgEAAgICsHz5cri7u8PDwwM3b97ErFmzEBAQwGe1EBEREQCJk5vAwEDcv38fs2fPRlJSEtq1a4f9+/crJxknJCSo9NR89NFHEAQBH330Ef777z/Y2toiICAACxYskOoQiIiIqJYRxDo2npOZmQlLS0tkZGTAwsJC6nBIS3IKcuAR6QEAODXiFOR18NlSOY8yIF/auOT9lATIzfhUcCLSHZp8fz9XV0sRERERPQ2fCk46J7cwV+oQJJFbmAsIAkzqVmcsEVEZTG5I53hv85Y6BOm4OME9Lw9rmOAQUR3GYSnSCSYGJnC3c5c6jFohTiZDXlGe1GEQEUmGPTekEwRBwCb/TXV2SAoA0jJS0GdvgNRhEBFJjskN6QxBEOrkVVKlcg1kUodARFQrcFiKiIiIdAqTGyIiItIpTG6IiIhIpzC5ISIiIp3C5IaIiIh0CpMbIiIi0ilMboiIiEinMLkhIiIincKb+BHpooIcQJEtTduGckAQpGmbiAhMboh0ksmX7QGpHp7p1AUYvZ8JDhFJhsNSRLqitjx64u7Jkp4jIiKJsOeGSFc81lOSO+lvyOXmNdu+IgdY2rxm2yQiKgeTGyJdZGRa8iIiqoM4LEVEREQ6hT03RDooR1EEE4PCmm1UUYjSWT85ikIANdx+LWFiqA+Bk6mJJMXkhkgHdV8cC4hGNdqmCfJwVVbyvsP835ALWY22X1t0dLbG9nGeTHCIJMRhKSIdITPgr3NtcPafh8gtKJI6DKI6jT03RDri8Z6Cc7N8YWJgUrMBKLKBpf9r/yPfOjehOUdRhI7zf5M6DCICkxsinWRiqA+5YU3/ev9/e3IjA8CIf16ISBrsxyYiIiKdwuSGiIiIdAr7jYl0UG5hbs03WpgLCAJMRBG8ToiIpKRxcuPl5YUxY8Zg6NChMDGp4QmLRKQW723e0jTs4gT3vDxsYoJDRBLSeFjK3d0dU6ZMgYODA8aOHYuTJ09WR1xEpCETAxO427lLHQbiZDLkFuVJHQYR1WEa99ysWLECS5cuxZ49e7Bp0yb06NEDzZs3x+jRozFy5EjY29tXR5xE9BSCIGCT/yZphqQA5OY+gPeuvpK0TUT0uCrNuTEwMMDgwYMxePBgpKSkYN26dZg1axZmzJiBvn374t1330XPnj21HSsRPYUgCJAbyp9esToU5EjTLhHRE57paqnTp08jIiICy5Ytg52dHaZPnw4bGxv0798fU6ZM0VaMRERERGrTuOcmJSUF33//Pb755hvcuHEDAQEB2LJlC/z8/JR3SA0NDYW/vz+WLl2q9YCJiIiIKqNxctOoUSM0a9YMo0ePRmhoKGxtbcvUadu2LTp16qSVAImIiIg0oXFyExMTg+7du1dax8LCArGxsVUOioiIiKiqNJ5zExERgfT09DLlmZmZnERMREREktM4uTl8+DAUCkWZ8ry8PBw9elQrQRERERFVldrDUpcuXQIAiKKIK1euICkpSbmuqKgI+/fvh6Ojo/YjJCIiItKA2slNu3btIAgCBEEod/jJxMQEX3zxhVaDIyIiItKU2slNfHw8RFFE06ZNcfr0aZWrpIyMjGBnZwd9ff1qCZKIiIhIXWonN87OzgCA4uLiaguGiIiI6Fmpldzs2bMHffr0gaGhIfbs2VNp3VdffVUrgRERERFVhVrJzcCBA5GUlAQ7OzsMHDiwwnqCIKCoqEhbsRERERFpTK3k5vGhKA5LEdFTKXIAg2ypo6hZikKYIO9/77NRxecSa4ehHPjf43CI6iIJf/uISGetbAuIotRR1Cg5gKuy/y1I/Vg9py7A6P1McKjOUiu5+fzzz9Xe4bvvvlvlYIjoOWZgInUEVOruSaAgBzAylToSIkmoldx89tlnau1MEAQmN0R11eO9BB/crHPJTo6iEB3m/wYAOPeRL+RGEnSMK3KApc1rvl2iWkat3774+PjqjoOIdImhvORVpxQiF/8blzIyBaRIbogIQBWeLUVERERUm6n1r0V4eDjmzZsHU1NThIeHV1p3+fLlWgmMiIiIqCrUSm7i4uJQUFCgfF8RgTPziYiISGJqJTexsbHlviciIiKqbZ5pxtvdu3cBAE5OTloJhoh0Q25hrmRtmxiYsBeZqI7TOLkpLCzE3Llz8fnnn+PRo0cAADMzM0ycOBEREREwNDTUepBE9Hzx3uYtWdvudu7Y5L+JCQ5RHaZxcjNx4kTs3LkTS5YsgaenJwDgxIkTmDNnDh48eIA1a9ZoPUgiqv1MDEzgbueOuJSK5+XVhLiUOOQW5kJe5y5FJ6JSGic3kZGR2Lp1K/r06aMsa9u2LZycnBAUFKRxcrN69Wp8+umnSEpKgpubG7744gt07ty5wvrp6emYOXMmdu7cibS0NDg7O2PFihXo27evpodCRFokCAI2+W+SbEgqtzBX0h4jIqo9NE5ujI2N4eLiUqa8SZMmMDIy0mhfUVFRCA8Px9q1a+Hh4YEVK1bAz88P165dg52dXZn6CoUCvXr1gp2dHX788Uc4Ojrin3/+gZWVlaaHQUTVQBAE9pgQkeQ0vonfhAkTMG/ePOTn5yvL8vPzsWDBAkyYMEGjfS1fvhxjx45FWFgY2rRpg7Vr10Iul2Pjxo3l1t+4cSPS0tKwe/dudOvWDS4uLvDy8oKbm5umh0FEREQ6Sq2em8GDB6ss//bbb2jUqJEyqbh48SIUCgV8fHzUblihUODcuXOYPn26skxPTw++vr44ceJEudvs2bMHnp6eGD9+PH766SfY2tpixIgRmDp1KvT19cvdJj8/XyURy8zMVDtGIiIiev6oldxYWlqqLL/22msqy1W5FDw1NRVFRUWwt7dXKbe3t8fff/9d7ja3b9/G77//juDgYPz666+4efMm3nnnHRQUFCAiIqLcbRYtWoS5c+dqHB8RERE9n9RKbr755pvqjkMtxcXFsLOzw7p166Cvr48OHTrgv//+w6efflphcjN9+nSVR0ZkZmbyvjxEREQ6TLLH1trY2EBfXx/Jyckq5cnJyXBwcCh3mwYNGsDQ0FBlCKp169ZISkqCQqEod0KzsbExjI2NtRs8ERER1VpVeir4jz/+iGHDhqFLly5o3769yktdRkZG6NChA2JiYpRlxcXFiImJUd4/50ndunXDzZs3UVxcrCy7fv06GjRooPGVWkRERKSbNE5uPv/8c4SFhcHe3h5xcXHo3Lkz6tevj9u3b6vc+0Yd4eHhWL9+PTZt2oSrV6/i7bffRnZ2NsLCwgAAo0aNUplw/PbbbyMtLQ2TJk3C9evXsXfvXixcuBDjx4/X9DCIiIhIR2k8LPXll19i3bp1CAoKwrfffosPP/wQTZs2xezZs5GWlqbRvgIDA3H//n3Mnj0bSUlJaNeuHfbv36+cZJyQkAA9vf/Pv5ycnHDgwAG89957aNu2LRwdHTFp0iRMnTpV08MgIiIiHaVxcpOQkICuXbsCAExMTJCVlQUAGDlyJLp06YJVq1ZptL8JEyZUeH+cQ4cOlSnz9PTEyZMnNQuaiIiI6gyNh6UcHByUPTSNGzdWJhrx8fEQRVG70RERERFpSOPkpmfPntizZw8AICwsDO+99x569eqFwMBADBo0SOsBEhEREWlC42GpdevWKa9WGj9+POrXr4/jx4/j1VdfxVtvvaX1AImIiIg0oXFyo6enpzLJd/jw4Rg+fLhWgyIiIiKqqirdxO/hw4f4+uuvcfXqVQBAmzZtEBYWhnr16mk1OCIiIiJNaTzn5siRI2jSpAk+//xzPHz4EA8fPsTnn3+OJk2a4MiRI9URIxEREZHaNO65GT9+PIYNG4Y1a9YoH4NQVFSEd955B+PHj8eff/6p9SCJiIiI1KVxz83Nmzfx/vvvqzzfSV9fH+Hh4bh586ZWgyMiIiLSlMbJTfv27ZVzbR539epVuLm5aSUoIiIioqpSa1jq0qVLyvfvvvsuJk2ahJs3b6JLly4AgJMnT2L16tX45JNPqidKIiIiIjWpldy0a9cOgiCo3IH4ww8/LFNvxIgRCAwM1F50RERERBpSK7mJj4+v7jiIiIiItEKt5MbZ2bm64yAi0prcwlwJ2iwCBAUgGtZ420Skqko38bt16xZWrFihchO/SZMmoVmzZloNjoioKry3eUvSrnkroDDHGdn5vpK0D0Uh5P97m6MoBFAoTRx1nImhPgRBkDqMOk3j5ObAgQN49dVX0a5dO3Tr1g0AcOzYMbz44ov4+eef0atXL60HSUT0NCYGJnC3c0dcSpykcRjI/0GnhfsA0ajG2zZBHq7KSt53mP8bciGr8RgI6Ohsje3jPJngSEjj5GbatGl47733ylwZNW3aNEydOpXJDRFJQhAEbPLfJMmQFADkFOTgle2vSNI21S5n/3mI3IIiyI2qNDhCWqDxmb969Sq2bdtWpnz06NFYsWKFNmIiIqoSQRAgN5Q/vWI1OzfLFyYGJjXfsCIbWPq/GD7yBYxMaz6GOixHUYSO83+TOgxCFZIbW1tbXLhwAS1atFApv3DhAuzs7LQWGBHR88rEUB9yQyn+a///NuVGBgB7DqiO0viTP3bsWLz55pu4ffs2unbtCqBkzs3ixYsRHh6u9QCJiIiINKFxcjNr1iyYm5tj2bJlmD59OgCgYcOGmDNnDt59912tB0hERESkCY2Sm8LCQkRGRmLEiBF47733kJWVBQAwNzevluCIiKiKFDnStW0oB3ilEElIo+TGwMAA48aNU97fhkkNEVEttbS5dG07dQFG72eCQ5LR+KngnTt3RlyctPeRICKichjKSxILqd09CRRI2HNEdZ7Gc27eeecdvP/++/j333/RoUMHmJqqXmrYtm1brQVHREQaEISSHhOpEgtFjrQ9RkT/o3FyM3z4cABQmTxc+sRwQRBQVFSkveiIiEgzgsD721Cdp3FywyeEExERUW2mUXKTmZmJ69evQ6FQoHPnzrC1ta2uuIiIiIiqRO3k5sKFC+jbty+Sk5MhiiLMzc2xbds2+Pn5VWd8RERERBpR+2qpqVOnokmTJvjjjz9w7tw5+Pj4YMKECdUZGxEREZHG1O65OXfuHA4ePIj27dsDADZu3Ih69eohMzMTFhYW1RYgERERkSbU7rlJS0tDo0aNlMtWVlYwNTXFgwcPqiUwIiIioqrQaELxlStXkJSUpFwWRRFXr15VPoYB4H1uiIiISFoaJTc+Pj4QRVGlrH///rzPDREREdUaaic3vL8NERERPQ/UTm6cnZ2rMw4iIiIirdD4wZlEREREtRmTGyIiItIpTG6IiIhIp2j84EwiIqpcbmGupO2bGJhAEARJYyCSEpMbIiIt897mLWn77nbu2OS/iQkO1VlqJTfu7u5q/5KcP3/+mQIiInoemRiYwN3OHXEpcVKHgriUOOQW5kJuKJc6FCJJqJXcDBw4sJrDICJ6vgmCgE3+myQdksotzJW814ioNlAruYmIiKjuOIiInnuCILC3hKgW4NVSREREpFM0nlBcVFSEzz77DNu2bUNCQgIUCoXK+rS0NK0FR0RERKQpjXtu5s6di+XLlyMwMBAZGRkIDw/H4MGDoaenhzlz5lRDiERERETq0zi52bx5M9avX4/3338fBgYGCAoKwoYNGzB79mycPHmyOmIkIiIiUpvGyU1SUhJcXV0BAGZmZsjIyAAA9O/fH3v37tVudEREREQa0ji5adSoERITEwEAzZo1w8GDBwEAZ86cgbGxsXajIyIiItKQxsnNoEGDEBMTAwCYOHEiZs2ahRYtWmDUqFEYPXq01gMkIiIi0oTGV0t98sknyveBgYFwdnbG8ePH0aJFCwQEBGg1OCIiIiJNaZzc5OXlQSaTKZe7dOmCLl26aDUoIiIioqrSeFjKzs4OISEhiI6ORnFxcXXERERERFRlGic3mzZtQk5ODgYMGABHR0dMnjwZZ8+erY7YiIiIiDRWpQnF27dvR3JyMhYuXIgrV66gS5cuaNmyJT7++OPqiJGIiIhIbVV+tpS5uTnCwsJw8OBBXLp0Caamppg7d642YyMiIiLSWJWTm7y8PGzbtg0DBw5E+/btkZaWhg8++KBK+1q9ejVcXFwgk8ng4eGB06dPq7Xd1q1bIQgCBg4cWKV2iYiISPdonNwcOHAAISEhsLe3x9tvvw17e3scPHgQ//zzj8pl4uqKiopCeHg4IiIicP78ebi5ucHPzw8pKSmVbnfnzh1MmTIF3bt317hNIiIi0l1VmnOTm5uL7777DklJSfjqq6/Qo0ePKgewfPlyjB07FmFhYWjTpg3Wrl0LuVyOjRs3VrhNUVERgoODMXfuXDRt2rTKbRMREZHu0fg+N8nJyTA3N9dK4wqFAufOncP06dOVZXp6evD19cWJEycq3O7jjz+GnZ0dxowZg6NHj2olFiIiItINaiU3mZmZsLCwAACIoojMzMwK65bWU0dqaiqKiopgb2+vUm5vb4+///673G3++OMPfP3117hw4YJabeTn5yM/P1+5XFnsRERE9PxTK7mxtrZGYmIi7OzsYGVlBUEQytQRRRGCIKCoqEjrQZbKysrCyJEjsX79etjY2Ki1zaJFi3gVFxERUR2iVnLz+++/o169esr35SU3VWFjYwN9fX0kJyerlCcnJ8PBwaFM/Vu3buHOnTsqz7AqvUuygYEBrl27hmbNmqlsM336dISHhyuXMzMz4eTkpJX4iYiIqPZRK7nx8vJSvvf29tZa40ZGRujQoQNiYmKUl3MXFxcjJiYGEyZMKFO/VatW+PPPP1XKPvroI2RlZWHlypXlJi3GxsYwNjbWWsxERERUu2k8obhFixYIDg5GcHAwWrRo8cwBhIeHIyQkBB07dkTnzp2xYsUKZGdnIywsDAAwatQoODo6YtGiRZDJZHjppZdUtreysgKAMuVERHVZbmFuzTdamAsIAkxEEYIip+bbL2UoB7Q0wkDPJ42Tm3feeQeRkZGYN28e2rdvj9dffx2BgYHlDiOpIzAwEPfv38fs2bORlJSEdu3aYf/+/cpJxgkJCdDTq/K9BomI6iTvbd7SNOziBPe8PGxa2hySpRdOXYDR+5ng1GGCKIpiVTa8fv06Nm/ejC1btiA+Ph6vvPIKXn/9dYwaNUrbMWpVZmYmLC0tkZGRodGVXUREtZ0oigjZH4K4lDipQ8GpO3chr9rXi3bMuAcYmdZokzmKQrSZfQAAcOVjP8iNNO4/oEpo8v1d5eTmcSdPnsTbb7+NS5cuVevVUtrA5IaIdJkoitIMSaFkKKy0x+jU0FjIDUxqNgBFDrC0ecl7Jjc6R5Pv72c686dPn0ZkZCSioqKQmZmJoUOHPsvuiIjoGQmCALmhXOowSua91IY4qE7SOLl5cjiqZ8+eWLx4MQYPHgwzM7PqiJGIiIhIbRonN61atUKnTp0wfvx4DB8+vMzdhYmIiIikpFFyU1RUhK+++gpDhgyBtbV1dcVEREREVGUaXWOtr6+PiRMnIj09vZrCISIiIno2Gt9A5qWXXsLt27erIxYiIiKiZ6ZxcjN//nxMmTIFv/zyCxITE5GZmanyIiIiIpKSxhOK+/btCwB49dVXVR6gWRNPBSciIiJ6Go2Tm9jY2OqIg4iIiEgrNE5uHn9COBEREVFto3Fyc+TIkUrX9+jRo8rBEBERET0rjZMbb2/vMmWPz73hnBsiIiKSksZXSz18+FDllZKSgv3796NTp044ePBgdcRIREREpDaNe24sLS3LlPXq1QtGRkYIDw/HuXPntBIYERERUVVo3HNTEXt7e1y7dk1buyMiIiKqEo17bi5duqSyLIoiEhMT8cknn6Bdu3baiouIiIioSjRObtq1awdBECCKokp5ly5dsHHjRq0FRkRERFQVGic38fHxKst6enqwtbWFTCbTWlBEREREVaVxcuPs7FwdcRARERFphdoTik+cOIFffvlFpey7775DkyZNYGdnhzfffBP5+flaD5CIiIhIE2onNx9//DEuX76sXP7zzz8xZswY+Pr6Ytq0afj555+xaNGiagmSiIiISF1qJzcXLlyAj4+Pcnnr1q3w8PDA+vXrER4ejs8//xzbtm2rliCJiIiI1KX2nJuHDx/C3t5euXz48GH06dNHudypUyfcvXtXu9EREdFzKbcwt+YbLcwFBAEmogjh6bVJh6md3Njb2yM+Ph5OTk5QKBQ4f/485s6dq1yflZUFQ0PDagmSiIieL97bvKVp2MUJ7nl52MQEp05Te1iqb9++mDZtGo4ePYrp06dDLpeje/fuyvWXLl1Cs2bNqiVIIiKq/UwMTOBu5y51GIiTyZBblCd1GCQhtXtu5s2bh8GDB8PLywtmZmbYtGkTjIyMlOs3btyI3r17V0uQRERU+wmCgE3+m6QZkgKQm/sA3rv6StI21S5qJzc2NjY4cuQIMjIyYGZmBn19fZX127dvh5mZmdYDJCKi54cgCJAbyqVpvCBHmnap1tHKU8EBoF69es8cDBEREdGz0tpTwYmIiIhqAyY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU7R+D43REREVLkcRZHUIUjKxFAfgiDd072Y3BAREWlZx/m/SR2CpDo6W2P7OE/JEhwOSxEREWmBiaE+OjpbSx1GrXD2n4fILZCu94o9N0RERFogCAK2j/OU9EtdajmKolrRa8XkhoiIdI8iBzDIrvFmBaDkwaESzjchJjdERKSLVrYFRFGatp26AKP3M8GREOfcEBGRbjAwkTqCEndPAgU5UkdRp7HnhoiIdMPjPSUf3Kz5ZEeRAyxtXrNtUrmY3BARke4xlJe8qE7isBQRERHpFCY3REREpFOY3BAREZFO4ZwbIiLSObmFuTXfaGEuIAgwEUXwInBpMbkhIiKd473NW5qGXZzgnpeHTUxwJMVhKSIi0gkmBiZwt3OXOgzEyWTILcqTOow6jT03RESkEwRBwCb/TdIMSQHIzX0A7119JWmbVDG5ISIinSEIQsmznaTAuxLXGhyWIiIiIp3C5IaIiIh0CpMbIiIi0im1IrlZvXo1XFxcIJPJ4OHhgdOnT1dYd/369ejevTusra1hbW0NX1/fSusTERFR3SJ5chMVFYXw8HBERETg/PnzcHNzg5+fH1JSUsqtf+jQIQQFBSE2NhYnTpyAk5MTevfujf/++6+GIyciIqLaSPLkZvny5Rg7dizCwsLQpk0brF27FnK5HBs3biy3/ubNm/HOO++gXbt2aNWqFTZs2IDi4mLExMTUcORERERUG0ma3CgUCpw7dw6+vr7KMj09Pfj6+uLEiRNq7SMnJwcFBQWoV69edYVJREREzxFJ73OTmpqKoqIi2Nvbq5Tb29vj77//VmsfU6dORcOGDVUSpMfl5+cjPz9fuZyZmVn1gImIiKjWk3xY6ll88skn2Lp1K3bt2gWZTFZunUWLFsHS0lL5cnJyquEoiYiIqCZJmtzY2NhAX18fycnJKuXJyclwcHCodNulS5fik08+wcGDB9G2bdsK602fPh0ZGRnK1927d7USOxEREdVOkiY3RkZG6NChg8pk4NLJwZ6enhVut2TJEsybNw/79+9Hx44dK23D2NgYFhYWKi8iIiLSXZI/Wyo8PBwhISHo2LEjOnfujBUrViA7OxthYWEAgFGjRsHR0RGLFi0CACxevBizZ89GZGQkXFxckJSUBAAwMzODmZmZZMdBREREtYPkyU1gYCDu37+P2bNnIykpCe3atcP+/fuVk4wTEhKgp/f/HUxr1qyBQqHAkCFDVPYTERGBOXPm1GToREREVAtJntwAwIQJEzBhwoRy1x06dEhl+c6dO9UfEBERET23nuurpYiIiIiexOSGiIiIdAqTGyIiItIpTG6IiIhIpzC5ISIiIp3C5IaIiIh0CpMbIiIi0im14j43REREuiS3MBcoyJGsfRMDEwiCIFn7UmNyQ0REpGXeu/pK2r67nTs2+W+qswkOh6WIiIi0wERfBve8PKnDAADEpcSV9B7VUey5ISIi0gJBELApMQW5ggB8cBMwlNd4DLmFufDe5l3j7dY2TG6IiIi0RAAgF0XAwESS5IZKcFiKiIiIdAqTGyIiItIpTG6IiIhIpzC5ISIiIp3CCcVERETappDoBn51+PLvxzG5ISIi0ralzaVpVxAAF6eS96IoTQy1AIeliIiItMFQDjh1kTqK/1eHe3HYc0NERKQNggCM3i/pM6WQ8wD4qZ907dcSTG6IiIi0RRAAI1Pp2q/DvTWP47AUERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckNEREQ6hQ/OJCIi0kG5hbk1/oTy3MIiQFAAomGNtvskJjdEREQ6yHtXX0naNW8FFOY4QxT9JGkf4LAUERGRzjDRl8E9L0/qMGAg/wd5RdLFwZ4bIiIiHSEIAjYlpiBXEIAPbgKG8hptPy33Efrs8qnRNsvD5IaIiEiHCADkoggYmNR4cpNbUFSj7VWEw1JERESkU5jcEBERkU5hckNEREQ6hckNERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckNEREQ6hckNERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqlViQ3q1evhouLC2QyGTw8PHD69OlK62/fvh2tWrWCTCaDq6srfv311xqKlIiIiGo7yZObqKgohIeHIyIiAufPn4ebmxv8/PyQkpJSbv3jx48jKCgIY8aMQVxcHAYOHIiBAwfir7/+quHIiYiIqDaSPLlZvnw5xo4di7CwMLRp0wZr166FXC7Hxo0by62/cuVK+Pv744MPPkDr1q0xb948tG/fHqtWrarhyImIiKg2kjS5USgUOHfuHHx9fZVlenp68PX1xYkTJ8rd5sSJEyr1AcDPz6/C+vn5+cjMzFR5ERERke6SNLlJTU1FUVER7O3tVcrt7e2RlJRU7jZJSUka1V+0aBEsLS2VLycnJ+0ET0RERLWSgdQBVLfp06cjPDxcuZyZmckEh4iIdJOhHJhx7//f1zBrmSkODT2ufC8VSZMbGxsb6OvrIzk5WaU8OTkZDg4O5W7j4OCgUX1jY2MYGxtrJ2AiIqLaTBAAI+mSCj09PdSXm0vWvjIOKRs3MjJChw4dEBMToywrLi5GTEwMPD09y93G09NTpT4AREdHV1ifiIiI6hbJh6XCw8MREhKCjh07onPnzlixYgWys7MRFhYGABg1ahQcHR2xaNEiAMCkSZPg5eWFZcuWoV+/fti6dSvOnj2LdevWSXkYREREVEtIntwEBgbi/v37mD17NpKSktCuXTvs379fOWk4ISEBenr/38HUtWtXREZG4qOPPsKMGTPQokUL7N69Gy+99JJUh0BERES1iCCKoih1EDUpMzMTlpaWyMjIgIWFhdThEBERkRo0+f6W/CZ+RERERNrE5IaIiIh0CpMbIiIi0ilMboiIiEinMLkhIiIincLkhoiIiHQKkxsiIiLSKUxuiIiISKcwuSEiIiKdIvnjF2pa6Q2ZMzMzJY6EiIiI1FX6va3OgxXqXHKTlZUFAHBycpI4EiIiItJUVlYWLC0tK61T554tVVxcjHv37sHc3ByCIGh135mZmXBycsLdu3fr5HOr6vrxAzwHPP66ffwAz0FdP36g+s6BKIrIyspCw4YNVR6oXZ4613Ojp6eHRo0aVWsbFhYWdfZDDfD4AZ4DHn/dPn6A56CuHz9QPefgaT02pTihmIiIiHQKkxsiIiLSKUxutMjY2BgREREwNjaWOhRJ1PXjB3gOePx1+/gBnoO6fvxA7TgHdW5CMREREek29twQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3GjJ6tWr4eLiAplMBg8PD5w+fVrqkKrNkSNHEBAQgIYNG0IQBOzevVtlvSiKmD17Nho0aAATExP4+vrixo0b0gRbDRYtWoROnTrB3NwcdnZ2GDhwIK5du6ZSJy8vD+PHj0f9+vVhZmaG1157DcnJyRJFrF1r1qxB27ZtlTfo8vT0xL59+5TrdfnYy/PJJ59AEARMnjxZWabr52DOnDkQBEHl1apVK+V6XT/+Uv/99x9ef/111K9fHyYmJnB1dcXZs2eV63X5b6GLi0uZz4AgCBg/fjwA6T8DTG60ICoqCuHh4YiIiMD58+fh5uYGPz8/pKSkSB1atcjOzoabmxtWr15d7volS5bg888/x9q1a3Hq1CmYmprCz88PeXl5NRxp9Th8+DDGjx+PkydPIjo6GgUFBejduzeys7OVdd577z38/PPP2L59Ow4fPox79+5h8ODBEkatPY0aNcInn3yCc+fO4ezZs+jZsycGDBiAy5cvA9DtY3/SmTNn8NVXX6Ft27Yq5XXhHLz44otITExUvv744w/lurpw/A8fPkS3bt1gaGiIffv24cqVK1i2bBmsra2VdXT5b+GZM2dUfv7R0dEAgKFDhwKoBZ8BkZ5Z586dxfHjxyuXi4qKxIYNG4qLFi2SMKqaAUDctWuXcrm4uFh0cHAQP/30U2VZenq6aGxsLG7ZskWCCKtfSkqKCEA8fPiwKIolx2toaChu375dWefq1asiAPHEiRNShVmtrK2txQ0bNtSpY8/KyhJbtGghRkdHi15eXuKkSZNEUawbP/+IiAjRzc2t3HV14fhFURSnTp0qvvzyyxWur2t/CydNmiQ2a9ZMLC4urhWfAfbcPCOFQoFz587B19dXWaanpwdfX1+cOHFCwsikER8fj6SkJJXzYWlpCQ8PD509HxkZGQCAevXqAQDOnTuHgoIClXPQqlUrNG7cWOfOQVFREbZu3Yrs7Gx4enrWqWMfP348+vXrp3KsQN35+d+4cQMNGzZE06ZNERwcjISEBAB15/j37NmDjh07YujQobCzs4O7uzvWr1+vXF+X/hYqFAr88MMPGD16NARBqBWfASY3zyg1NRVFRUWwt7dXKbe3t0dSUpJEUUmn9JjryvkoLi7G5MmT0a1bN7z00ksASs6BkZERrKysVOrq0jn4888/YWZmBmNjY4wbNw67du1CmzZt6sSxA8DWrVtx/vx5LFq0qMy6unAOPDw88O2332L//v1Ys2YN4uPj0b17d2RlZdWJ4weA27dvY82aNWjRogUOHDiAt99+G++++y42bdoEoG79Ldy9ezfS09MRGhoKoHb8DtS5p4ITadP48ePx119/qcw3qAteeOEFXLhwARkZGfjxxx8REhKCw4cPSx1Wjbh79y4mTZqE6OhoyGQyqcORRJ8+fZTv27ZtCw8PDzg7O2Pbtm0wMTGRMLKaU1xcjI4dO2LhwoUAAHd3d/z1119Yu3YtQkJCJI6uZn399dfo06cPGjZsKHUoSuy5eUY2NjbQ19cvMws8OTkZDg4OEkUlndJjrgvnY8KECfjll18QGxuLRo0aKcsdHBygUCiQnp6uUl+XzoGRkRGaN2+ODh06YNGiRXBzc8PKlSvrxLGfO3cOKSkpaN++PQwMDGBgYIDDhw/j888/h4GBAezt7XX+HDzJysoKLVu2xM2bN+vEZwAAGjRogDZt2qiUtW7dWjk8V1f+Fv7zzz/47bff8MYbbyjLasNngMnNMzIyMkKHDh0QExOjLCsuLkZMTAw8PT0ljEwaTZo0gYODg8r5yMzMxKlTp3TmfIiiiAkTJmDXrl34/fff0aRJE5X1HTp0gKGhoco5uHbtGhISEnTmHDypuLgY+fn5deLYfXx88Oeff+LChQvKV8eOHREcHKx8r+vn4EmPHj3CrVu30KBBgzrxGQCAbt26lbkFxPXr1+Hs7AygbvwtBIBvvvkGdnZ26Nevn7KsVnwGamTaso7bunWraGxsLH777bfilStXxDfffFO0srISk5KSpA6tWmRlZYlxcXFiXFycCEBcvny5GBcXJ/7zzz+iKIriJ598IlpZWYk//fSTeOnSJXHAgAFikyZNxNzcXIkj1463335btLS0FA8dOiQmJiYqXzk5Oco648aNExs3biz+/vvv4tmzZ0VPT0/R09NTwqi1Z9q0aeLhw4fF+Ph48dKlS+K0adNEQRDEgwcPiqKo28dekcevlhJF3T8H77//vnjo0CExPj5ePHbsmOjr6yva2NiIKSkpoijq/vGLoiiePn1aNDAwEBcsWCDeuHFD3Lx5syiXy8UffvhBWUfX/xYWFRWJjRs3FqdOnVpmndSfASY3WvLFF1+IjRs3Fo2MjMTOnTuLJ0+elDqkahMbGysCKPMKCQkRRbHkEshZs2aJ9vb2orGxsejj4yNeu3ZN2qC1qLxjByB+8803yjq5ubniO++8I1pbW4tyuVwcNGiQmJiYKF3QWjR69GjR2dlZNDIyEm1tbUUfHx9lYiOKun3sFXkyudH1cxAYGCg2aNBANDIyEh0dHcXAwEDx5s2byvW6fvylfv75Z/Gll14SjY2NxVatWonr1q1TWa/rfwsPHDggAij3mKT+DAiiKIo100dEREREVP0454aIiIh0CpMbIiIi0ilMboiIiEinMLkhIiIincLkhoiIiHQKkxsiIiLSKUxuiIiISKcwuSGi50poaCgGDhwodRhEVIvxqeBEVGsIglDp+oiICKxcuRK89ygRVYbJDRHVGomJicr3UVFRmD17tsrDCc3MzGBmZiZFaET0HOGwFBHVGg4ODsqXpaUlBEFQKTMzMyszLOXt7Y2JEydi8uTJsLa2hr29PdavX4/s7GyEhYXB3NwczZs3x759+1Ta+uuvv9CnTx+YmZnB3t4eI0eORGpqag0fMRFVByY3RPTc27RpE2xsbHD69GlMnDgRb7/9NoYOHYquXbvi/Pnz6N27N0aOHImcnBwAQHp6Onr27Al3d3ecPXsW+/fvR3JyMoYNGybxkRCRNjC5IaLnnpubGz766CO0aNEC06dPh0wmg42NDcaOHYsWLVpg9uzZePDgAS5dugQAWLVqFdzd3bFw4UK0atUK7u7u2LhxI2JjY3H9+nWJj4aInhXn3BDRc69t27bK9/r6+qhfvz5cXV2VZfb29gCAlJQUAMDFixcRGxtb7vydW7duoWXLltUcMRFVJyY3RPTcMzQ0VFkWBEGlrPQqrOLiYgDAo0ePEBAQgMWLF5fZV4MGDaoxUiKqCUxuiKjOad++PXbs2AEXFxcYGPDPIJGu4ZwbIqpzxo8fj7S0NAQFBeHMmTO4desWDhw4gLCwMBQVFUkdHhE9IyY3RFTnNGzYEMeOHUNRURF69+4NV1dXTJ48GVZWVtDT459FouedIPJWn0RERKRD+C8KERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckNEREQ65f8AopLGRFalu1QAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "for site, result in data.items():\n", + " timeline = result[\"timeline\"]\n", + " km_estimate = result[\"km_estimate\"]\n", + " plt.step(timeline, km_estimate, where=\"post\", label=f\"{site}: Survival Curve\")\n", + "\n", + "plt.xlabel(\"Time\")\n", + "plt.ylabel(\"Survival Probability\")\n", + "plt.title(\"Kaplan-Meier Survival Curve\")\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b43334bb-c77d-40f4-99b4-c2f9adb37d7e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c760a0ba-5450-4ae8-9c2d-774c9bf84515", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nvflare_example", + "language": "python", + "name": "nvflare_example" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/hello-world/hello-km/demo/km.json b/examples/hello-world/hello-km/demo/km.json new file mode 100644 index 0000000000..39dd3694f3 --- /dev/null +++ b/examples/hello-world/hello-km/demo/km.json @@ -0,0 +1,172 @@ +{ + "site-2": { + "timeline": [ + 0.0, + 10.0, + 25.0, + 30.0, + 40.0, + 50.0, + 60.0, + 70.0 + ], + "km_estimate": [ + 1.0, + 0.8571428571428572, + 0.7142857142857143, + 0.7142857142857143, + 0.5357142857142858, + 0.5357142857142858, + 0.26785714285714285, + 0.0 + ], + "event_count": [ + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "survival_rate": [ + 0.0, + 0.1428571428571428, + 0.2857142857142857, + 0.2857142857142857, + 0.4642857142857142, + 0.4642857142857142, + 0.7321428571428572, + 1.0 + ] + }, + "site-1": { + "timeline": [ + 0.0, + 5.0, + 10.0, + 15.0, + 25.0, + 30.0, + 35.0, + 40.0, + 45.0, + 50.0, + 55.0, + 60.0, + 65.0 + ], + "km_estimate": [ + 1.0, + 0.9166666666666667, + 0.9166666666666667, + 0.8250000000000001, + 0.7333333333333331, + 0.6416666666666665, + 0.6416666666666665, + 0.6416666666666665, + 0.5133333333333332, + 0.385, + 0.25666666666666665, + 0.12833333333333333, + 0.0 + ], + "event_count": [ + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "survival_rate": [ + 0.0, + 0.08333333333333326, + 0.08333333333333326, + 0.17499999999999993, + 0.26666666666666694, + 0.3583333333333335, + 0.3583333333333335, + 0.3583333333333335, + 0.4866666666666668, + 0.615, + 0.7433333333333334, + 0.8716666666666667, + 1.0 + ] + }, + "global": { + "timeline": [ + 0.0, + 5.0, + 10.0, + 15.0, + 25.0, + 30.0, + 35.0, + 40.0, + 45.0, + 50.0, + 55.0, + 60.0, + 65.0, + 70.0 + ], + "km_estimate": [ + 1.0, + 0.9230769230769231, + 0.8461538461538463, + 0.7692307692307694, + 0.6923076923076924, + 0.6153846153846153, + 0.5384615384615384, + 0.4615384615384615, + 0.3846153846153846, + 0.30769230769230765, + 0.23076923076923078, + 0.15384615384615385, + 0.07692307692307693, + 0.0 + ], + "event_count": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "survival_rate": [ + 0.0, + 0.07692307692307687, + 0.15384615384615374, + 0.23076923076923062, + 0.3076923076923076, + 0.3846153846153847, + 0.46153846153846156, + 0.5384615384615385, + 0.6153846153846154, + 0.6923076923076923, + 0.7692307692307692, + 0.8461538461538461, + 0.9230769230769231, + 1.0 + ] + } +} \ No newline at end of file diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf new file mode 100644 index 0000000000..9de6ad8d7c --- /dev/null +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf @@ -0,0 +1,116 @@ +{ + # version of the configuration + format_version = 2 + + # This is the application script which will be invoked. Client can replace this script with user's own training script. + app_script = "km_train.py" + + # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. + app_config = "" + + # Client Computing Executors. + executors = [ + { + # tasks the executors are defined to handle + tasks = ["train"] + + # This particular executor + executor { + + # This is an executor for Client API. The underline data exchange is using Pipe. + path = "nvflare.app_opt.pt.client_api_launcher_executor.ClientAPILauncherExecutor" + + args { + # launcher_id is used to locate the Launcher object in "components" + launcher_id = "launcher" + + # pipe_id is used to locate the Pipe object in "components" + pipe_id = "pipe" + + # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. + # Please refer to the class docstring for all available arguments + heartbeat_timeout = 60 + + # format of the exchange parameters + params_exchange_format = "raw" + + # if the transfer_type is FULL, then it will be sent directly + # if the transfer_type is DIFF, then we will calculate the + # difference VS received parameters and send the difference + params_transfer_type = "FULL" + + # if train_with_evaluation is true, the executor will expect + # the custom code need to send back both the trained parameters and the evaluation metric + # otherwise only trained parameters are expected + train_with_evaluation = false + } + } + } + ], + + # this defined an array of task data filters. If provided, it will control the data from server controller to client executor + task_data_filters = [] + + # this defined an array of task result filters. If provided, it will control the result from client executor to server controller + task_result_filters = [] + + components = [ + { + # component id is "launcher" + id = "launcher" + + # the class path of this component + path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" + + args { + # the launcher will invoke the script + script = "python3 custom/{app_script} {app_config} " + # if launch_once is true, the SubprocessLauncher will launch once for the whole job + # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server + launch_once = true + } + } + { + id = "pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + } + { + id = "metrics_pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + }, + { + id = "metric_relay" + path = "nvflare.app_common.widgets.metric_relay.MetricRelay" + args { + pipe_id = "metrics_pipe" + event_type = "fed.analytix_log_stats" + # how fast should it read from the peer + read_interval = 0.1 + } + }, + { + # we use this component so the client api `flare.init()` can get required information + id = "config_preparer" + path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" + args { + component_ids = ["metric_relay"] + } + } + ] +} diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf new file mode 100644 index 0000000000..7765396787 --- /dev/null +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf @@ -0,0 +1,24 @@ +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "km" + path = "nvflare.app_common.workflows.wf_controller.WFController" + args { + task_name = "train" + wf_class_path = "kaplan_meier.KM", + wf_args { + min_clients = 2 + output_path = "/tmp/nvflare/km/km.json" + } + } + } + ] + + components = [] + +} diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py new file mode 100644 index 0000000000..0ca1b24d94 --- /dev/null +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py @@ -0,0 +1,110 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os.path +from typing import Dict + +from km_analysis import kaplan_meier_analysis + +from nvflare.app_common.abstract.fl_model import FLModel +from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( + CURRENT_ROUND, + DATA, + MIN_RESPONSES, + NUM_ROUNDS, + START_ROUND, +) +from nvflare.app_common.workflows.wf_comm.wf_spec import WF + +# Controller Workflow + + +class KM(WF): + def __init__(self, min_clients: int, output_path: str): + super(KM, self).__init__() + self.logger = logging.getLogger(self.__class__.__name__) + self.output_path = output_path + self.min_clients = min_clients + self.num_rounds = 1 + self.flare_comm = WFCommAPI() + + def run(self): + results = self.start_km_analysis() + global_res = self.aggr_km_result(results) + self.save(global_res, self.output_path) + + def start_km_analysis(self): + self.logger.info("send kaplan-meier analysis command to all sites \n") + + msg_payload = { + MIN_RESPONSES: self.min_clients, + CURRENT_ROUND: 1, + NUM_ROUNDS: self.num_rounds, + START_ROUND: 1, + DATA: {}, + } + + results = self.flare_comm.broadcast_and_wait(msg_payload) + return results + + def aggr_km_result(self, sag_result: Dict[str, Dict[str, FLModel]]): + + self.logger.info("aggregate kaplan-meier analysis results \n") + + if not sag_result: + raise RuntimeError("input is None or empty") + + task_name, task_result = next(iter(sag_result.items())) + + if not task_result: + raise RuntimeError("task_result None or empty ") + + global_result: dict = {} + all_result = {} + for site, fl_model in task_result.items(): + result = fl_model.params + all_result[site] = result + timelines = result.get("timeline") + event_counts = result.get("event_count") + combined_arrays = list(zip(timelines, event_counts)) + g_timelines = global_result.get("timeline", []) + g_event_counts = global_result.get("event_count", {}) + for t, count in combined_arrays: + if t not in g_timelines: + g_timelines.append(t) + g_event_counts[t] = count + else: + prev_count = g_event_counts.get(t) + g_event_counts[t] = prev_count + count + global_result["event_count"] = g_event_counts + global_result["timeline"] = g_timelines + + g_duration = global_result.get("timeline", []) + g_event_counts = list(global_result.get("event_count").values()) + + g_km_result = kaplan_meier_analysis(g_duration, g_event_counts) + + all_result["global"] = g_km_result + return all_result + + def save(self, result: dict, file_path: str): + self.logger.info(f"save the result to {file_path} \n") + + dir_name = os.path.dirname(file_path) + os.makedirs(dir_name, exist_ok=True) + with open(file_path, "w") as json_file: + json.dump(result, json_file, indent=4) diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py new file mode 100644 index 0000000000..542d316c47 --- /dev/null +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py @@ -0,0 +1,46 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lifelines import KaplanMeierFitter + + +def kaplan_meier_analysis(duration, event): + # Create a Kaplan-Meier estimator + kmf = KaplanMeierFitter() + + # Fit the model + kmf.fit(durations=duration, event_observed=event) + + # Get the survival function at all observed time points + survival_function_at_all_times = kmf.survival_function_ + + # Get the timeline (time points) + timeline = survival_function_at_all_times.index.values + + # Get the KM estimate + km_estimate = survival_function_at_all_times["KM_estimate"].values + + # Get the event count at each time point + event_count = kmf.event_table.iloc[:, 0].values # Assuming the first column is the observed events + + # Get the survival rate at each time point (using the 1st column of the survival function) + survival_rate = 1 - survival_function_at_all_times.iloc[:, 0].values + + # Return the results + return { + "timeline": timeline.tolist(), + "km_estimate": km_estimate.tolist(), + "event_count": event_count.tolist(), + "survival_rate": survival_rate.tolist(), + } diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py new file mode 100644 index 0000000000..2aff5c5cc5 --- /dev/null +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd +from km_analysis import kaplan_meier_analysis + +# (1) import nvflare client API +import nvflare.client as flare +from nvflare.app_common.abstract.fl_model import FLModel, ParamsType + +# Client training code + + +def load_data(): + data = { + "site-1": { + "duration": [5, 10, 15, 25, 30, 35, 40, 45, 50, 55, 60, 65], + "event": [1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 2, 4], + }, + "site-2": {"duration": [10, 25, 30, 40, 50, 60, 70], "event": [1, 1, 0, 1, 0, 3, 4]}, + } + + return data + + +def display_results(results): + for time_point, km_estimate, event_count, survival_rate in zip( + results["timeline"], results["km_estimate"], results["event_count"], results["survival_rate"] + ): + print( + f"Time: {time_point}, KM Estimate: {km_estimate:.4f}, Event Count: {event_count}, Survival Rate: {survival_rate:.4f}" + ) + + +def main(): + flare.init() + + site_name = flare.get_site_name() + + df = pd.DataFrame(data=load_data())[site_name] + + while flare.is_running(): + + print(f"Kaplan-meier analysis for {site_name}") + + if flare.is_train(): + # Perform Kaplan-Meier analysis and get the results + results = kaplan_meier_analysis(duration=df["duration"], event=df["event"]) + + # Display the results + display_results(results) + print(f"send result for site = {flare.get_site_name()}") + model = FLModel(params=results, params_type=ParamsType.FULL) + flare.send(model) + + print(f"finish send for {site_name}, complete") + + +if __name__ == "__main__": + main() diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf b/examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf new file mode 100644 index 0000000000..5c81903a41 --- /dev/null +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf @@ -0,0 +1,7 @@ +{ + name = "fl_km" + deploy_map { + app = ["@ALL"] + } + min_clients = 2 +} diff --git a/examples/hello-world/hello-km/km_survival_curve.png b/examples/hello-world/hello-km/km_survival_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..b06564f129043e66cecdb5eabfc3a9231ad8072a GIT binary patch literal 61673 zcmeFZbySq^_cl5-k`f{ff>H{obPg)rjWkM2w^BnGbO?x2gLHQ{qICDr2skjLzz_or z=NbKczrW9U-*>IE)>-TPcm9D{Gf&*lzW2TFy|3%qPq?PK0x1za5eNh#RZ^7I27z!O zAP{y7AwKYpc?

5GV+wB>P17wb{-hURv&tq|@Jvr;CdsFjLm&pWfbWcx$0DDl@vm zcZ24st76=bP_n(f$96%W&PCHuf|(ngi}-ZeOXoDk6B-$f^#OSoCu!Gk9q3-$X?&3m zazVOXJRgD9A$s@pt0osU=HF=040y_%@Lyka=yaI>{S9-78i?qO_>ND{C_U<=V}}vyvzSNbQCu;sQsV20RK`z)Caf!=f+e)*y#TbLemag^Z(PEwiak* z%jd3keaJU34-jfS`x&_vN-nL~^`5ktg$9u`yVpeyR^@L!)<(@Qqd#5Stc{*c%$P&c zmmt?yf>yn8$KQzgbG|YOeFSZ$56I>l|C?tbj*P*0RKE-dZaiTCk zBV~J*=VkkoQ2gB1(2rl~{SsxtuK*A;U@KDRX+og`gYw^2ek_R209asc8fGs6sac*9$gp&UX z)4Q)!gDl97Q3f8tF~$0mp*zDx=&jt?ChJ6?XZk0&_ON1V+jjKP)guWcckVN$gZSLrvGW zC7mpIjHEdWyRY|NG7~1E2&!c726e_lF4t;)r8wAd?g6GMGj+NwoLVK%yc}r1R6)TZ zX!vm`%O&gl@m}*jGjU6#=nufqUMX*WeK``lKGwKSywg$-GNDd7Y6YG>r?hUa$Xp&B zR3=Rmj=I{1KtGwd_br?{KAun&$c5QLrFg4)+3Xj+X4EeBnqBaB^eI=wP8U2p*1dk* zGT0G`Z_ns}j*zcyHo`MhZkxD1ah~T7{2j@tl=k91-T3i@2Gl0isjg=9De*?7t-De) z7`2&=$_3U*LGB+1L*>qeg28f-7^#!l8o-P5`#4e`-9(Xxa3V)XD4emP6%s<0FEIsqME^>b(^Nv@+jg zB=$bD7j&0VTdZ|HH+AjAn*hvZ)~?$ANx1EF$!}{T*<5`@`ob|L2uXm%Mf(RzUSA$H zx5ABo*Zf8Lse6VlY%2QcJpVWI0tu0@pPb;$^+HSd4 z>eA=y_T6;0s(`iCV$evA%lHe#g$+oY&x+9tU$B)t26DA+p1GLbqAjAV-}OCh^#5> zj}}?_JY%f);|&U!%ZXH%HZhKuFKj#Vp@HK0zOXx*i52l4cq=8m)yDZ3=ks2m0tdx; zs$DEFvet>I6Dka1KRs>ol{#NfXf!L^_li%FAN>uiLO4p)?5_L; zP4nC$xCRI3}n%`UW zSsto;Bn-=Pem35|w?+$Z(x6j7Y@+sb5C!EW$sVS4KKt_Sq`Nk@J;|tMKI1}2W8H=L zu*X-IGy_O?L=z5V&)zYmrXohCNpDBWGIwFSWTce#tIWsn&dL zvp?F-obL8U%F=}Xt_V%Q`>p!{zhL9BW*$+{j9B?`sH0H^1(an9-b;HQ%B07@uxz%y z?HCU1T!FVRyAdiEabPe${xj` z0YwXr9I6b=13onYgu`^E0r5#!TvLXQUBEaH_G$&~Mx)@b85y}NBD>Qpl(zXe1d zOK;6$1R+|~Uu4b7{dJ=!Z5MryZ}Sg~K9Yc3B5;G4JLO9EA$_zdfhtP^N>MAHHno~5eP8R2?|?k8 zdzv<#%sCuw?-6)9R{fkCcu=j{R=$?gi(r5U=gT9=E*tZEmrmwgpq#={yGJ8naD6rG zD62HX0c6r)IuAT0{4$ERxuckGWaZ8W#*Y`@L~%`W$fQSF;B=#qjC)!y92`qu*btP5 zH%a^oV~??;_nL$X_ZT_VPL2Ocb;$rr49v-TovNM>{x-B3%G>taQ}(SIc`|~@WFGz# zi;E8G=bf9adlm*I2tIibZt}IA{md)ONPE=G>)i6Yn@(9&FHrdwo(Y@YQuAT|{hqm@ zj~)XQ>6S5KhwsT5ZO9li*EZ2Rw%~ynA%)imE1h(E%1q)R;4Q21-bp#Pw4X+xBug8n zkG@mEG@>y7pV#Q+>(q38hmVeI*io~1iSG1F69uN}TEXDm8D}r7r)l0~?%}veVKeRn zyvY|KR9oA$^qnl6pK(U|49sI%11(fKiuo6CT(@mybnkw=qdehQuK~dknL$d=uHxb- z5A(9(=K+xl1(pTXepS62X5#)cn53C!g*o%~ht@B%ow6a1xG9}f_1Qgkm6&9Gq6h+b zz7f4-xKSW3zf4acry==UHiHpsrTabzUv199#Ia%Z{hOgPut(n6C+t|V}LS)gV zaeZdd-WR$)mQ0$j<z=kFm1ULjMZkD`j|AHBFruiSx+YGhd= z2`zk9Mub%78hRj?J&^TuDJ(k!YyW2c)Q~;-HQC8oTiW^8(gVIvgoQfSTdz;A(dL2W zbG8}8CkGFL+V%tX{VcWT8E#NxYbM}uSjxUcMqdPpTP|1YWk@*j3-cvNLQoUX?v!32 zkHgXVn^zPOL9(#ZYCaZ@^;L9@tNjcxi?GXstBc>#LbP&j(&I1{h z1y1xpCEuV;Uw#lg;LLda^1bHnBpI^f95##S_*F}b>#9Xm|crub2@WD_c#{PDigUFGjK zNJ0;nE@*Tt(GICJV>qhK$X^xuo;<<8UO4HSJ`Ki)o}0D_D+8Hc4L?t6LvJZDE z%$1aA&T)=*Ov+qJ7z|LiMClb!#mudw!17Z0@A6`E?1&BR3x*$fuUEJQiOTCKbT3b{ z1mh30#Zfz9Gcb-6O?;34+3WE(w_UGYVpIr=+?FKKT{E}N?SVu{dzIx&twLKE2%F2b zyMw4_ur{LTAb&6IvMHP6C98GR(B%?E%cR~ra6GNkIC-YbUA<){XGIXYe0 z&(^FWzNtO>K`mAqn@uD?TS!gD%gK0v{VK-Q-T+8WZ!p@GW2LOcNbpIlOl=q8H1Igg z23ZFe^IR4Rblj%fiHRcJo~>JOAMls5=%lOHACP(yGO0;Xir{$MN(!0VG>V??r6X9; zKhAel)lq{cagd%(8s?A|Bzjp6t!P{g$VL=oR9XaB$GbT@g2)4`{B_%TI?qv!ZDf=| ziKPA|Nk=5g@nz*7cE~R2{4DzA`8gD2?VGcgLcmB7Zni^nHgCPH+t52JQKYIiiHi4{ z&;4)yzKujKECBh6=cxT@>UCiZJ3(xdwguE8)|)ES80u=9`1&rRzMRz{?9``s+IPbt5w z465|}w74a+Lx@)PjGHj=KGEUd-3Y@L*Sx2u@;!%Msqh=L07_`IXo#ydZpE~=bdPi- zKr9)VWvUlc@{J|zh`DFd_wJa)%2I zz(P3E31G)^O`rs6#lJN5W<;KM%g?dz!d~;8ARYatWxkG31wDrdQ;)={<;`~G z#E%Rr1$iMbA@TzE7#PR5SORKfrk~`opm5j*?q<`iK^uHzRum zTkaq-qS|;xqbR9X**e-#Pg@3yrKfIZkN^eb3Hy;5;b}^Go@Jh*UXw~;>wBS0Y0p2V=cix6yx0kK`-@Uch;cnp_2)cbkT5_cAto?4Dfx=B0T zCCpGucW67Sep>UDLo0Oye9+_Rz^?~GE9AP^wDC_VRu5a>BDZ-M*TOkcxK8Vlwz@0i zaY{2a(K~7`VPzI1T zesG`;*a0`14$3Jlu|4umn}c(9>_~6fl^dwvB>$=3crL?Fu*lRY2^5JEPes$CJ4!?i zw0CnV+}c2R1h)o*X&@fTVhT2GV!B>uYhmJbTJ+SeKcmEly+j++7GqXo*GPoLU}?h0 zm06f;XRD|WqnNsIeujs+?mO_9=!(9J#OswH9MPCN4Xz{#P#u>@zmo@eX%&`@>g;C| z%rid=in8sZ!rny-$awf2a7L_Um=V01+os>W^)Sc0lQvZt2y%LDl^`y)n_&%8<$4j~ z9^(TX9C&Ei;*M!7xasV}dSMTae!R-J?$y$*@8UVjB|7RmW zOF@L6VcAW?!b!p;-NHogBli(&XNtD&B%HAxgseH2N=$37RbcCJ?#yDpro7(bp+%b* zTjYcK;*lGN_YUEKS+qkd<)F^YC!vSi;|!&8IMLemlf^lKA>U$FMtoZkF)4DuO7|Ff z!$0;zJLhGDY|`Y2{DFbDBeiKl%&TNwEg9hl+ubRhET9Vg(d~};*4g^jm!>9$*SIJ2 zolvh2(cz>QMx>PA1RXhF4GlGiq|S z42LY3UwD%cZ8C!_GG{YS-cB|Jmm;yHJ8RnY(WP`X2)&_XIz(6+FdLr;~Sc3=e`MDO7>?@54`p_}{&uQ?!tFj86k=)-jw@5~8tGb)7^%S(v9oX&FN!J^?>$#kC$3f%_r z#Tfb>7))Qf6)=>Z<2+`R_<;=42ewLqJ^m~$?>!vfG+y;A-@1>CW)l z*Y75c$F4D;OZJbE#5=5b+Np}hIpYYOlX!0f@Js{OSZBsJC}9%%U`J6xpZ_Y?C3mCB z{;HT9h>4cQOh|CS@gN<4*|wM-zpGi2G(IPXhU%}N7nV~E+|3EEA~$Q`K>Q$|rM=%l zN}B|)C_Ws-)-m^b8B#zt_&tV0QdI^(w`OPH&=GD+yc|6;p0}aw@8Yu^smSd!(+s!#t-I&;8F}E!TezWm z`OEu^3d0XZ?eQVr7u2-{kbKq^H}dV9a^FPF5eAug6qkhzwsA*Gggb-)_XLfV7-$^cyM;sbH}8 zZ`d8RnYYxbVR_ZXBP`rQ9M6i7K4hYTVn02!mlgM&1ce#57v%Q=8X-9UZ>BOuf)2ys z5E_h<=uy5)s{8!eI_>soj+k3Xj@7N#YnLF_{rQgXjcLDa4miKv^Kj~!)zS(Wc+ZgLw!7+|(hBcY8rpF>+b#iCzHPF=UO;NLt66a~ zHc(!jP8&?$uS`ODM8Gr`YZf)GzNu^b6-zu74sFNwnB%Ep6B>16GjuG19@hu@NX1hn zfTnzrfE}RazMw8t+i~GfG~&VTbaHR#on-+7(|OJ-EeH;UdQ)WwM_HG;0;8L(8PyEK z0YORh+GFYHXNA>l-k@sfJ5SPs!7`RVpQ1SY=!w~8*)&jc=DNAeW%g;Xw>Llz^(cq> zzFTlCrI}6MGxm|Jk++Anw*!rc?NtZZxB~LYC&SfpjQSsQQ88zDY%|JDIi~zPk^wXL zpiI_vfX(;DtS>f1ZjP&;bW?T-Nl;P!NH<$4_VH@wi0T7c+lLXm3&!E4_m1S38oMb3^Ay7;iM zP$kcbsJ0r=o|0pI&HYkfx%ASS0VgkJWUH7I1UAc z0$CGRB`I%-Ee?9p_`m=)=_UKd@nE%S&YA?ZD2kuY)`v%3#O=%IaueT`F@HIlp<~nc z9%+NfibgY%DejT9+!Tngz(OEE&%kzN5SyJBz^g~TeOOo09x`$nnCgENqgY^ zZG`O4wEJ>1fLrJ6aY*^%pi7K-%{BSuW^5BGUo5(^!*0sUj(e0DTq~bewZJzcE1IV_A7Z*?<%5?{=$*ID$i87#N24%aF!=kn`=YWCX;;d9b z^bm`0QgR)lzjg?{quCQ~j*~`@AoA#CzKaXijj6AxFhuEb*JV!8*8(xc-NmAdZ>8@P zO$QljlgClpe}L#Ni`z6xE4vRz7*n~LwBXm$?Xavld;(RYjz z53Lr}gi^302D@=&Vt7Hg_$%Ydk5%Z_M&HD;^G;P2_*)0*@{e`%;NEh3SZlaAHL?Q% z>ZlZcxI4{MYy1#{5>c@*mYM5#fwT6YOw0+E*{%)bq-ML|DE+=ywQo$@xXU=@mFaSd zIYTKoj=n{o*MR)PH0OsTk8^hPCZy%SGI#P@B~k<9wiR{mBV^czut9J(p6?>~rEu&1Y4OR!v9;!1sRAu{vU;4R&M+gf9?SIlyt!kIc zU-7=;F}kB_JHiHA z!LmX&9j4P9Kt{)zbv=cw9=bc<`8{~#?v+hiq9@E|Twcayv7l*_3TY(*{jh^*JoFa~ zIR*gxgMUER{@XSlWrJojOuyKMX+O-oCt!25sMtXeewS?<{ES1iFEQ-(1Y|qBU1q-c zqKXKT<~GA`zGPa~wR-5=YF74DofWS_IiW;6>c<#C*oH3=fnk8?`;AuZa<*aU>FoDb z1~J4=-cU>>Oj*ikE`Fh_ac6eZ4>vVDw9q-t2rg8_AR!v#q&&Ze5WN3Dadv<+o`a6f8|h_dGbkw|5kaGdKK)AnaHQvv4F|^FURF*U4wwRF$02v zcDsAcl}Sen&jM<6=4W%p0$>xrcg$*)48HGWd~t{RE++}4%}w>zAzU1jP!cm@-C5h> zGk)Fvu(%&WX>i&e2T1jhO_8=iPovpZ{cuOykus#%xYC0zb>qB1qv$QL=NA5(E!ZSWt49@YGZw=2l|5225vg-PGJbpIUCj@^`>*N4zKKBIzu)5h;Z#?^vQ9A}^ zRswim1K*Wkfn$czN0K(GmY+vwQuo`TTzqu=qc`I_m*5<3P;*o;J0*UkG=0$0Wa*>2S{u`-}8;+ ziAeso&j10Ii}8i6`XBH~fLnszsqoMD7WnsAl^+|U0ctjX#>|gbLc6>j?dyAeb$aCJ z`*(1Modhg7cb>gpKLjW$xZtP_FuZ8Y1u$H`9`5Dk^2c9&!IbAXA!wIye0hw^LU_#s z&#iq=7kwrG-AD>Rez(^$ToM1@&O?_V*Cd16W2I4C*GvFiab>#Vv^n}!%=igC*L5n( zlH$KCUK?>a^4QpUnh_L;K8LHN=L6IgLIc|+3o?Cacx}5_6=uQy?|&1h!^0ic>rrJbvgs6EX;uC=`TwxE~!#BvWylQr?X8P+X}*A zAo9fc@0+S8GC9sJ%M&*OIdX)W|FJosPqI2=5VX@2Uypk>Zsxa}gx+pP7Y`i3n{U~rh{gqnvHCCK}p zanTo*h~pzC`m=tewbRN#7;?pgXpajHYP`!@`;N|K;Hx-WAq)+346s*u z_sbgC)FgImOy7g6EilCupg3uKyxy4cp)aU$r)t1m*Dc*^S_wn3wb}Yiny>JWP?BCh z4nY6N=QV9RbKl(bj#BKBk5cVYhWIeWX0~y9H3NjDJ+& z`y#ybdJPgr7(?7x&Xs2UNG6t?E%nCh2-niaUnz~uJ&YczXyw=p#qY7pR!;*%Nre^L zuXbIIq*M#;10&SIr}nxT?D#9{R2E%c<>!NL7v3)a{k80yc`(Dk$61E;U?-S^BVPzN ztF~iVyIg9|4ueCj;kthk)!aGLoE zN*yAW$IaijfSdeBwKv8L00~^nl|Ab0Xz7}j42V=2JS*jumymj96&|LbRiDrPCpzCF zs|^`n^eEL*t5GVw)Y*@WFx3B6elsbaxhS!fC~ZWmcJeLxAs~KUz=% zy@Ao!NnXhRkqrE)Zu7oHZMoADgU%~LbNQ>wEujkYj4V- zi0VRl%!@a`xSYX0v@&!5a(%`=!4jE%)KB~7XE>)jgK6{Lwbw25 zM3!D#)`f~YG`GtciNLH97&p0Gw`q4B6Uq9S{dG^K$t=IL){F>E4(;bUg*==q3fQq- zm?_^ndUiZ7kX>rMLt}g%<<^Q4h4;*qPtgH>3QSkHq#fXzFY!A7jUJk1=_uFvB-CG-L6cJiJiZMePeSGD)U>>8SV&3AZlAOeE#Np(=J)#f?#@nIIB@idaP>@}zH@`@6;HU!i8&ht(l^XzA6rRPJvNXDwryj!>AC zq`Qt|GkCZEez_#Rq#V7qdoZ)B_HB@d2Hf~(7dU}_E5+EUR4WQ zEdRsa<;nJ)p>DKrUXWSvSOJM(fy6H-4SYjB9Yb4L?Uz#d zK>&-V3DLG)ahWVA?@_kj-?z3RAD)Y7b2lR)HAhJ=ozx3d=u}iv0O+p+O83QK*q`%};ahZ}H4)8o1kY4Tzlvw(&Y`6tG`q`C1S>2Yrn>QE~-WU`41_%Y-;+FGH@~uc7<{#*anxF~__>%UZRlYaiC7U~Tg}8(4ly86 z`A%0=`>wTi8wgP&GR@=4_j3V3mq^#w%@=xjC9nm`B1&W{ko6x5)%^JUf*-;F-}!!m zc#c>y{AX1U>hdYjNRx^KT0)bb>x<(>Jtk0n$&=xlZk{L2J}$=wSX*DE9@)V2-k^TAHgYybX{HgzSu_qT&Mr9=U=3DCU0qI)mb$&o=|PZ>ui=Qu26k1DfUrpcMu^fHbbVt2e3C@bkL^& zGO6QH#%Nw_wYeThO`ih6i(BcQ~ zcZ^(`JyBZ$8EaQ$SMOV3>DT6C`X^O&$O2$HmIa&lBo^1y%32~Us?xH+9JMK3(`GE4 zZSHUNi7-1f9)5Qq&;9UW;Y(fno9#yjK4(%rx%p_ckuSP5cH+ndGFLuO?Go^~iIgRm zcf?Qh5A5&Yg$og$Z@R3U`Lh%^f*dl`&B<84tkxD|+x>vu4p=*I=PBm)cGOQ>62FW; zm7OiA#&N?4j;cy;op?!eKEJE-R_=R_%vV2G8&iYt1g@LVm@J3r^+4@&(M6Dk)V6&z zC=zU)JUZ-;3`Wu2XA?o3Vg$!&50GukI1d1y!W6a~`*E2Ikbp{A~``mS7N^2CFTlVQQhd)zC*S??f+gTO0&*Bg; zNo5%l=3oG2*s(6pSL-@%__p~zqlU@Ll^H}YdFKSR3*%i1#kbgy$4#mV25xs-L0C4npW>0pUOBQA1S0rMKSR$GpqF>c11uq$a;CxHwVNe`ieWMmZ5cMO3;@d4 zQ4`Wrx4nf`9szY|sYq~h(0-Qc|NdY|jrIAdon-jg@6y~~^j&+_X}Ue-H*m+a9JG)G z;1;uy6JU=5N^*{mDJ!AVAKqf=Cw+Oe&yY#ubD=QHbpB8i+J-Lrpg&!pB|Lu4x(nIQ z1lGayT7C?g;PCe*LF+ZRAdOVLhlP(6(-m@M%+$GDL6JR<)Myjaf8__^S6DpuuV;ra zDqr`MZo%W`Did5-i|$q3QPUQTV;|IC39V3~;34hSD90f^`Y6oVd)-f_~TFFcjNfebt0Lb7aG zhv}>3u^Y67ooc@2%@thBZi}L(@k3IxY!2Xb7Ajg?@!BxcxgHVf)tXO3P2bpDb+RJo zO86(u-`=di=zr*p7Ta?iyz$Trsiw2q{lz!J5b6a?0Q|M{`~%N{El!sFf$jK{=Iu2^ z>R=TGtd8v?MwOQUVkHM{-b{ zGkh;SW?un18}GsANrow9KxPg9i&*^522^H}0hd22D#zz9{N8L^7QQj}xe!Pu6+OKq zU9lle))g6+v`sqdjkhh|>tNxW5^{Wr{XlH^@khju+H(pvKaRpumf|-4)+$-Eh{<#u zCtFqN=qS#3{y;SsiDo=}P%_u&f-KUb5{+tDG~wqOaA14^zX)%GC*JH8y4v!Im~k3aFLepgFDLrgT>&$ipKG1K!XfumnpUemS{f=o>&v zd)}2J0U#8Oe~=}+%L4W@cF=PTm;L=*iLH$2yHW|JqCJn_KVYGFQ3Vc)mX;L;$S3gI zu6aWA*^QM!F*Xm5DwPzIR?)f#k5-BGme)p)9w|f5`}VcS$G9GT&&R?xbt`JVKiuGp zEo2|evGC-fgffLyewBU+&jH6hO~FEn%gO|A!H*Zgcct%~xw>dQE==iAvg&_?(IV{1 zL>-wBPzT?lFpuYMv-L{Wp0PQXX$CF=MABKREXaW*yaUK7iD8waq| zz?|W@Ual>+&S_O!O%A@_asx$eg7Xb^hE6E@6^%XaOs2YQV)~@Yca8N}QzFL;drrf` z4ixsoqnT~XmM9Noc|NiZU{r-kMb9RlF!Km-ds9+AO^)4p5y!uSbtpt?Ln=ufSpQ4i zs-nqmCNnN|_XYAMtjRDXB%C+h)`BP1(Q$0Qg!8pw_o@7`G@|6D0#TkSzgtK*& zUXW_YKL%j>%m{=S9m)7u-cFcJufy)X&E|c~RZ3uaP^aQ(-Q0Lr?aQ?lYW8h7IJ~MsXAS-U0VxI@p2zhOJ*=pnI*N)4v`3yICx2d^)z3Qmyb_&g zPHgAT`@SFz{=9j>XOFIYdV+3X&eoFVVP0$#)wxGWT~nQ-xRw-BfTL(kVvS^yE}5;9 z<5^IFI6c232k)#XcPfR~y{AmT7KcRuIqW5Mj|>RV6y|X+4G22dM6%M#O6L^YL#;IF z2c;7O;y&M^IzPH&?)@7P{=XP{WkjAi?WKfsgiS9QkA#u(`TAfL-MmmJ*rIFXKpbvN zt-4z78YWgWy>tH42OgQ3co8nj9VimMs0G~=87^;9+7NiecgXouyDT?|k2}wzf>1Q< zrqWnRe!lQW@6eZVE2xfABU=l-Ht`U1j;d_*-2HDi{1-+Z?R-_AD{TXP1%$WtYx}_i zm`*19k4hYUyI2y)axyz|bPx`U7S9@{SLKPGlw3JtAlPJHo6TvgeGW9ZLpDIh2Fdy$ z72J+3(wzwF-F%r*c@v_y%Fm` zmmv;}uAJy|ST~k{la&to<65u)Ll_5*VR;XJkZj5efy!9wrxZ%wb$k=+ZIz}hBTS|f zwa3{gnB5QO-i=eUTWR{s8iD^RcK&8zBFlHQv)c;g6~eXdxd8>J-qkyhAJ}pC25hhZdSN3F80(MR*LNV0fSJG3q&NrEmOLT z*H3+t+(hVO;OJ=xF9Pv8U>$a-3VSq_(l38CmGKuK`0%f7iEG8h(H?AT?5XY`smNO|^MSox0qe;agkyjdUAk z-ZQ9o!))XLfy46ejOS0jUiAO}@UPAE|BXp9BD>B3S|b<8rtY)?NItUu*_Eolpyd^Y z+;nL>x4GJNE5bBAV~D8AL)3Qp2gocy+F;;G@>>AB{fEC9Jp=Zv5wGQd(_8>7l6I>6 zL9+$)(U!I|A#8omRzFNUAz(_J1}FtjL}C8)GIPE!z*dX{4TqlqT%g!<@KKKyh{OvJ zS^Pr@Ax-`e34(zb06l!&1k>`I70K5>Y(1u*}1nW4f1y}ls2xMd1}?e;*=sw#s^ ziyMB28*f`Tlg~o@6UQk%12tm41`AXtd0-p|P&Rv_w(>C?VQHXpy_LV+_K?VO8o&s) z`}6Ob*nNFBjzJwU7<3JQL~U%+H2jP^trNXk#A9?lIEO>wzIqo9{y?`(embTryY615 zIphji-Fh2n{&YOL_vtxXY0F{%mQVYIsrc{jR31c!4R!_^j?!g+i#%z7%NzOA=lFCD zV0WZ$6C@5L7I{dvt!gZX+W{kX6C?6D=p@Jo`WahiR)MPD(Aa0uYXay^{{%FZR;TUx z&O6oT0vP6f4B;Sl*wfhtpbg^y1Ebk48~bBAvsvPyHf!FJ`PwtncVHxr0NVhoAJD`; z0G30wbpHYx)BWnM0h%p!`nhGaC(!j5olu*pk&3JrOaQ#mm*9j_Ow0d58NdjzWBPEn0G?^MoxyzWRcI2XL6XV6_fBJmp4chC z?+(_r0S%8mEqe_B&7v{i#pDsi4opDLtJszH_4$^|Oj-FB!1q)# zHPeb@S|1IGXu#7v)K7R#js2*{+5mV}Xg(x8_YCOK|6mW4J#S1xnA`02${ey*BJP*< zI`Lr|`42@!PSh-b{EXXvH1mmAkzE?5ryrIPD|x7>E?27DU+8HYu=5=GkPr5`v9TfdE(?sP%08cmvSg zYNhabZ6HwQK&1U#4gZczX9lCQRz0SJKn>H!4Xg|efQ!`N>v&g<5LFlX9LLrONFg?K zhvaAs?_Q&UH!%n+kOb=);jIRGb}~KE#8!s46jyHG4gr*+7^0`Jyi413j)+5e z&MsRJxL7_wRa~NT+13?8S^GdPx% zAzo6hbThc9VqS6(7&zP8nr!r{ux813djsB`gC{YC)dpxT{Ap`XZiW(%v@DgA+%vdy=fF^VZ+$%=&eZFHBFEVh&C9|6}gSD&D|Q8|$F?a8m( z2e)L{%(1B54oy|=?gGS;$^d`6KfdD1w4N6pI>!b($ov{LLq{l1nMp~J0=!b<$VdQ* zr?M-y3*?>{sY*N9f$*$FC?hvEr3LxhU~vZqSgKRx>~m_8#Nb;P_ESNchtTw(ZLD-|7#nfNP?1&eG! zY}qIhU0-$CfKFq--QxTIdjt}H+aBMgb6l0v$Rlp8+_};^lif8$}FZ81q&dz`9`a zjz5_(tVhgeZ1Jr+xj6>{)qST{GF~yFaQ$q{pZY7Hojs^O9#?^HvT*7v(=)h*z=~gG z%O5E0I-Rjm;`XL-Dzim@v@{}OT~izZ(0ofa=`=g;%&E1KW^8KYB2zOqq>NX&>j{B# zdOS+yNljlElQi(Yg(Z@v%=c9M5%}LHvF;+N(P?X57G^$s!I9e${f+YBGT3(9b*$u{ zKg}?7vS5*Kj~-vdD=>@U?{&rsnm0T6wiDGNu&sg!xTu*X_}LP&EVcEp0u9$@_0AXo z_BrFN>;5pKxt_*~#vCUhex+WL=QxJ2JiXycmHwb`oNZxTr?gYBEQ;#n6SE|suv%BT z8DPZJ)g3Qf6@HpK>#hHP%7sMgX5ZovIR*mAzm09YIvqH=a19WGgu){!!nEijKXIgvWgO&7Q%|% zq_L}L+k=Cz-{6Dd*;4~S834jj>j9#=Y;cik)lH;u#J*dRFBRkVe-ZbVVNrJ7`|!|W zfQX8Kgdiv(DBT0nEr@g}(j_f9Ah&clfFL<^Do9HXDka@r(nEI+!+($7&mGV6`}iKm z`&qf>y7pds#ktOPZu<%+)x@Rippdn7xINveTUioIcFT`~9ETxe;rJD)YbjZH@vKKz zIb#kOD)e_7W<|ll>%o6+Ux7`ZZMf}H$$IO(=ksTaCQ#?K{o`dcBZ7K{;)pt~neB;| zlu?$a&#K;Z+noNnR?BCX*z;|=b9lx?s+z8YImk8v>ijDq8N7fYgu1Gq?Cl=*eI_%? zdd?WQzq+#!^c)q--q$AWi4ap`c)E{*N3PnB+|% z*w?aoX~oPe4ns$0oqVG>CMO-PPS37%OTbQ7D!mOoLG)$}hX+87B6RI-0l{$}YnC62 zkT`jotaUtt&0?6qQzQxgr}HSxJu1e8Z2zov>Ss6J&AoF9k$&%spOZ4D z%enI8wkL9U+CJvuXxvr~$B;uXL}!I!g;hF8j@Te`(=Ps2gBtK( zQk@YW=pBJ#PKRY)XMMw@z|~;QmXh8O*IdT&_nWt!ld4PL2I}QS1zTG6rS=hmi>>S1 zH7=eucRKs;5(H=CIvD9O%&rP}Kco9@niP39+F$<<5Tl-!m*L9!PPFsvq-PFO*O5cQ zDu|X$?K)05qlBC>U0!Rod!tK;>>^pO{mwAw_Lz+UoKU)IA1QJ-d2^m~ap)%e@un7k5G8Qp^^wb&CPLR*3@DR4+H;by5SGF zY{{l}9bGrL@jm9o>O{#;75KFD*Fn1b{+$4$;5`(*uwNsOv(E*85?=Uc2mANx8%1{1 zlwGNFZQ1MciOxo@Lv^V79xp85F)Mb{JaixX*@yz2YQcLQ3H2f`V)yo!qjd9bLyZ)c zijw;jD`oH>UZ?WK0{N+7CY)Cs0MgQKjh5nFQ@S7C-4&sZLED=bm~FM-P_WGJoqV(J zDwYeSA^S=JxqdwJ5s&t8R;nhhQ$4fSVM1_=#*Z#9@+Feq1g@ zH?a)WskG}seiJDTbe4%h4muRXJYSQ_qeFsHD{g7cbUCqm5WeyK7!2_)5M41(3a%ad zs!>O6d(=mH#Fkhq=XWr8^G%>MFW%UDz`H0~^0 z3AGik?y>g9{^Ty0th!Ht4u5c7D%jaKFEs<9G2V_Yy|g@%-?}-(h4Wag?!+ZiCgh_0 zF!_yZXLB7+>NJdf7P3le+;FbLYd;(~hu_~ErbM-4u{0-vjqng_MXeYvZG3U5SSfy6 zyMFip<%oi4+3EcpgE~J7YMiUKcKC)2S-y|UF5VVWEerc3tuUH*BE3HIp-wtjNI{+e z*NdY0<-KQAML}^0 z@n9*aU#vB*9^jjmg+KUOiXm`rT3oY8!qs?(5@=zg?>QoKIpDovsxRN^Qrnt%qIGA$ zv+xJszQM}^E)xB5t#`SBo;=hAC*spJt|-rL|4r3_rNTSmG2YcX$K-$IXq?0lF?UOa zIGcpN8B=v3+u&FhM%lQu9FEunF_Wo^*6A67K8hTUlzO{YUP$(gRRl?L;T2puk^RKY z)0Hgc>wutk{!7piJLvJB+BEcJr8Gv&ojdE$h|0jwCY}qOywn$11Wh@pb*}*|gXS~Y zOx5Wl>ywi@ie{nu>Gy63Mbsx`w)4*u04VlA0IM6YlmGF+_G>tMG*8t@^fE8gVN6f9 zN=f@^-PQ#OL3-&Ub27gh**|Pb)QQm-pNgZ)aUQ+-e?EnRO3dT3*HM$XWn|5voj($p zk}P(-k|=`qDN>uZ`4Y`BjU7m#n)%hN|CIliBYunpIdh-=&3ylp5C5y-T<}Yvu^hDT zu#`Z6_+R>kv}yvN^Ie?Juv;=%eB06hIq{OtEi z1KKtSuA0w4p9U9m@&x?tFG!j7|Em|@Tmig(6(ZW##?=7z>IYL1MzETGEVlZKg@qMr z1Vl~#!Rd`>0O&IqrF2CA5G@09fS&cPo*K9T{VBX)IEg3nGyw%@j|-3Cj>rIfOfVA;PHhYL=}YZnx-TJF8_En zdO1W_=dZ;4?SE{b$c;Dc`}Ee&@BP1kZ$LgQ1BiJR0<)K62~;{&xVr>Q5&%x$ho$Ob zwJUGm;5|5r_Vo{9(>?*}nJe~@PhBW50avJFCB_8-K}km!V8$hN;a>sr#Q&+JjTH-+ zt72hR!UjK3Ujd>OHcb6xo5M8;?|!QCY=XH9Il!0q)U~hM?~|dnjmfaU3jA@5l>#Ea z+4%)}KBqoU29x0r8(rtROWVUS4r-ML1J91wdYoEl++NnaSur|W(HgC92&j`lOk)+F zvH*JgD5w0*?o@2`XHPeqsi%J(^uLXI@kJmY$G%b$O4bm9W<6k46+3?EDliRD-A7A2 z{>T^RYFSvetP<;dMxd9)q4Agd?-CH&{J{0;7tT#p8#V^zMk&Xp>L32IS3#3@qR(ubc0-7 zST}=b+fqn?3pfm&ZhWvdvs+J3hHc z#k6*%J;FHR|L_?mH{MkPK5jo&mu?XO!|<$rEL}ukk!=nndV!OB{@HLKDCw_$7o4<5 zp7nv>kL#z$B5PQ*(j4aZUGHruEBb7aA1R-c?5c|$73c>-t0_gBQ{ZU2iscB5B^e%m zw!9GZL@QzgAd<1q*MRw{fJvARZQnt?t(Iv)^w#%&EM+cEq1s>#yuF1nC$sue2pPx7pnv~!@a3W-=5KNW5Ro8qQE z72)-m3V>tU2)}zuVeF>5G;TrgMIoGhxE7$b+&qS$(@_Q6qZ}VN4+N+7v`Y2sdj!4K zsqbYAFekO(gphBFImiz$yyBX_!iomvJ>ce5j(({HnwUGg)*ZO&2mAfgSRA$xOe=L) zA6Q^vqxe<5L9F;K4hAcH=Dni3lHrY=p%cQA(={A`_HL_*HCB^0kj7P87igfCcKX@T z#NeC&xHD1q%?eT3tbbc*F88)+$I4#*iV zjsX`s_hI#wdY%2`%@4qNeBx5NpZYcfs0&i(Kr!2qCa%l#YrBGL+9&}RFhJz(By|Bl zufSvQDM#X0#I)cEeKh$LTxt}82YG7Lo96LH0Y?{fMtGBlRb3mFNa z!MSm|U3CCP6}{NM82e-Aa40qRByYSvBk76-fU^y`FXuN;&?w2g!Z$+yskoVnGBq5> z77KxM*9Pn2<}n-)@!m`^ZWHqmRyHo)Sb z)c{1?2vggN-)3p`v^SgdoX+Nh2BKax1*O$lj@Sv*VMJl;=wZdF6eu1 zE$KpJFJ|5+UZS;;CwKnj)Gf+iG&tMnI6i9(f;43FJlTp-)_QHJo5}_>EOtzUzKK& zem8*kA=m$KvT~*M5D1XiVgW!btDEqF?}K#Bkhz?Avji`1dfiM~tg4+D zsLS;;a(Da_hMd6@92s~i!A)&?e*qY&wiV-eK}Nkg1)Sd~4@U{Kjk?O3Zt9^W6|A&o zBF{3t5@NK0)`F8%VZya>IH;edfrK*G6TM+eHx1k9W~+%jBIEIw(aKyw@5MBY<|eec z?7+P1n24qykLPEEHFcc2>Hhm-^N!t&LCMTgvVs5c{}D5)*Vo(P3Y9{Q%3dqJ_oovP z@P0HKdb!BpnS(1%N^aCL)hPu3tx1<9w)M;H1*9f!tBE&NC4)v)sY!R&Mq1E}+=Cdd zdses)kFO?aEXk4XR}V->d~&KK#m8wRg&logyYGMh+^DvDDL3a_kfKq*(3vC&3vO=V z27s<(&DsaI%oqGUm(9v4c>IK1T#O(Z25uwdM{EzRxygGgN~Q|0ld<@=L{v_?j(>2< zVv3E??a?*{;M}~6xHY>k6)?}{U>9z=hYM3iE!$*%dc zbr5|BA=PHHbn*Mctj9GsYw%7zSXus$72GTU0BId;PC?A%=9H-SBXlUw&MIu; zbvifhU3!aNX^Kt~F4>xDj*zEpP2CBi@CYCrkr^5C`c*CtAvMJ|slc4LR9VOAP{#W* zRW&F^>c$83b$Kc)wvrYC&c8evtt(tBoZR5%slP7!Fm-1lf?V>>PyfUd2Tl zd@3m~zKK`gU4M4FgLzLKS3AEXo`!0HZ$pYW*Ao3+y%fy5tT!%CJ-sfuJ$0IVtgY5Hk0Wps^`=OY{ahwMSZebU4!sYYf z(p!a-C#t#+drEtQ#d`sWqyJ+Gm$+9;pChugpzh^aZKp=Hh_9Qgc3x)XtY`->oTw{;0)t7`!f@TV#d-qQQhET4u*H7ySgS#)CJ<*NQ(X zUgYZGKBoUP=rGgC%-?L}XRwMNPB3R^RnGfiVO3QDCo<%^srclDmA@B(_YLwX9h9w( zf3Fmb@=eSP%eMTq{*=ljNpxTs~f+`jvh->$R%i1KicDpwut@T+un#XpZ(Y$yKF`saX%AQceEZm-nw|PilG#HQY6bnb?Qr{-u#uHNkdhkhB#Eqll z30y5$f7y*Ee9}fb4DZr)B`4S;coFIw7XIcVQMujz%-t zTQ2pO>%FGK*f#QI8|s48_;7J{IeryU^ZgfLxsn+_cK@zkKwBenTt#UZx_{`rw7Z#E zWB9t#D6R$pW1HOVnC=^2vf&}b7}lP&I(L^Ah5I1Shhg-wvPNeQ&(84Rb0-#;-g;zr zVY$g&j7#HoQMX+z->6^6y3cB?Tqb?dRJO*uIyRYuraSHCr#Xr(^0x9!@emuGxx48a zL(cE77=2`!yY2KPnbC@2=YIanvk&2X0kONW3TT@7?C z;~Wt~r&Thtj-2lx91UT+6^wSyfxK1$nZz6v|T%#RhNRRb&y2wlIOI0<>wTv@`J3keyY|< z)a`pOSkvv%JTcqE&D}7+yy6CVkf1&DDGVruR%IH_@Xi!@|5-4CCEv(6RNB4$${^v; zH&G@o@zfTaB7rXIOaA8@R{X$;Y`Fj%+~7d329@=qWobtxt;f;@LPhfFluj4 z0s)$=sp3L&2AfEDxEkY2*)VeAPGGFS8)ST4tNLQoZY{E(b2@}_l!4aMS zWUWV%w6n*?CNEJ#v%>6Y4^g&l3hqIeQ8)jS5|v(Up88QeWZZ>v3nw>5n3(Ebm9@Xu zx(0G(grfyVnY;;(c+F%@jw(g{Q|2){bXCrC_* zv1Td!j#|2{H?5)JdWxSi)3Yz7Dt>w9x;L)#?ZX`xoODz5`-5t%$@k-0L*@i(HmvIm z%!kpV&(-ddJ<-Iej1RjLA!PAPb%_O{f|EIOG0cK(Xkb7|UrRy_+uYl4SD`Sr9Y z)-jqRyeWFQ({i=>XysZ>Wn|XD4Cxcd<(UmOvFqN%A6BdvA(cUKJcib%QhjAaa&cg8 zrr_R|7W0jUD2D~wwH%e^8=lmh&A20RJD2jy{39=ZY)Bd~%ll~a&7Pp3VNk&GPv-K9 z@ajcBJDqmj_a_CR=9k4wYqs-g`Wy$1rtx%_5=JcWUH*neou_6s7+%z3nKOL45hM06dKYTh6!YBgD=a)-VcW)f)61~IA z73h*~YEJcavNoXgo!g^z?vbFZ&VR_jx~SN9e2v}@Z_OH4ppUP0rYAIPn}6@lF)@QL z)w1KqPH8lW!Y8-rlU$O)@Ml0`Q#vH;z5zV^MKur)k?^7=6I;U3a~V{Dw4`Yfk-Xk7 z;BNKK!dNn1P`AIzvM_Wwg5y`(C1l;PU5KOk$@b$IlH;tkaEvrvIx|y_k7`LgQ@@`A zKD?kiqpA0i^7?yROZtzjsMvHPU)nH~HT`n+%JWazkFrDwb|4}nZkJOxl@cD{aL+K- zX?R%2-!gWixAa;eJ&58`Y7VY{!T&H{^kPp`2@5V!W9ECSz`3m1kj3ik%@6FW_|(J$ zp%rgbf%?-P)ek2eg-S}Lx~j5W}m>F{npX;ioGO*!wL@)noXSqP_t7MyOJuWfh^J)3rM2`$hzy74t2#!%nbAS890d=}k#Bs8;?BX=3_4=Mi_@1``U8s*&%CIu3Wx=Cs+`GQelWgC8&!m5K z{aF9R^XvrWgxYuY#uOH1STmA$zZ&iy-Z%F&M2z=B6{oeJ9K(lstGfen`yMMxtL~lq zre{tQMU{po-n3`6`iCUdvAQdB28W55zLp&O{v`*|%GJ|TO_6ih&+3Yk0&BcmXJUJ` z%^ZzJ9JAez8|mzBz~;{F(;Ksx6iOJC)w%loJu`FU6daQ>IW6vVt!=fRblV}i)`nL_ z&BM@?KS*-e$J`84*6QCJNj6-CRb~&1!;S{7XgkL$6@_Db^GdG^6{MA3qk;*UIIIk? zbh{C~BMn08HzM8!K_^%-CmYLbiCuP4>r0jG2bUzox@_OCoAb3ERFyAGwP+p3_<*TP^<|AnhA)%s zWWEgy7)LVLO$N-Xf+IGrx}fWc&y+&CS?!|XXT>V+@~mN=GP;qh^y4H-@Q;_za00hl zIUDJA@V4Eo3y=C2lB+$l?Pk*xjYWXfnNh>)=#;yA`?G8-^`Z#rKmX?oKJcy$Cv(|s zK6H0GsM(o~p>Sg8v71|v?^X;ZK4%LhQHj0W!h1F^0yV(DupNjNM^7eTL<U|MTqoXO}FSe%$Euwsy0uEh*;F?rq@9!0dRa z+w0h}@U12%T}Xn{z)&b+G{uIO6#5SIDh#p~a?~!3)(JZD59K%ulMjbG$uG^@^y!R; z^>5#s9~nH4j#wT4wJ7iCf}jvrY4idEhFuE%HQAatH(AC*Cgd)YQS3%y%{A=8y~7&A zi4f$1>wk=6jf0z;@ig%3EmX1Ko1@aJ;G0s<%`CSp(aC~N%}J6nXq?Y@;rwTjS`?gy zJli=-1lM4#ZjE$n5l5Fu!0P&AV7mMddbfmKR$>G#5|ou7BYw?^m=13Y+C4ZVgo|@} z18PVHyKrlk)KwNuKr~BwEDeKHdmWbqLz8g^AOq(Kg=DS94{v=H+S_C~t7$KHl+b!K zv9WZ0mJ~-4{dL2R21d>#%BxaRC@fvVMS1-2_UQTN2jei6l#O~N+E8r;^c~~9K&Cpa z#cyvfIj)U75_H>0(ywxCk}~%467m0Pj6d0fM5JeAEX@CiX2uj6)>*yTat(swU*0CF zsWC;b)3KrCEMU)qfxDjW zfjo=l=r)DWx%Qne%QD2(KSNiMxvOh**cV*w8Lb}~(UzB&KW=)*h`IU@EHB5dhbua( zNj~zyd0RO0D-6CTb||m82@QI<WDbZ(f=r4Y~Smfh`-Jz3biW4EG_u2JY57^8I_+;HapLfnBG< zRo#a&VwYugOENbTCDAyL8OFc2pSxs^)1ai}JP`d;PVo)~0@=cw?GA93MYP9pwu5O3 z)0POD64UmZqg75hw}qVLUc7kGQ3!RDM96mI^)B@u4lj*X*d`ta^prk!Zr3)drwoFQ z&}>uO2FC@0_CG4U6JFxJyC}cA*tyU|VH5*Yq?8QZ7j9Tid@25@Q`(F_IoVShYSxv+ z@<6xj;pRDLYGKL4j7^~waVp6|KTh^nmE}B2{oe`yW#SOXIX%V(Ymg<*DD^y@((Hj; zHOaAZ>wBV}2Qje^^c4122D5z5PPp7QCX7Vpx6#MDB^JF5yJ9Y@L+`tig%y!LXPH_> zhC2YwR*dH|&9ojaWMM=c82< zEc%BhS;}13$4nn@j+!HQaq`+lyORV>Vm((rzCiV5Wj%g-k@-TAVcp8{hTGdC|4A~~ zSHB}XNCPwonn=lSmlX3rPZJy@h8EI#hr4i!6?iAY%}Is zb|$B0>j!x?p*}!=ZqXfsy%#TF8#IC1>|v*R|NVpA^Zlkt)jvXk9~v#~j7Z+#!GKW+ zMRHj6>QeHoYMjZ6*`q7ed_0l%Lm%Uh342DxybXRtF?DjJSCfTp-1fLvUF+1HBFZdB z1b8~!^_UauV=xGlrCt1@kH>VzeI~3m%mvR_Xw^qBXcBmju57xP#>y%qP%hUWc#yKUN>umO9PQ#p-_qqWi z3(Jq&GSd~d|MNk3ib(Af={`@4OeC#sgwv~>N4Lnxo^KrYQ+^Yk+|3d8(#+TRkwlt% zxK!nOWiY7b^>(@vLTs{e8sLc5uIK2iEQA8xoY>wm55y&pb(=_-0rLiLn+W0yy9(n= zH~Ih?%s+tUidDUK_1BQl=Gf-ct`kc%YHp)UF=uKuM^V`2o{9kmrDA>yggqM)Ul^d{ zcQ$jmk;761R&f4|WWNo}tT~L5xy^ZbEt%uTz$kC>LjNQ$?`3LUtCz0pN=iyR_E3+# zT)-+jn?MGW-%Gfn>uf^dtx`wSQ?Bup(GiFvnwfnDwqY8o_|NS(xMI3@>7zQBCO`K zccAG_H9tlzKIly3=l5Gm7xuFf;Ikd$SrX)-by;rr;6u>fMOyYVmnNDUUWI)Yp5+t( zxeDZnXxm(pn3%Z4tdok|+#?df(K)b?C^VOIh_iA{CC;=xR(`1~xsxMxM(-YwJJrwRgT zp!xk`X|one9i@eM@E?L)$4t-yA!8`&hqfd`^R5Mxg8EL zrG}yD9tCT?occMcesub+kq!b8qyLqtJPuD;Z#myNhspogE?I8%!X^WBzmox9sJ>!w zT$5}th_hbOzrGKG5yu91x<)$OfFryaQtX33JivbX|DXTF-!z@S=!jY&mRD82>X1Ed zt>3+e-h)6|`Tz5xDci#y61-K_<_Q8MoHv8@EsGpH0_E0-yH&{`zFz-`)=N1qsa2_wq995&=gCJ9N0m<7+1vwWPXp z`WKaHIA1=8%gIpTT!3Ix?B7c?-1pRUyV!Je`!FgP$ro)H{UvyzNigqC>pbZjn39Ll z%SP)e9D!1m17!WMRH3ja^clFlIOLacK+r#RJrdZ(>P7BhbU#_J#&Ocd?tS5A2&V9d zH6~vh-8XPOnv?|_!v3G#dDZ!o3}&tSBdd`OO1Gbib7Sp0oV~=J5ZLzTzg}K>yRsr& zzxpN{s_SxhAmKgH*`EzIVBd`ef8}gATOi5*L*6XVYaczjCEtcQZTwkF zi)MTvopfrp*f_mU>Cy;~x=#`~x7fmkD^zIIy@Ni$_5J+#Pm=%(O2=Wr>~tX07UI8T z+WE6x0y;z^!bqeQ_9Ga{W>WCu1n=I<^*f$k=rpb&+A4NvQW%{3(T|kr7m%xbzmxn1 zI+3}}w`#l#$6}Wj-cGTVwB)DKaUL47lGS<24Z&vZ4K~%)GY=V9UPebkHJU@cH5C`g z#^-C4)uP@vU8eWl0+rY=`+S!GXKqi|Y*joDMw$l>zk@j^vYiKMM~TJ~oV{j!k>B0^WOP)fG_{gE}lXzb5(2R=UOW*adHG>$Lu6!f#CzEA-!8vaJ3L>Jq; ze7sY)_h&jVL~P4}jL*0xT~9j_)MS(Lf@jOxTkMzM?gml)$y~O>mSEriIFf{r!64|b z+?cUKYc^sO@hEg+d_GB}>$p^-n)|64`V~0&kUN;G;#lLL$IM&;e+9~VY=!6xD&8#+ zr65&*Z=LGqaj0S_cTsmo+hq(Kc z5!O_S29nhKOr0oD73GP}(_^AXEwM29LnfE3)2$ZMtUX+F`;HMhM z=52SkQ55KfazBtLDS>k0)BgT!I{n($YA1)AoLP!-u~@Ov9dL!~U@aPzgJ=TfqCH=1-d7_;R^yKA(yD-90NYY(w(gRkLE;BQ;6?1lK zI$C1NW!7=a3CHE;XPIy+b&!`?_NKq}0*b^X#$Ry-y^dbB8cY~HM&oS7Ee}|XdfAn- z3)Rb+(I#6KJX)W*3t)oB7gabca6fyyqM35^nJ_!VzkdSSbIzI^6LZ(|U=7alG+Qo? z!yxlTQ;2%L28%lwP|uW)X5Il}x^3ywp)ZFE4IFpo<>lq%E*ziYfdGNYgblA;>v1-( z{v@NWP{uAI6na#8;m-&6(kUdVl$865j7tC;0WO!GVtdKl!NGyqF;#~MD%80-maE2yEqQcEsPe&n^}sBmgP@oGv=~2~H!<+%t_LEnYaev_1kJl{Q}S56 zBSx}5%l~l2*lM`&f%nNC9DLyqFd3Z*s!8>dm+~OrO^Bzt{0vy>RhTAtE$mC+87xv# zQ?soO7sb_kpDJK$J8?f;vnAEK;UXiXkJ)fHsA?6htn#ZyJkZ}&p=U46&cOJ+AbChy z2sk&h-B!Pi-s{!IgkW$`(7Oa)>tEF*sK)wA%~eZxmqKo^b*Q9L`3MOwbBKu<^4U$? z2h~(Gpk8eRRKC3ij$074;Li{H3R2QcP{o?Ln}aIabZ7oYiQA?jct|owdUNW=QJ%1G zpfOZTB%cxq0$xFItpMf(nCzC)y_N`;NjwEo$w<(UjPe| zf>Pk6*=8)Kk=^4BO$rsG915K{_mu$7M~@z*kP_?sYkB$aq}!^5a5 z>|hq3tVoiOOWW?9JE8x2i&x7%cUZJPr#i0=b*`Pb7aTdQUjTwlJ5>-r-~97` z*JdV`FFUGEqMq_wy94Efj(8s`KB0n~BDe5T^X`OGoVW*iW+X10hP7V=NgwKiB@Z3+%xl=PU70FM&~u&!cnZ!ed_tQ2>~aFvt!G0>A{2m4zHO6lP>bY79$ zmC1QipsK&PP0eTXu0KaL^!v{(ueNv6AlPr4^Zxd@hJ9D`_r3&RQIr%>&rF?CGcIuY z0caNmYKFG=Y+8;10qGmK`G>?)oujZctlR#}{C318RRRI#{>rV+LP+1FEf|= z1F%{9-{$>os?O8&>l*^IQi{nVCJqjjn;iQ0qnVYf3yJ~UQi`L5eVdt^AfYM0doLxf zV{JH;JDF?-o)XJY>Pj1z!*Da9*WrW%Aak)wvSZ4T5D~ot3(`JxYTEAlyXqh43 z(kblJTSmFRs&9oig+U_2{4bfd&l2~g>v3{7N1kc+o!h7BL!Ea`Y#&8@x*=GpcbC_z z3n=G8eGC=Z>5MlyX}iOe>T6|huFly zhj3nn8Qv0d&c(v)?zA`9G-Jr`sk|1mZDjx+@RV*$De_3KUpy%)Hy3a|P=|CKxb7mC^Hi_&)~dCW?e9acEHy&hh1|e_RjJwrPf%qwlj=T8<7w z=dxP}*IVVDCy-!_9-0xu^ ze^AbL>;4WP=nj;#0*Et|CyoLjHq9&^Sy$F4-_5=k!q|T&UMIeSN=Jw9Vm1IWe+SR% zwDu~erH%_Zod$MqfoD&gD_XCvxE4*tC?9cG{)b%g;43rPph?_!zrB9s-U}Ll#bztT zGa$Tnf_ZF4o>&d!K9>PkTXd}M*?NfAHa`&nff@WR8*fgGR|$$7RZpX&9zBFr1cxpZWDc=5v;xzIlZcl{d+_Ud4JxHQaE5bkDl$67;_C6lr z<#*~?3i4@>-*%{xT%zFMq0ne~LvRTT!Y>kmUTx+% zd9>2;5+Ep907+St3&En#Ap2zE;_6GX#A2)vG`kS5Zbiy8tD-9V`zpATEiXviSVSxK zTZtP~sD;U3#~>)r+yg-w{1$MU*Wp8-^x7k#nENa&EGeXqWQ8sCEuWycdH~?*F4o9K z<%31<0^5f_>O575<4ESt*DA6zqi|avW1^&_bkmQ@u)$9?l}RM6^+qilfd|@{%MN)Pa4>z2jg3W zZ6AV{P`2QqcNw(;uDyfnS-5t1SvWO+!l-ne>b53gcoMup1BM4eL2D$3)(QS|1)Ozq zINlF1#evhseYjg?B;tMCm!-(%x~5GoP8CeSq5sO0a1-EPsw;y9_kfB?eb}u>-`E+$ z!^3&oxn1Ic$WCs%I7noz>pxUxNC*MwyHs8oB)uKa_fL)iN=2*Yu+Vk`+}p>;xwKD> zO+`-oCl!LBp}>5w-~DE)JBQn(FwJkU(%vW!2q{fsn2MMm~!v(C%y(f3sOxE-l% zs>9SXSc~>9Wt92=-TLZy`H2!!#jbH@L3o#x56}u)ocabPR+<~f*FSqJ3C2JksaeamlGjPA0&4bIKsaYe$}Y zHK%xK1K8oBzC^h4S*bIB-+1qnK~MBvP)N3NVuAqns+-RHIwhH2Kw*a2cylpP0?Ide zU1rmLApY@i&sKkY7r+)lzGeKh(da7Jt0Us=B?1G-yi%_eNk#>Kd47XDfJArQ**qz! zSDDU^?7fsUY>E&n06Yk8A1W9a8Lk7Uezi?0NT3(H?XVE{*}8>@)jP|z_Yd@j@DsWq zD@xe_x$eDwvTZl=`Kyht(>)K^Pmtqus!@1kr&(KQudrh;$pEbW-p zU;(+PZAwd-Wxvw>g|}TNs3^57+~G*zd8c_obDx77YK#(Q@{N%7NuakAP+8NzkWnod z)}_6&mNj+mHq(GJTH{vKktEnZ@icdPEJqb#q@SPPE=ln~_oW#P$h~0SdxE?Grx`?v z&n&Z?7r;9FvJS&r;h$?s>PR>2Hq&D9<9(jr&bPnYpVltj5=O}g z5;Z=YCBg5@^3$@B{yp*XXSWO_`D+B-cQVT?da-tw2*7U$sBTq3SEq+FgabK{EXX_7 z97)Fs+B9E&KBDsUz#uqIZ3goPl^vEy2VacH^H5Q&kF}~`uF3ZJ^sU>!NiF5T8fe`6e^-3lTK?>b*HgJU#G8J zVGn1k%h?2qC~@7;^NHIB8=!p<>E3hHe?@O^Ie5Hx7l%zFmmbU#RE(mVlDlR;X%~MJ ziQHg6QQUSDzP%5D9Do)w4!{WzS7AP>Eey+R@m&=twM$eQLMGWi=f#{^7C9fJp_xFC ziJEN+P0We`>13CCU2_^Oq-f~?6_(s}d8O`kWvSI5tL11(7Iv`3>}JndsL%e&GbbtO z!Te{PqMjbH#sJiY7{mUyM!Y(04`xHqDC=O?1rJy9LdmdJKM$GW%<0XcPFZ&MsJ^?a z(!x1$MKrJo^nI8)_D6_B5vu-RBQ266mZeMs|T%zX3q|KlhJrIe;0Dh=gb9%o2Tt zil@~kY6s|4Nir}nbo3HaSqYKnmO4~_H#3xMla>U%9v$^UeZk!&tJfsdd`i`>>sD`+ zp(M#~2eNWc-X@88R}bVpl@-GrnK<3A3!Q@I5)JZO^fYvAq1aQcwbajKdgfT154%93 z`N2K}>&6=}K}dRl;BJx6b`uFHX19yD5HEs-?&MOeyQfO(J#5jp5bZIPB7+EsnaXj? zIo;X_;Mg4S9cbux+QWcSr``K-(%g)?;8_7PHpWsTzwo{4)q#q(sFFGiE9+07DD;p= zlhrQVbi4kfV)tR->VuAt>2~ZHhb$}258NlyV*YD6Zi{+ogxq|(MJe9P`(8*vr^OGE2HmIqtsT}!ELevb`Ip1=N+AE`#(uiCp-yE++5MO>9p78T$8$mH-7x42QuOdm>nl9%@mKGWTTKihA>I zS|^E0`)Q8O1YSlA_HStZ{u{rpHPkjqb*2*reZ1&Z3yyliFp7+r%Bxy0+pc2=Ob5k# z;_ds?zQO``XMZ~`(mkIl00_i0lJALC0eFj>IybRceB->o^lYMg>Jm2F`PXZ6rpQc> z6q{t(O;)Sdd%5RngiI%+g8tidD4dCNQpT64 zfD4bqw!c9s(b=aui8cj!=dawl*rbIdUdSb%$98N$#0(G{7C#^U&qH|D>Vf`c(l#~; z4Arc5wq$vd>ia9*C0+*Nbanyt`gI;IyA@dA1e7)lAje8kr7roeBlCE=-g^NFF3y?< zign?XCp*~t?yA68*8+gV9wApY_xDrm`{VXoiIT}K|CdF$KL@!P_DVXDNK}80Qx_dD zuZqWig%3kH6&1dAzFi?6G*8esEarT$Lg4X#;*{*YsFZxv^ywV6?mg*`rK_%_u{0 zaq*(}@0ExHST&nU;k!z+dL~Zqb9EK`G5hN=OU-pmz1_L6JG`)?#kS z@M~9XZ*Nv|19yPso zUeRPBW|`xh^jU!fruMDT$O;i?demy4*_62sdOKgYZYwjN=~+civJjkpYxCkyckUnejC;=*=ZXf&7@>9;ljb^$T52!z!oWk_0q7R9=)AOC{pM92C=?C< z^4Ng7SN+`adcD1WEfotfL^IeGY37rN33%`YZdP*+Z&Y+@@{Zp^dxOk3V{yi)g!{_x za_|<#?d`kg5(49ta#(%>Ye>Vy!o#8!uCE+*wYHc72qmu!A#_=CA?%@`WyhIK&>;N) zQOrq3#(M!(6*j%Rv>+wwd3C3EaNB*dh4zmJ{AdjfudDMdXM&;m%^aM(A2kkNeYkC| zQHs`EjWgm5@Yb6@AYTwPt~=>p({Y=926R-r_tX8Cfm|V7JmXJXcXm4ebXvA8zXA zQsy#kOy;@^n2#CsmlX>I?7+a-#N>}P+57HN%Y)X!eOBVHgX&UGCEQNkD1`Xl9jqf6 zlQcWzMitO|LS`9;H8$aA=)m7CsL3Cgn(E9<)u8-+K`!wU=BCgAN;Mvl$_td3)4-#K z1C-J>rj{hC2JXdYKmZ2ZCdaK+VE+m8K3YRY(}7?R&3hX$5owSGNV@a=@-`wSS+8${ zxLg&vHG_|LNJ&wo&kzfQ97RaD+BN^|Kec-MuYoexm-%|7nxeMfj6_{K&(yW=UD>_N zKqq)_O;pP6;5Juq2%R(;9|%4^v5;^zmYajwXCot&K_W_Mmw?box5JQdIxTx|e=1Hf zGxGuQ5Q|_2A-~$B7UCDVZsx~tR?cro+^snn6cn`N)%)hp{rKmh3?OEU7PcSOgMqSV zVfmjzD%A+0Ceu&BRFnBwU>lpDJ@OgIbQk|GKOP(!8tpXE1l*mP>94uI3&gIy?U0)^ zf=aU`)F|@Yl^-7;9=Pk8zWr+NVC==%g{Ss*E6(LsJFq@Cpm{E(P&qD`w$d^$FAu?Z z5mE}Ec?fJZrH7>|{^w7w-VTIHbafu z9biNFO2-O78)|Na13 zxmpY6$Dtc;4DIWYa#W7PXbnRCwdax`*!$kWIzs>%Nj2jV?tDsV_xTibe0i}6+_Af^ z%x91*PPdn0sO*_F4`s3YK_MYeAv;IQtnT~?!2aV;_8|EOU;cHf5IOjN9Y{noK=dC^ zj8rVZ_5If_?a65V^Qi6VQuZu9@chb7tY*94XP>mpCVZgpEwtT`r~7Ns5s5g_n?wt| zYk!Zjv8IVC(F{rYkOJDg%+$=x!zZyiY~%My`12dBLWHZW9Osz}c5n9=*z+-py1W_M za2d#59wIFKa2c~|*ATf+f%&tvm$bE`^9?F>VTqpRA~ChHy817#{ymB=KNbQo1i}Su zndeUc=2{^!u9jC?l!M>y!u~8g82#tB)2Klr$@AzqwdmQWan}pY)g<6P{2{`OMD)*< zuHkRyffnuPM-Y?syOr!;T$?UvE)$w<@ecU_4Cn6~dmK@_0;8Z3a++sZc@18-} z+a;nZ_{%*HhGL!;jGCIVl(@f`Zrfj^d^7*{S;Snxn-2>ks41_*p3_}qTsmy{$IsMi z9I$AP<=MNDH(o?b@2qw&LkZz$=<|IBqR(B@acCnHyN7kFA@mf{zdn5C>{&VBta(8@ z8HLyf_#Q+Li^wriG)%f$Y$x`lZdcrc;0>|-s zK>)6y)%Qbb$lLZb5dKFge}wQ=J%Qd!7wJT8*;y3Bg;g%sB5BBjg~zB3-`}n!GBQ## zN0oLHBNtYKn&-Ns6CqQe_Dr&;|tIuUzmCx zu%a;s@gLADUu1nVf2Ez-rp9HAmSre9Ww~x5JPZ)6fqp_dL@8Z z#dpx7d@4Nj*XM`K5O!)hw{8DSDd5lAmR!{It|Dq@8Aj|VEaz>_bWeST1ziPJ5RO<94=uyfBrKB z_pJ}-K^zM!-<-s9MwlQB(!LD7u%~Nr+=D`it0QLMwL>F4kET^BYeY;?@z?>u-I$UNvG5)TcS-a z&THmS5Bou6hySRY5P!e=JP>G@Mi@!U3;fknc-W+g6!qM0y*h`8i(uS{Mkw|aQOMu& z8dll1t=SBfz1J=<&~J+7U;%e0WU)0}=~H$zw2gRX69f8T*a7!{c0i_A!%rc?gNEPe zfyCDgILDe9mkv3j z4W)M3z;j-nFB7?2%Sg_mo|US2ZIEQq`em}5lSp>`Vvmtvxz z$J)n=!2&~+gv0j-rEHyd5K?TNkFeQ}5v}fjYob`{G987hoj>+p*H@Ayr$GinZY5$e z21`*dATgdr^BEe zXI+iq?8;H2-vkNR{8oGS{VxfZhq8-IKSyc4F_MFRKkmrRaVz|_bFw8)6Ty9hSH^+* z)mK-z%jd98}J!WmR3Ox3@AGlRY{m-rE&L<${1d&y(;h0Ut~482JK`Lb&jE- za;x1P8`O4grZO1Zf`C-Cbww3ik~F}rj#+L@Q(Np3b~Lk;C?HJ@@B48DMMk|-$flNP8k+_V zz%Z6qf9BV4aZY<~vy9Exd=6+Nm(rK7kQsq|JNoZ$Kh#^#`?|zA z>%BpgVI`X&7-QgVHNa3jQT*q3!lc((POaX8NohSpp>=h2(u~;e>S-o$&o$F8?knuk zc3)eS4wj4Xw1g{tiqBxsf9U5F^)}Rk@F&$yRr-~6=lX;X*Qfc^PBo-6CvD?iN+r$M zKIc6PP!v%k{ofX^CX`l4ReRdo8$BuxssRZhpNRea8jnEB7eCIh zdcu{Icj&Vg#t^_fA!oPp4EtUXG#r=k{FRUEMqg<^iO|4^gX_xI5nB1w-CNqNFXc*` zOXw>*+WKm$G6+#3#Toka&CT`YYA`2Ad%QcUHtN^&FLtxl{>SbPb~jFF zbsbK50g`%6!mjx^BQCs!2HmwyvnGDLl1Wl%nvU)fI9mQ@NTG;`mIMB}2Nmg~IdbFu zC08(CMidnlN4`9ch*;lDMeR7d_5sgi);iZ8FJRI#IyZ0Mn;<2Vt{>-O%`hAspbM0M zYCL~6!_V)sF5__^KuFN3hNfY+V2FxPAC%89VkK7Bwo0c(97%U3jccjFLPJgDA3y29 zlLB0D2p$a8A8&swBt>{_gC~UwiOgmMXb8+mQHb5~QM&Lu1!em1dcAr2?kr>)u~1r)K-8_n zXD|Pmp3WFum2TJgY`uG0ZUOx@J>a=r*)q(dPRKC4O^VWg-46*3LoUGP;u3@_k zC#gLP-u=KS0(-7Iq_OLHHX=3&nhvT)X_}lD*kV&o);7@G8f0tZV~WZ5Teyb=J-$Z}!_1B$&m4=xN!j_jhc|EV=E@Js)o?gS~MxS?})4w{vxZ2_7yv zM{iMLvC(j61a35#fxpjJ#U_wL~&w?d^K7s5?IB5yQ)u{rfz$+Pk>` z$1C0C-2;3!ZcVPKI@kH7C|p$tPF9v?3A-t}B|$Jh6{D8h8&n&)y5dM)Ez8h*`5DWV z`nU@OBhR|D2p2_-URrl_L`vHnk73nd%SH^ZzAlr;4~kM~kC=@}jQ7vB80{_41H0SK z9Gg+XI$v1D2yJ$PcP)>XmFL%B^qp ziu;6Kx!Wrwzv}AhlK!ngLPV4xFhc@Ems>3WUnCswEpar-(<^-!R0SPM6Q{Zvg+Nwz z&#k%V8VT5QB1As=J3~K^@|g<|3h7_pOn`lok=G#C?UnDT75F8M#BzUqu^|&gmJr#- zL80nrAe+p9n05NmNa6gTtag;&o*cr=f4(zLkkYada$(csNB#`O3e$R6lFzy3xd9uBYc z0cU*cEu1`mh!y-IASOI}=_oVn`SS@wG9q?JpbRNq zOp26y6Os&UV#+o{wjYOt2+5s7INE zF{+l1IM}ilnUqoXL0og=~`*0+7cV}T40=studq42)pC7n2$jZt( zD7ohQhu$&N%voit)ru3ejqM1G&CuocJvA4V~8+PHA%EpZxA@# z&yd&u_enQ=Fip&Nn2g~GQX!3Oh_~+^pZm`OW))(J>CtCJ2vcdZ-!=R@4xV^mk zpR;%|&Mg&%F@QI8?G^`>41=uKZ4e6V)n+3J`HLGb%?+6U>uEs;7y}Axbz%}4huWzT zES8qe8-ny0L9-?$_##tic?)k#^;b^$udCMA{d^O^HZCj-o^2^e9fv47y4Mh+A3#ED zY<|%7!=BXq?@O0P4jMsIu(_b_u1l(istHqsr850|vJc`b5XeEYcIglQ{?GAxSg!_H z{$&U~_*X+wjA(&1_9H|1^9=uaou>{tk!oy7lFC-Tfh5U%N>hV0!vO)+K-Ue?H`rl= zuH!itYd=S@h6oT1$-P*GzWYi~b~>FjqEbO(V9@!oLR@*QTxTyo{riGy+S=x~B#Nk9 zP>nwWd3g-b*(+$$6xde~z=sQ4_b4HG4Vi_ndR;VT zi;m5z!bGnYq&}Iw4)yjy8!J(~6JFati?hL7OVF209qLX&t*wF1g zr1GTc#lz@kZu&jJJrUGV)r?Tb(y3T0t%$T{Pb9{Q((q(W1P63Eb#a{(fT%{~*Lewd zYm;AW<`HRDdxonbt3i5-#1J4Vv$49M3zNb}EL!5vB0L7jSUW4)s3gkfR-m(V^sk=- zDqtjaekmg%)J?%c!Sf(km9(8YQ=owvg0iT+*aWRL61`&~deyu>tik&b*Kc%jTC*MM zmsF6(32@OaY$=4&+zofX50b*xNV2k1J+7&SdV0x7Mh&?v3xfAR*@LIF$rOU?Jt&(j zxnQ`v@m^a;`-(xz){CF@Tv~>)zFNl;iMT~)8T;JOmEsI2`z`9Z-!Bv$6?M*YeLe>2 z2a?QEuY{bAK~fj|i}kg8{YBFOeuJ(oG?j0FY>9y?o+bdI22**8klS`~x~u*927905 z-N1Z_%KDoo6G3KzvmK}MO47k2f~T*@qE?J^0)pae@_6p`UWhoIm%#d1m=h1_10n}D z*{mM&bm#d)B!`hbcZdTyI>VAtZe&zyJflWn_V7DND9Q)k_MriLr4jn$_D|kK;%k>K z`5skRpGY4C=^_!4Tdtx^a)`_Y(%9bfd$9_+@DC`V3Q8Whk}@kGX}2DK4aug6%7LN0 z`lpV82wJ+xvqNDXQng70HQnSHP|Wuci6K7%yz38&pFITrfAfbfi4u}CRt})%nSSf- zi5?6+GREHvuHXc&IcmPe^v8;WX~e9KPO4!JBtv^V0g;h=#%atU&w{8b74qgEKXXlI z+PXu%n1nFmEHB9nqMpC}>b3j30-A{EoBs1U!!MBoKi*<^g!VPQv6CJ$d4#a@N2|Gy zn}gnb50WODYnxxnRUzR!pLMmI)^p-Z``6)9ETMSDh+Ugf=@FY4uR_r-?tQHbM67?W zSb0NQS?pxAM|u0_2WE;{<8$8d47iX=A)g1EV8SEiBSzB(Q&2&$%$jPJM>KDn!huz6r$P^T(4Al z;c-OxE!xPZNrjke&i8#@962$@2RjL^jNnoaC2Xij#?fQS25<>$V@e^^j#&qHl~hiY zu42xa1OqF2mat5|WN{X;>EDK0mu{(kNEtXsX7QqNt>rs&SvT%W+U5Ex6}--WyhCv) zc4eUMZruGY&u!yy_?C13`WA#-02O^8(kThzTevcQ1^<34z)x@084}L^z3y6gv%XPG z-%=cvw<=J#K)n9*;Wi<{aKc)+t#tYIC~kr@7#|2T3lXWZc}KckS~TI^@5Na`BU2Qktd%FWzI<{oQ|sNzVi+xFvf@d_m8QM zFL-r%SQWjCtPr2LLixar@%#mRftU0ePmpx6{<~znYOLYE-y|1R(vVfJRQzfvt8VV3 z*rAw|`sSypaaV;xno|q!)FniY{nw6WoDpHU5u0-R0#ds%f2VxF+GXJ6S;nvL5fWoKa4udc-Z{EIiS-3gqz#^FkgW9@?f+JoCrNy@YohxuL2Nkjw*MLsAi znE%|U2M~MD!XmR}jcbf^^6;oZWX^i_Y&D6uj$y9-_W7-_DF<;@5s|e3oDy{N7&+p; zEnU19H2w9(mm~>DHHB7=Y9t`8$`CN`LAD4cDta#wDO~g)O0oQLZ?I&qq5~XAB%%w% zJ=ZV)aP$MDBqLu|2FZ})WW+Ppj)q4+`gx<9MhYU5P)4bn2wV(glh|lZCP;(2Dm8uVn$`)m7$dLkntC=(S-0 zf(_}B9csi;{k0Bhpx6M3fKG`+Mv&}I4JoZ)UCVbwqJ>_YIgf;E(DA+&Z(y-*higd? z>Z`!=d($V4|YS=v3_WzM3d;v=_LypxT^l9_!L=h zT|h_eW6?;HpLg~nvO!Qnah-;imIG9~!f!?HFB(zm6=$xmPLD@0qwLI&iY%_uUrbYE zxfbVJ1+78_f&E{hvH#D94uKi45-w9u5G9TE*LOu$J>cgj&pxyo6L5 zUKtlqJVd%91Z*1tBdHE%x&CdOieEA2cl{_Vr#S`yG5>8Zi&A~MmuBgO^> zw^-%2A?UZV()%SvC5^(jt0y%= z!Y%rfVGAa0@cxp)R0#Usx?~IO;or6hHgh|0SFUy~4K4WeF0v^{Y3~cbHVNTl+D}pw zju-=R5{wLsArt@pV3m$oVc2O1)IZaO$Sho#Q}f}FtW6^@np&2obx)?BLGLWJZkdFt z(h_>#NX4%Qm+ea-*_$CVP^`vaA)DEI`eXkre#57;m+uvIDp=`W7!Cd&J33kRtKY^1 zLhT4-g!G}ZPj_{3PzkQNIBxN-vl(Q`^gDJ~+g9f4=ri$*{$0SvvWBtJ5m=R@n%Wm{ zc?4jWb;e)>ByJd_J}IK65-<#A915vaMnn8Qeo`5L=6a_rQa#M+D#v}zlioN~m+p&1 zeQ|~=jqs-$vJz0qk(Lv+?A-^wbw*z@!MAcY|M}K)wal4Ty?V}SETvUZ*ibJ-y`Xs;!w47`xu3ttuL}kI&h{nc9tb=>B+^*5bJ5 zFs$DOV9njz2WW4w3ZLXoYT8?NoxhfymT<%*v`wG}Dw1)hhSi~`wfN7X?k?UGz>QQe z=KfEPOM5K;5Qn;=ZsBWgBdp&>0Z#}5tVKZ5W$FadhDF4XTZZcuk4p7^u?l$ZgXL|t7FWp@t$ z%hzSr+{J@VYfOD%)fa@+%r>5~D!<^aHJqR&G+6|ZVQd>~c$4yHJz>1(81M-ZkQnAh zRt+BitTYNVeOJ-7Dp^n01G5J_l%+=+rVP9fOWlVCS+mD&IlFKFzLbK0PW9AcZ`Xf3 zt^oeVlf9eu?)S}Nt~EULf6gDg99>*@ZL!bZwYf)vnK0RSd8RgvRaw()A`r`CHzJ*i zwz%(gW5vTu2$%HLy=5myl>hzfH^v)W|GWhk3V+XKW#L(`5GG>Quu9aX*i3r0oY@Lc;zV88>dGXQra)nSFJBlFk06#?|UP{JUd)sH-}z$ z?YQz%EceNPYcw({+~nRF3XI=O{Cyv0swd<9#NF zixN4~;r{%+)HRcB;nQhYT6w!+0pitNv(=IrvCi?)+Gh>j7MCL|&_#*kq5T;9;=^TM z9(|N;Ukgf<5DCCu5X9%1l3=1(m8ovh6UGX~j0qokjDL;91t*4l4NepbP;3?JwysrF zdns<-l1iavp&r(3P|>n@3)bvv{oXyeyGu=YCW>z@AjLTI>2Xa~t=(>UMXK(MYrjX; zknG{CNmSu97!!cqPfO4i{B2O0k7BOVBOm(ogUx!^=O#qjw>hL+6KHRRl{fvzcL!jD z!L&t;7Wn>tNb`-C;RS4D$!~ojmaR3sOM>B~-8(ps z;NYa9!b*%YAJ2YxCG)`s`BoG2pV;*m)$}-j?21k8)6s&wfBwETX*-RgC<*$|duHz* z{CLx?r9j3!^}UPU7yF4_a!jTtEV|(r;<(by-H1C3|9ah?{i?gdW24nIiYzdSCq3=5*F$cf~8J=&Ub{ zD{Q182?c|cw@6WK|61=JU;ZmjZTaxMY5J5xxf($)>sQ6_RE>+dQ3A**&btQc{jda~ z-1-}Qde$3UaLLb`Am6WNv)39f(frwMRw~$0=>SwrC}I(m9|_{ABL2Vp zvFBSgBb)C>?G;*a0`9|e@gp_6V)lfJ{&X=mUY{TD`@gS07%lMsYvDCBW{zWUvYu|L zJ6oCY0<*L65*FqmvexG9p;kf`mq+f7n|#q)ctwg?rzc#k`*ZtQsa{kq7Vr9P*MFqu zi;&|pr1xtex9e<3U-C2luG0HS&fV|?R&uxJ_V}Rl7@i^`r}JX1p(E{|JCSv%6)?1_ z1PPwkd8C*W;^MGhyVOK?#yi~{a&O~iUXA%$ybhr)G^Zy7!#&_0oaDkIVWC#+I zT~ki2x^E5c5w(!tc!jfMzIXT)y|==8gq4tuv0Z{++~0dwO~aU^Pf4k;79vMoaLSuF zpH61sn*dsNWqoRMuCLQsyZy9x@{Vo0_;?=E+wC3Co<|>7(q`$kY;qcs1rPK6Wk0ff zcMLZ|*mxg%U<^5{By6UidrW0L&)bx?Hdl*C>~GgyUUL13UCJy#g&g1g0#SqUUp6(* zNMh99=iUYjQ5!w_YD>};hr2_A6{o#hQl;2r3pP2o!b)L9|Krc$au`Ymr0Q{qu;lgy8zWFM&}2_L;z6W|K31w7~MO ztNs6f4DlrYt4s3UxX`>c7^LKavRhM`_d(#+UA?eb4hdqfEZfQoHrv4KCun%7Akhp0 zp5n|-G7R(M&rwfFw0lxdNcKW}3M&SF8_H> zE%T>zIlr-2#lu%e-v}VH6H9;!8h;{B7KF@lM8>}%MU#5y>Fgzl`#(Yc13!Fy&*}Vm zpI0ZS?}4z}vCk~-(|q|tsxMte#EFuXHmL@=!;Gh>4ZR#IKt^GuU2xW`-f{$rySmp` z3V*>Yv{h%qMguBmB+d1!&Jw8O71h+#2*rG{d-m{<5S}V*w18?0`i{cnxcSF`kSYX{ zF^<7JzaW_EBM87K6-cJ-U{c-brSg#-Ug!X|Br!(d2DpF>U;^OT zOAp2#SqX!#Hhc*Nya}lty2pt)sXr&vkZbkVfPDjnB15DC4t}jfc*hxFV~&=kK4rF) zQ$XkhgJ`X;#BqEXTS>;^2gAD5k41(qhcQZtA}MJRA>us->b*mexonO#N7)d4O3d!h zkiFr4H^rNiNg$bakCqXG2w?|Ti(KU$P-h*2!lxOzeVq**c_SuLea_q{BohkNg?zZV2dJp$f_ z8_7G1q}a4XR>Vw@>E3i}0}zXk{mwb{RFB?+%Dp+rV3s}}AkUgkElq{Njz~7}43GsY z=Y#;H6@VjXMin5GH-Y(n1yaCER^E0Way5aOfBbt+_OF+^-P@4E4X8;_EtypQ@GSun zMoHmUM+f0RcNKL6Aj3r7!W-o-Hmv^$M2S9;7!MWdVrNNe685K%-_Icx%H+M;>DmDE z4`T>nwsgP!^h!?vwzQCCn1*7hLwQ%ipc4CQ!F-2}=7YnE^=_C*AvP#-L7GV>O-78d z_CmyC+V_|SiwRTM(vpjSF{rX&Lgy*IFDnw5ycVm)kx`7V(;#Wz*`GFH{3GJJObzqM zQ)WUcbUqLV&2e{YwbU+MO(N>+c5A!<%g**YziW-nV-eEJN_%HAG2+Q(BGaOX%$S2% zlAjZaeT8Eoi3Qz}zJ`Y`#f=ukoHd1_BFW0chwDpAFNkw|C&#{!l!apxGol-ZrfMp--7!?v{)a7(SjrxX1uO6n8ymUf48I8-|X_ z-DM~;x1l*muF&lxP>qn{Hef2IzTfy48Fp?zc6+|uN}k$>EUDoHal)l0wd(#$`_C3c z8~tR;eDFJCi_#^lHRP&0!f3O3A2k9aQ%^z7?p5j~B$c&Uar<^Ra1~9guyfyi1l~Vd z)OH%A6m-?iIqoNq1j4vhgT=NSZYZ(Pkn%d&We1b|P8~aX4;d1Q4wKi~OUbfgYZrTu z2}p_o?<&5FUonBnoWV#o|LP@+>C%{cjlzBjK?bO`F6b`n33gh` zh;`~GD9)_n+CWk!Tgmq3PO+U;E3?gkm0l{*fIS0eEkS~LMZ@Avt?zPOccd5etCoOK zZOLSM0eqqh$!kxzLt~G6l`Q5Svs_eX?U365&!eId{G^yw84hrpq@1wHg$wlbKS$KN0Cdt}PFhKNl z9VT0=(@g*TVr!U?zx%UT9pMLPzTf#jsx%#1g(Z-Zt9rD+=c5cDxZ@=8U^9ejRGiVgup&gEKM!^buYjx=)eiyXVkJUu`SV7e=w#heMU0Mas+Srd& z^f6EIprHQo)F3?k?T}o8MjxioaXnTEpn_y|>f0T6~p zjPke&7KUN3a?w|5kHw4L(=Gda3ucG*G%tb4xv@H;b*}*d{l29)6lB#118Hgho0=ZGG$A=TWK%C8sf*O^A7lbxK4oO{go~w4SleX ztKU!Q_iGAp+tXWRn#?)_$s>Hq)#L%F^Miw{%Yr{~AVXCuGh)!;H|l3ErJCiz3PH9n`skzv7*x-`}>hwOlmB$2KrAY%D*+b*$)_c>zSGJ!1+MHaH}mCwYMtZ#koA-M3*EKSjt&H~&BW{yIJ~|M z+?GWyAk{6sNXd)0Z5KBAnBfNEAM86YmT2=@xwnS+mdEu;tzU&JoUZ#wjil1y8#M+3 zht6xjl=Y(KOdJ+Jrl@ZX;vil8>Qf7bP~2Z77)U0MKN0izM? zR|R_yM*`9qv*)xiP6LdDV_Sm9$M7^LT)f&__wISgUXUK8edz!Gzi))|xb$hRgU?bg zpt+AYY|(CfLR`vL)5Dsj)bu%qT`*DV14>wkVXf%xhqG^toUZq3hbuG=?OBuz+g(RJ zJsABLDy-<{P1*(^=1iM+G%%}?m}pP){-x7DTvfI5*>$uG0x-qt*-hE1G}LxiU;VhZ zcQS5n7$@zth=Uu(t=(q2&_9+4;i9LjVf)&ctFYa+KEq&bv~o1hU^j$SYx~ojF&ast zJfMoKEt`$~LaL{=uv?4$YoT(i22C^Uov20M*donlK>8dmw(!3hYyOuweyKc8lLnjJau+3)qAfj=E zV##`_>$<(5Vwx8XGzkMTw%Ydnmz!?^c5i@ei3yrt_*B5bJ1)}G^CmYH%?Y)3^zchY zwlG2Y?Q~93j&oBQU+Ir1K;6YoOQz`@7=;p!2!t6zlCxEBsh>q!43PRvdxsvB=Suwq zjbvO_7e-J++lWO1w}Me>KnB{co*aRxHIj!aNy5PWU&!+GhBzY_p>KR(SNC$YUIWj2 zoP{N#*0OYJ6-xLXEsZdc`J#|~NL#_IWY@}N2#(uQ92S56!#Cp)msT)1#YhxChVV#J z3D;}y>xi4RA_fxaR7@={zAH8NMqbZjA_m#V3`b9xnt%k}H{1zu)9DZj15Iu6_2SKL z2SdtpyybK6HlTAU1B5Fc*6Y_qU16LqC|6_mEAh^+Ap>E*96|<}b4HZJ7A4)MW@LP? zE9@&`b`&SIOWR1+gl@`WOTB;Y%d4WW=h#x5#E6%=z8CoNoY`s*vHM~GmIikwAzjR} z6m9@XX$%V-f@EDHU6%1n>B}DSnOfEiHFPl>NE*2R@Y!u-Lg6vDQ?{$eaLK&ldLPCkM_H}SFP^7I0M&113 zut91V1bOw~B9b^6U+*LJ1O7xzA4T_3ICjPJ@q7OHrF{p8vz${s=dinkX?!;kJ$PB` zIpeu+Z#lTTXjt%!kXtyytj@aE<@1TDXa37B(qMAo#NFh6%qRN_#j;}OikKlQB-dc{){&6c zm$4fx(T@FDG9Lc6Q-MC4XnfONx}QH(Rv*TOcLHA0Q=c)2frfNi9+&&-yw$y1SC>g# zg;O3#bNS0k$`}Bqvs$q@ctV#wX@hA%nKUystBN&$Vt6-NvE@>_kOL(B5ir|FKyB%V z?ovy~QKi6HNbDk0S!O6Cv!5qT$3L@MtPo3qYjr)6|3&rGNm7$ze4J;PlzfdqzOBVa zeYcL|LTf`K2QHTBGAG*iy9Q>f8(qsx3ZigyHj@XZ_pvLNn0g-#8*#owf+DNL(^>_# zM0`w{_mibK8FLo+m#eE?G<@qdRd`|~C7!QZDLGb_C}>|6K5@iL50$!4@)8Yc?mmU= z%lx13v0L69EZ5#*z^87@-05!EsQ^MaYg+rD?Q0EDs+Yh!=S6-RS@J&UgnkHf$(eH^ z<>-!B)3r&NNriDJL|E!kEVAPY7aMjkwWMNoP{Wptp7c09MKemsjcn1j%(*B;nnEm; zX4Pp~WS&Be%)93-b|$AQh?a@ojaGwt)=&tAc?es**c{p;;JN*NfE5fy8aZ3B`m4+3 zj);=sg2>nuwj^O1nZoQlkp^jMe3Qo{p}po{uOC?ispz*eUk)97lFz`Be2tnvN=5N# zM%{E8>A2oWB8lSCby+Q200NhVgPT_G%xJ$pkdZvyrYe$~nofqJ(~Ri0Gd{H_hTT-d zg9XsDOniMB@Zd-HkK|cSNtjmd>mE+U5H=6V=4#?HGJSJ%TZrwJt|{|Zo<-rWvunQP zPfM8vFhbO%@!GYni{_X8>*=T{NDqmSuM1OcBRR+_3IDl9XE`P8*33(-#`KKXN0<2T zJ?^O4%g=(6H9 zBkiQ^RARn$q0Dd5XOr)|rohFuJvWQf7j0c^c1>QF8V^zHJu#PC8sy(f%&0OyeIQe#C}zV$_PE;9*2_k$ z5l(5-#nEC1E5L$_LwuTaNi5T|uTi}H$l89!hX|B7!}_Bv&x!tRws1r5vl6K>b?e3G>!qiFh!BvD%TIK~r*ub#L&N!otMbsytf zRUtjQ(h1FyaW9;)OVDx}k;Y?}gB6Z6jK;B)6UW!3x>FM>t0h%mAMbs5rqQ|D^?SnE z_!Bg@Zr^<*%qV~w6mU|atxwd6?KnaF<^CYZMdB6I%LCfj`x6gd?9b+Q%wTMKY)Du> zHt#K2qr;*kbyW>m=1aoQr*!W62i_Ik)B_Cr&RqJyk=N%O^x{Tiv@-6{P;}OxJiQLoqq&=J8d~w?J7<6xek8amsiHUC>Jf?2wuAE zA>VhYaS~s+&Chu4pw!m}O)+k`s#ZS@37ffE{*nn6f#>%JEYy$E8Y%2k;q}>>HOzg} zAYt)~couuKCt5|1MRN5y`M2ztykcHXl3al(p)31}$?9XlfE3ATmocUb(r+f?ogQ2GZl|>7Og*x=Rf}(8%7189 z=WkWKT4%GwE`Q`y=vPPV$xH8y^bI#mtR*sOZLh|ZA21784nEP|NSRdCajpB=^$(O$ zr46(?#s`Bs>UUb$xow(`utdw)-fc%;P)M2?xE_=Ci7Y5IT#Ia?z5n_SJLdMwCKp}F zRQ3?|a@wKH%$U43uQulzk@S?y7F?sgiw#8`wu)mUDO33qtawMYk`LC>O>~k`?A>{R zg%q->l=~${BhJ&V-EBXn;FzZof9^#Ng!E?>2vcvLy#avMKq;qU;vAlWm*$aiAeSN` zI5%Ta@l#FC>15{`!w-E|csVlu*TFTv;o_?DD zSwkaprr)N_Li(k#?di3NM5|8mz-Xyx7U7#m2mDDD_-`V(!vPKXc%!H={FmMV6>>h7 z_x_6LIhX~F@+HoES<&@yxN~Cp4mV{jvGu!L!qihT|s)Q&UDPjPvV*WT>3g$adZ_$SRlSke`q9*!1HoG{6O^h+*>mmul>X;w`x*_s zUHKQh%!XueDQ$WpK*StdI@9^hO!9yY!IJ35tuy$|tf%T%AJJ7-@N)8{2GlUitZFl< zqB7Xx+h^{qC`Ry=hg=sgn=G)3-%`-vmBIR3pnVd|qwbV;us41M%Q&v{+5W*>vWHhg zI+ySY{xb~{tVB*#rM*_-Z?))Iyj}R;`h+qKfOg%S!$qW8KAb=zx zN5H^t`Drnw&zrQbLBdUxA&ChZ?o9q3T~LyAKn-4 zl(L;>BY*hR#K{F;Gq+22ip=-e&BNZP`r#g0Wyu~7laFsU|DF$1udNdA&3B#iJ*rx=rJ)vGIR&c(ifrVT3UHntzNPR<{`{r!;HF2!)TCzU8{f`6I5s@} zs5S1 z%~~#5s_7J`{rs(YSDy*Ib86q<<4!vT9V^?nzk=S29bC-s!VxTvoz8q8+Ne27Uq&Ce z9UH)>CAmWG)??DoMta*|y4LatDPN9>fLe6h$tJGX`%r-s`&28^z~) zag`R)o(XW=Hl-Ajzqv*&D&mMo!z~^z2VN(ic|3;~x)QzoxRWE5mag*kwjD#$nf(mw zJX*FK4W&%cSNthN#hz)!P}~-s*yp+*Vx=2`19!z=MtG=CRZQrDBXPdV<}rSa>H6@s zh?Dh4zm&3g>QOeG?w51FfH;Gntx3SJ^7VLZ<|>15+;aefRZ^7R%72z=azM+625z%*jkvy+V9mknRba#*+5DH?)t&23NSRSy8^$xuc+XI84A- zygGc%)c+FQ-B6;F`=_U~AGQ=-p|kFLTzZ=4j=gd%$L{yOGnU<-6;D9NY+Z9U{+O`N zi(jReq)!OP-@7Oumsh452%ZmnsQh$&smZAr^&9hMbnoy@&efAwLVY?oJ~_+j)U8_3 zRpwtG@B5C|(zxdCpnqSKhH(`y#J+QVb5&y*>dID1%du1bEQyWftL^3YDm&L zna=%qOL6k&1xa!a>~nhdi1?do>2Fht?jN*PP_^2{98_Jg zFkxy#nVn22=%K`?;3}uVsfP}qq^nsyvwWuOL}qT>cez8g4pM2o#gocmirE(zcq&V@ zrB|hw$zDNQmh5Ax!H*31(ci2(y0+VzEv6r`&MiswKANr5*7G5YiI4Fae-j>1Pu2a+ zvc*Z(s-$Zvk}+Q>_}%;c|p+MbA?SFp?&{p2FXwf!mcx=L-_lj>CtbW?)0 z(sQF=A(_%>d0MpNbnAo=r$<%bbq9rHUI-4gFG$SXVo3IEqNzEO*-x8#py^!B*{&&L zXBtsuW7gyVzV=|VYvTe}(j;RQxNSaoeA4_yV#=4-mLi+_%I4YdnRp9aS~yz`OTj@W z4uzF##V5t52F)$*HnH?eyG$*uw;cT3P^oTl#C2b9*g}SLF#n}D?lYKiikyU@y3KL# zQhaIE1Xs^1ybL3*kJzTCIvL|D*6Y_{dDwP^hVVk~!^1-|VX9miouB&Y_8$+s6WD!R zQ9g#4tD88X=H^JX67lTkcZ-pgA(5TzIV6ty@fR-3O>3v++&#sz>xE0*0$9Kz$uwFiwcD^+f1nx3Wqm! zbET*cn@O9ItYhpJJU^%>*fh`;&Czu%T^{-)KILBFibiFm zCt=eEB)QCqiM)3LA}gIP*sM4h&>y2rwwc-A^fq6%E%g!JCiluXzI*Zq{(AiiEDQt`kYz9~yrAP*dB0A;LFri$=)Inek z3MvJrgABRV6(!PfN!C(VL=hx0FfIu;=XuvOi*<`sJL{ zx9|I&rhT5@^BhQ33lJD4*V!9~@BXI$(g~$x&{SURadRsX#CUUGQ^j@bhG|RQ7!Yi9 zJ_UBb3?m)P^~pokes5XP0&#R7c+uF2TtqSG2=limqv?wKFxa6FP`o{!Ez4J);LJ~p zLiyz=8oV~L8+|FKc5RLnh{pzuC(^&uK_hJ>pMrtYaNpny=~Q_UW?Hm$OGKrtcS(^1I0xJq#Q^BT!7>Bab=Mn~DnZSSZ>eAW`8 zapuaJhW)kS7NWOkf~*YLmeDp7@bU3+FW4`5bPTI^WOT<}CV(lU zPF)$7^{3s429NFV@qoUV9WN* zHQEyKobW@d2Dlik4_F5EH~^Dhks*JGwjn)CU3)%rnf+|v&Tg9sJa(ZXh=P7?vpwzB z-d03f^8R^3g`hG|(NFtB4U~Og^{lr$O@Q$~P-Lf;S@WJ{fJ?SuK< z5DSS@O9gSE^r2yw)KZA*}Z{oV0^WVlcqWT5A(dR*35 zvDX;!kN`XK>uF(S$FbR>YLW`NiIi{r^Eq1`t|cWg%9%L0+!I9QPHVaE#zo>qrX6NE zBk7Qi0@{G|js@AG4sV}B>H$uknGK^dH3oOcrI08q;X-fe{22g2N<={>v%9o4-aS`uFfY@j(v52a4>Mvz z5Xvoi&f9y5h0Yz>yQ%)?_V+3IKIhaG^KxPn|D~UTAb&8`3_iyjcFL8gBxrrdR0+>0 zpi1`C+VUL{WORCBPeJv%@IN!h3UZ`akXqmE;@1VUlN;RTOfnIg_9#;~&+`Wpw!X(h zR6rr|Ow+UwKb$Lfx--Is6)dC69V`Bh>5n`dCJ^4k-O{k9&pf|?ap6OyZ|1`7Xin}@ ztK}!6oVM|&g>}LD7jrFY`S4DYTHtWAOIsWpZ3kEc<}(%3!UhwKt5|8bcGVXe~Yd!rXFr$Tr)^rN%1PM2F5R2LUX9Lbjce2zf; zl8{&p5eF>t4gQi-n)1N49Ny6HCM1u7>{d7|@s{;U7cBZqLiaZ)2dm=74TFk+%i|t2*P+pZAsnfg+7OCHGXGfIG$s#7jhrRWyghA zGw2ASTyG!NGn=?7En?rHkdPJEoz7e|5T39^Rb}yM*pW;lw?mU!7)ka2UpMB#;ALag WX4SmZ@jYUGCSk!5U)~H#xbQC+{ei&% literal 0 HcmV?d00001 diff --git a/examples/hello-world/hello-km/requirements.txt b/examples/hello-world/hello-km/requirements.txt new file mode 100644 index 0000000000..b02e27620d --- /dev/null +++ b/examples/hello-world/hello-km/requirements.txt @@ -0,0 +1 @@ +lifelines \ No newline at end of file diff --git a/nvflare/apis/dxo.py b/nvflare/apis/dxo.py index 90b9cc5819..368248e722 100644 --- a/nvflare/apis/dxo.py +++ b/nvflare/apis/dxo.py @@ -29,6 +29,7 @@ class DataKind(object): COLLECTION = "COLLECTION" # Dict or List of DXO objects STATISTICS = "STATISTICS" PSI = "PSI" + RAW = "RAW" class MetaKey(FLMetaKey): diff --git a/nvflare/app_common/utils/fl_model_utils.py b/nvflare/app_common/utils/fl_model_utils.py index 2d84daa14f..486acec19b 100644 --- a/nvflare/app_common/utils/fl_model_utils.py +++ b/nvflare/app_common/utils/fl_model_utils.py @@ -201,6 +201,9 @@ def get_configs(model: FLModel) -> Optional[dict]: @staticmethod def update_model(model: FLModel, model_update: FLModel, replace_meta: bool = True) -> FLModel: + + model.metrics = model_update.metrics + if model.params_type != ParamsType.FULL: raise RuntimeError(f"params_type {model.params_type} of `model` not supported! Expected `ParamsType.FULL`.") diff --git a/nvflare/app_common/utils/math_utils.py b/nvflare/app_common/utils/math_utils.py new file mode 100644 index 0000000000..fc590fcb0c --- /dev/null +++ b/nvflare/app_common/utils/math_utils.py @@ -0,0 +1,89 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import operator +from typing import Callable, Optional, Tuple + +operator_mapping = { + ">=": operator.ge, + "<=": operator.le, + ">": operator.gt, + "<": operator.lt, + "=": operator.eq, +} + + +def parse_compare_operator(compare_expr: Optional[str] = None) -> Tuple[str, Callable]: + """ + Parse the compare expression into individual components + compare expression is in the format of string literal : " " + such as + accuracy >= + loss > + + meaning accuracy will be compared use >= operator, loss should use "<" + + Args: + compare_expr: string literal in the format of " " + + Returns: Tuple key, value, operator + + """ + tokens = compare_expr.split(" ") + if len(tokens) != 2: + raise ValueError( + f"Invalid early_stop_condition, expecting form of ' value' but got '{compare_expr}'" + ) + + key = tokens[0] + op = tokens[1] + op_fn = operator_mapping.get(op, None) + if op_fn is None: + raise ValueError("Invalid operator symbol: expecting one of <=, =, >=, <, > ") + + return key, op_fn + + +def parse_compare_criteria(compare_expr: Optional[str] = None) -> Tuple[str, float, Callable]: + """ + Parse the compare expression into individual component + compare expression is in the format of string literal : " = 0.5 + loss > 2.4 + Args: + compare_expr: string literal in the format of " " + + Returns: Tuple key, value, operator + + """ + tokens = compare_expr.split(" ") + if len(tokens) != 3: + raise ValueError( + f"Invalid early_stop_condition, expecting form of ' value' but got '{compare_expr}'" + ) + + key = tokens[0] + op = tokens[1] + target = tokens[2] + op_fn = operator_mapping.get(op, None) + if op_fn is None: + raise ValueError("Invalid operator symbol: expecting one of <=, =, >=, <, > ") + if not target: + raise ValueError("Invalid empty or None target value") + try: + target_value = float(target) + except Exception as e: + raise ValueError(f"expect a number, but get '{target}' in '{compare_expr}'") + + return key, target_value, op_fn diff --git a/nvflare/app_common/workflows/wf_comm/__init__.py b/nvflare/app_common/workflows/wf_comm/__init__.py new file mode 100644 index 0000000000..4fc50543f1 --- /dev/null +++ b/nvflare/app_common/workflows/wf_comm/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py new file mode 100644 index 0000000000..065624ac8b --- /dev/null +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py @@ -0,0 +1,144 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import time +from typing import Dict, Optional + +from nvflare.apis.fl_constant import ReturnCode +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( + CMD, + CMD_ABORT, + CMD_BROADCAST, + CMD_SEND, + CMD_STOP, + MIN_RESPONSES, + PAYLOAD, + RESULT, + SITE_NAMES, + STATUS, + WFCommAPISpec, +) +from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue + + +class WFCommAPI(WFCommAPISpec): + def __init__(self): + self.result_pull_interval = 2 + self.wf_queue: Optional[WFQueue] = None + self.meta = {SITE_NAMES: []} + self.logger = logging.getLogger(self.__class__.__name__) + + def set_result_pull_interval(self, pull_interval: float): + self.result_pull_interval = pull_interval + + def set_queue(self, wf_queue: WFQueue): + self.wf_queue = wf_queue + + def broadcast_and_wait(self, msg_payload: Dict): + self.broadcast(msg_payload) + min_responses = msg_payload.get(MIN_RESPONSES, 0) + return self.wait(min_responses) + + def broadcast(self, msg_payload): + self._check_wf_queue() + message = { + CMD: CMD_BROADCAST, + PAYLOAD: msg_payload, + } + self.wf_queue.put_ctrl_msg(message) + + def send(self, msg_payload: Dict): + self._check_wf_queue() + message = { + CMD: CMD_SEND, + PAYLOAD: msg_payload, + } + self.wf_queue.put_ctrl_msg(message) + + def send_and_wait(self, msg_payload: Dict): + self.send(msg_payload) + min_responses = msg_payload.get(MIN_RESPONSES, 0) + return self.wait(min_responses) + + def get_site_names(self): + return self.meta.get(SITE_NAMES) + + def wait(self, min_responses): + while True: + if self.wf_queue.has_result(): + items_size = self.wf_queue.result_size() + if items_size >= min_responses: + return self._get_results() + else: + self.logger.info(f" wait for more results, sleep {self.result_pull_interval} sec") + time.sleep(self.result_pull_interval) + else: + # self.logger.info(f"no result available, sleep {self.result_pull_interval} sec") + time.sleep(self.result_pull_interval) + + def _get_results(self) -> dict: + items_size = self.wf_queue.result_size() + batch_result: Dict = {} + + for i in range(items_size): + item = self.wf_queue.get_result() + cmd = item.get(CMD, None) + + if cmd is None: + msg = f"get None command, expecting {CMD} key'" + self.logger.error(msg) + raise RuntimeError(msg) + + elif cmd == CMD_STOP or cmd == CMD_ABORT: + msg = item.get(PAYLOAD) + self.logger.info(f"receive {cmd} command, {msg}") + raise RuntimeError(msg) + + elif cmd == RESULT: + one_site_result = item.get(PAYLOAD) + for task, site_result in one_site_result.items(): + task_result = batch_result.get(task, {}) + self.check_result(site_result) + rc = site_result.get(STATUS) + if rc == ReturnCode.OK: + result = site_result.get(RESULT, {}) + task_result.update(result) + batch_result[task] = task_result + else: + msg = f"task {task} failed with '{rc}' status" + self.wf_queue.ask_abort(msg) + raise RuntimeError(msg) + else: + raise RuntimeError(f"Unknown command {cmd}") + + return batch_result + + def check_result(self, site_result): + + if site_result is None: + raise RuntimeError("expecting site_result to be dictionary, but get None") + + if not isinstance(site_result, dict): + raise RuntimeError(f"expecting site_result to be dictionary, but get '{type(site_result)}'") + + keys = [RESULT, STATUS] + all_keys_present = all(key in site_result for key in keys) + if not all_keys_present: + raise RuntimeError(f"expecting all keys {keys} present in site_result") + + def _check_wf_queue(self): + if self.wf_queue is None: + raise RuntimeError("missing WFQueue") diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py new file mode 100644 index 0000000000..b11130c9a2 --- /dev/null +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Dict + +CMD = "COMMAND" +CMD_SEND = "SEND" +CMD_STOP = "STOP" +CMD_ABORT = "ABORT" +CMD_BROADCAST = "BROADCAST" +PAYLOAD = "PAYLOAD" +SITE_NAMES = "SITE_NAMES" + +# note same as app_constant constant (todo: we only need one constant definition) +MIN_RESPONSES = "min_responses" +START_ROUND = "start_round" +CURRENT_ROUND = "current_round" +CONTRIBUTION_ROUND = "contribution_round" +CONTRIBUTION_CLIENT = "contribution_client" +NUM_ROUNDS = "num_rounds" + +STATUS = "status" +RESULT = "result" +DATA = "data" + + +class WFCommAPISpec(ABC): + @abstractmethod + def broadcast_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def broadcast(self, msg_payload): + pass + + @abstractmethod + def send(self, msg_payload: Dict): + pass + + @abstractmethod + def send_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def get_site_names(self): + pass + + @abstractmethod + def wait(self, min_responses): + pass diff --git a/nvflare/app_common/workflows/wf_comm/wf_queue.py b/nvflare/app_common/workflows/wf_comm/wf_queue.py new file mode 100644 index 0000000000..6287f2f907 --- /dev/null +++ b/nvflare/app_common/workflows/wf_comm/wf_queue.py @@ -0,0 +1,58 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from queue import Queue +from typing import Optional + +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import CMD, CMD_ABORT, CMD_STOP, PAYLOAD + + +class WFQueue: + def __init__(self, ctrl_queue: Queue, result_queue: Queue): + self.ctrl_queue = ctrl_queue + self.result_queue = result_queue + + def put_ctrl_msg(self, msg): + self.ctrl_queue.put(msg) + + def put_result(self, msg): + self.result_queue.put(msg) + + def has_ctrl_msg(self) -> bool: + return not self.ctrl_queue.empty() + + def has_result(self) -> bool: + return not self.result_queue.empty() + + def ctrl_msg_size(self) -> int: + return self.ctrl_queue.qsize() + + def result_size(self) -> int: + return self.result_queue.qsize() + + def get_ctrl_msg(self): + return self.ctrl_queue.get() + + def get_result(self): + return self.result_queue.get() + + def stop(self, msg: Optional[str] = None): + msg = msg if msg else {} + self.put_ctrl_msg({CMD: CMD_STOP, PAYLOAD: msg}) + + def ask_abort(self, msg: Optional[str] = None): + msg = msg if msg else {} + self.put_ctrl_msg({CMD: CMD_ABORT, PAYLOAD: msg}) + self.put_result({CMD: CMD_ABORT, PAYLOAD: msg}) diff --git a/nvflare/app_common/workflows/wf_comm/wf_spec.py b/nvflare/app_common/workflows/wf_comm/wf_spec.py new file mode 100644 index 0000000000..9954e8a8c4 --- /dev/null +++ b/nvflare/app_common/workflows/wf_comm/wf_spec.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod + + +class WF(ABC): + @abstractmethod + def run(self): + raise NotImplementedError diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py new file mode 100644 index 0000000000..731ddea760 --- /dev/null +++ b/nvflare/app_common/workflows/wf_controller.py @@ -0,0 +1,275 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from concurrent.futures import ThreadPoolExecutor +from queue import Queue +from typing import Dict, Tuple + +from nvflare.apis.client import Client +from nvflare.apis.controller_spec import ClientTask, OperatorMethod, Task, TaskOperatorKey +from nvflare.apis.dxo import DXO, DataKind +from nvflare.apis.fl_constant import ReturnCode +from nvflare.apis.fl_context import FLContext +from nvflare.apis.shareable import Shareable +from nvflare.apis.signal import Signal +from nvflare.app_common.abstract.fl_model import FLModel +from nvflare.app_common.app_constant import AppConstants +from nvflare.app_common.app_event_type import AppEventType +from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.workflows.error_handling_controller import ErrorHandlingController +from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( + CMD, + CMD_ABORT, + CMD_BROADCAST, + CMD_STOP, + DATA, + MIN_RESPONSES, + PAYLOAD, + RESULT, + SITE_NAMES, + STATUS, +) +from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue +from nvflare.app_common.workflows.wf_comm.wf_spec import WF +from nvflare.fuel.utils import class_utils +from nvflare.security.logging import secure_format_traceback + + +class WFController(ErrorHandlingController): + def __init__( + self, + task_name: str, + wf_class_path: str, + wf_args: Dict, + task_timeout: int = 0, + comm_msg_pull_interval: float = 0.2, + ): + super().__init__() + + self.clients = None + self.task_timeout = task_timeout + self.task_name = task_name + self.comm_msg_pull_interval = comm_msg_pull_interval + self.wf_class_path = wf_class_path + self.wf_args = wf_args + self.wf_queue: WFQueue = WFQueue(ctrl_queue=Queue(), result_queue=Queue()) + self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) + self._thread_pool_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=self.__class__.__name__) + + self.engine = None + self.fl_ctx = None + + def start_controller(self, fl_ctx: FLContext): + self.fl_ctx = fl_ctx + self.log_info(fl_ctx, "Initializing controller workflow.") + self.engine = self.fl_ctx.get_engine() + self.clients = self.engine.get_clients() + + self.setup_wf_queue() + + self.log_info(fl_ctx, "workflow controller started") + + def setup_wf_queue(self): + wf_comm_api = self.find_wf_comm_in_wf() + wf_comm_api.set_queue(self.wf_queue) + wf_comm_api.set_result_pull_interval(self.comm_msg_pull_interval) + wf_comm_api.meta.update({SITE_NAMES: self.get_site_names()}) + + def find_wf_comm_in_wf(self): + attr_objs = [getattr(self.wf, attr_name, None) for attr_name in dir(self.wf)] + wf_comm_attrs = [attr for attr in attr_objs if isinstance(attr, WFCommAPI)] + if wf_comm_attrs: + return wf_comm_attrs[0] + else: + raise RuntimeError(f"missing required attribute with type of 'WFCommAPI' in {self.wf.__class__.__name__}") + + def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): + try: + future = self._thread_pool_executor.submit(self.ctrl_msg_loop, fl_ctx=fl_ctx, abort_signal=abort_signal) + self.wf.run() + self.stop_msg_queue("job completed", fl_ctx) + future.result() + except Exception as e: + error_msg = secure_format_traceback() + self.log_error(fl_ctx, error_msg) + self.system_panic(error_msg, fl_ctx=fl_ctx) + finally: + wait_time = self.comm_msg_pull_interval + 0.05 + self.stop_msg_queue("job finished", fl_ctx, wait_time) + + def stop_msg_queue(self, stop_message, fl_ctx, wait_time: float = 0): + self.wf_queue.stop(stop_message) + self.log_info(fl_ctx, stop_message) + + if wait_time > 0: + self.log_info(fl_ctx, f"wait for {wait_time} sec") + time.sleep(wait_time) + + def stop_controller(self, fl_ctx: FLContext): + self.stop_msg_queue("job completed", fl_ctx) + if self._thread_pool_executor: + self._thread_pool_executor.shutdown() + + def process_result_of_unknown_task( + self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext + ): + pass + + def ctrl_msg_loop(self, fl_ctx: FLContext, abort_signal: Signal): + + if self.wf_queue is None: + raise ValueError("WFQueue must provided") + + try: + while True: + if abort_signal.triggered: + break + if not self.wf_queue.has_ctrl_msg(): + time.sleep(self.comm_msg_pull_interval) + else: + item = self.wf_queue.get_ctrl_msg() + if item is None: + self.log_warning(fl_ctx, "Ignore 'None' ctrl comm message") + continue + + cmd = item.get(CMD, None) + + if cmd is None: + msg = f"get None command, expecting {CMD} key'" + self.log_error(fl_ctx, msg) + raise ValueError(msg) + + elif cmd == CMD_STOP: + msg = item.get(PAYLOAD) + self.log_info(fl_ctx, f"receive {CMD_STOP} command, {msg}") + break + + elif cmd == CMD_ABORT: + msg = item.get(PAYLOAD) + self.log_info(fl_ctx, f"receive {CMD_ABORT} command, {msg}") + raise RuntimeError(msg) + + elif cmd == CMD_BROADCAST: + pay_load = item.get(PAYLOAD) + + current_round = self.prepare_round_info(fl_ctx, pay_load) + task, min_responses = self.get_payload_task(pay_load) + + self.broadcast_and_wait( + task=task, + min_responses=min_responses, + wait_time_after_min_received=0, + fl_ctx=fl_ctx, + abort_signal=abort_signal, + ) + self.fire_event(AppEventType.ROUND_DONE, fl_ctx) + self.log_info(fl_ctx, f"Round {current_round} finished.") + + elif cmd == "SEND": + raise NotImplementedError + else: + abort_signal.trigger(f"Unknown command '{cmd}'") + raise ValueError(f"Unknown command '{cmd}'") + + if abort_signal.triggered: + self.log_debug(self.fl_ctx, f"task {self.task_name} aborted") + break + except Exception as e: + error_msg = secure_format_traceback() + self.wf_queue.ask_abort(error_msg) + self.log_error(fl_ctx, error_msg) + self.system_panic(error_msg, fl_ctx=fl_ctx) + + def prepare_round_info(self, fl_ctx, pay_load): + current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) + start_round = pay_load.get(AppConstants.START_ROUND, 0) + num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) + + fl_ctx.set_prop(AppConstants.CURRENT_ROUND, current_round, private=True, sticky=True) + fl_ctx.set_prop(AppConstants.NUM_ROUNDS, num_rounds, private=True, sticky=True) + fl_ctx.set_prop(AppConstants.START_ROUND, start_round, private=True, sticky=True) + if current_round == start_round: + self.fire_event(AppEventType.ROUND_STARTED, fl_ctx) + return current_round + + def get_payload_task(self, pay_load) -> Tuple[Task, int]: + min_responses = pay_load.get(MIN_RESPONSES) + current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) + start_round = pay_load.get(AppConstants.START_ROUND, 0) + num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) + + data = pay_load.get(DATA, {}) + data_shareable = self.get_shareable(data) + data_shareable.set_header(AppConstants.START_ROUND, start_round) + data_shareable.set_header(AppConstants.CURRENT_ROUND, current_round) + data_shareable.set_header(AppConstants.NUM_ROUNDS, num_rounds) + data_shareable.add_cookie(AppConstants.CONTRIBUTION_ROUND, current_round) + + operator = { + TaskOperatorKey.OP_ID: self.task_name, + TaskOperatorKey.METHOD: OperatorMethod.BROADCAST, + TaskOperatorKey.TIMEOUT: self.task_timeout, + } + + task = Task( + name=self.task_name, + data=data_shareable, + operator=operator, + props={}, + timeout=self.task_timeout, + before_task_sent_cb=None, + result_received_cb=self._result_received_cb, + ) + + return task, min_responses + + def get_shareable(self, data): + if isinstance(data, FLModel): + data_shareable: Shareable = FLModelUtils.to_shareable(data) + elif data is None: + data_shareable = Shareable() + else: + dxo = DXO(DataKind.RAW, data=data, meta={}) + data_shareable = dxo.to_shareable() + return data_shareable + + def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): + + self.log_info(fl_ctx, f"{client_task.client.name} task:'{client_task.task.name}' result callback received.\n") + + client_name = client_task.client.name + task_name = client_task.task.name + result = client_task.result + rc = result.get_return_code() + results: Dict[str, any] = {STATUS: rc} + + if rc == ReturnCode.OK: + self.log_info(fl_ctx, f"Received result entries from client:{client_name} for task {task_name}") + fl_model = FLModelUtils.from_shareable(result) + results[RESULT] = {client_name: fl_model} + payload = {CMD: RESULT, PAYLOAD: {task_name: results}} + self.wf_queue.put_result(payload) + elif rc in self.abort_job_in_error.keys(): + self.wf_queue.ask_abort(f"error code {rc} occurred") + self.handle_client_errors(rc, client_task, fl_ctx) + else: + self.log_warning(f"ignore result with return code: {rc}") + + # Cleanup task result + client_task.result = None + + def get_site_names(self): + return [client.name for client in self.clients] diff --git a/nvflare/app_opt/pt/wf_controller.py b/nvflare/app_opt/pt/wf_controller.py new file mode 100644 index 0000000000..36663799fb --- /dev/null +++ b/nvflare/app_opt/pt/wf_controller.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict + +from nvflare.app_common.workflows.wf_controller import WFController +from nvflare.app_opt.pt.decomposers import TensorDecomposer +from nvflare.fuel.utils.fobs import fobs + + +class PTWFController(WFController): + def __init__( + self, + task_name: str, + wf_class_path: str, + wf_args: Dict, + task_timeout: int = 0, + comm_msg_pull_interval: float = 0.2, + ): + super().__init__(task_name, wf_class_path, wf_args, task_timeout, comm_msg_pull_interval) + + fobs.register(TensorDecomposer) From fce9c61b41368777180fa650df5d4e4721c0e38c Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Thu, 28 Dec 2023 19:32:51 -0800 Subject: [PATCH 02/41] update --- examples/hello-world/hello-fedavg/README.md | 42 +++++++------- .../jobs/fedavg/app/custom/fedavg.py | 55 +++++++++---------- 2 files changed, 44 insertions(+), 53 deletions(-) diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md index d6309bcaa9..ee576937f9 100644 --- a/examples/hello-world/hello-fedavg/README.md +++ b/examples/hello-world/hello-fedavg/README.md @@ -44,42 +44,34 @@ With this new API writing the new workflow is really simple: ``` class FedAvg(WF): - def __init__(self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - early_stop_metrics: dict = None, - model_format: str = None - ): + def __init__( + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + model_selection_rule: str = None, + ): super(FedAvg, self).__init__() - self.logger = logging.getLogger(self.__class__.__name__) - - - # (1) init flare_comm - self.flare_comm = WFComm(result_check_interval=10) - self.flare_comm.init(self) + - - def run(self): + # (1) init flare_comm + self.flare_comm = WFCommAPI() + def run(self): self.logger.info("start Fed Avg Workflow\n \n") - net = Net() - model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) - start = self.start_round end = self.start_round + self.num_rounds + model = self.init_model() for current_round in range(start, end): - if self.should_early_stop(model.metrics, self.early_stop_metrics): - break + self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") self.current_round = current_round - self.logger.info(f"Round {current_round}/{self.num_rounds} started.") - sag_results = self.scatter_and_gather(model, current_round) aggr_result = self.aggr_fn(sag_results) @@ -91,6 +83,10 @@ class FedAvg(WF): self.select_best_model(model) self.save_model(self.best_model, self.output_path) + + self.logger.info("end Fed Avg Workflow\n \n") + + ``` Scatter and Gather (SAG): diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index 8f65913c4c..58c24ba07b 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -17,7 +17,6 @@ from typing import Callable, Dict, Optional from net import Net - from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils @@ -41,13 +40,13 @@ class FedAvg(WF): def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - model_selection_rule: str = None, + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + model_selection_rule: str = None, ): super(FedAvg, self).__init__() self.logger = logging.getLogger(self.__class__.__name__) @@ -72,38 +71,34 @@ def __init__( self.flare_comm = WFCommAPI() def run(self): - try: - self.logger.info("start Fed Avg Workflow\n \n") + self.logger.info("start Fed Avg Workflow\n \n") - start = self.start_round - end = self.start_round + self.num_rounds + start = self.start_round + end = self.start_round + self.num_rounds - model = self.init_model() - for current_round in range(start, end): + model = self.init_model() + for current_round in range(start, end): - self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") - self.current_round = current_round + self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") + self.current_round = current_round - if self.should_stop(model.metrics, self.stop_criteria): - self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") - break + if self.should_stop(model.metrics, self.stop_criteria): + self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") + break - sag_results = self.scatter_and_gather(model, current_round) + sag_results = self.scatter_and_gather(model, current_round) - aggr_result = self.aggr_fn(sag_results) + aggr_result = self.aggr_fn(sag_results) - self.logger.info(f"aggregate metrics = {aggr_result.metrics}") + self.logger.info(f"aggregate metrics = {aggr_result.metrics}") - model = update_model(model, aggr_result) + model = update_model(model, aggr_result) - self.select_best_model(model) + self.select_best_model(model) - self.save_model(self.best_model, self.output_path) + self.save_model(self.best_model, self.output_path) - self.logger.info("end Fed Avg Workflow\n \n") - except Exception as e: - print(f"\n\n =============== {traceback.format_exc()} ") - raise e + self.logger.info("end Fed Avg Workflow\n \n") def init_model(self): net = Net() @@ -207,7 +202,7 @@ def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[st return op_fn(value, target) def is_curr_mode_better( - self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable + self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable ) -> bool: curr_metrics = curr_model.metrics if curr_metrics is None: From e47ff4a926fcc817ff86c7725b224bd857f7709b Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Thu, 28 Dec 2023 19:34:04 -0800 Subject: [PATCH 03/41] update --- examples/hello-world/hello-fedavg/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md index ee576937f9..d6877883c2 100644 --- a/examples/hello-world/hello-fedavg/README.md +++ b/examples/hello-world/hello-fedavg/README.md @@ -1,7 +1,6 @@ # FedAvg: simplified -This example illustrates two features: -* How to use the new Flare Communicator API to contract a workflow: no need to write a controller. +This example illustrates How to use the new Workflow Communication API to contract a workflow: no need to write a controller. ## FLARE Workflow Communicator API From 660bdcf4562b88b60166a3f6cc8e420828ebf9c3 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Thu, 28 Dec 2023 19:37:55 -0800 Subject: [PATCH 04/41] update --- examples/hello-world/hello-fedavg/README.md | 19 +++++++++---------- examples/hello-world/hello-km/README.md | 9 +++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md index d6877883c2..8fb07cdf51 100644 --- a/examples/hello-world/hello-fedavg/README.md +++ b/examples/hello-world/hello-fedavg/README.md @@ -123,8 +123,8 @@ This is the same as FLARE Client API configuration ### server-side configuration - Server side controller is really simple, all we need is to user WFController with newly defined workflow class -```KM``` + Server side controller is really simple, all we need is to use WFController with newly defined workflow class + ``` { @@ -136,19 +136,17 @@ This is the same as FLARE Client API configuration workflows = [ { id = "fed_avg" - path = "nvflare.app_common.workflows.wf_controller.WFController" + path = "nvflare.app_opt.pt.wf_controller.PTWFController" args { + comm_msg_pull_interval = 5 task_name = "train" - wf_class_path = "fedavg_wf.FedAvg", + wf_class_path = "fedavg_pt.PTFedAvg", wf_args { min_clients = 2 num_rounds = 10 output_path = "/tmp/nvflare/fedavg/mode.pth" - model_format = "torch" - early_stop_metrics { - accuracy = 55 - } - + stop_cond = "accuracy >= 55" + model_selection_rule = "accuracy >=" } } } @@ -166,5 +164,6 @@ This is the same as FLARE Client API configuration assume current working directory is at ```hello-fedavg``` directory ``` -nvflare simulator job -w /tmp/nvflare/km/job -n 2 -t 2 +nvflare simulator -n 2 -t 2 jobs/fedavg -w /tmp/fedavg + ``` diff --git a/examples/hello-world/hello-km/README.md b/examples/hello-world/hello-km/README.md index b2c5b94415..79956e2a4b 100644 --- a/examples/hello-world/hello-km/README.md +++ b/examples/hello-world/hello-km/README.md @@ -109,8 +109,8 @@ This is the same as FLARE Client API configuration ### server-side configuration - Server side controller is really simple, all we need is to user WFController with newly defined workflow class -```KM``` + Server side controller is really simple, all we need is to use WFController with newly defined workflow class + ``` { @@ -125,7 +125,7 @@ This is the same as FLARE Client API configuration path = "nvflare.app_common.workflows.wf_controller.WFController" args { task_name = "train" - wf_class_path = "km_wf.KM", + wf_class_path = "kaplan_meier.KM", wf_args { min_clients = 2 output_path = "/tmp/nvflare/km/km.json" @@ -138,6 +138,7 @@ This is the same as FLARE Client API configuration } + ``` @@ -146,7 +147,7 @@ This is the same as FLARE Client API configuration assume current working directory is at ```hello-km``` directory ``` -nvflare simulator job -w /tmp/nvflare/km/job -n 2 -t 2 +nvflare simulator -n 2 -t 2 jobs/kaplan-meier -w /tmp/km ``` From 8733f60de1bf67106cb17e65c74aa53e25f5fb57 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Thu, 28 Dec 2023 21:47:38 -0800 Subject: [PATCH 05/41] Add Fed Cyclic example --- .../hello-world/hello-cyclic-pt/README.md | 168 ++++++++++++++++++ .../cyclic/app/config/config_fed_client.conf | 90 ++++++++++ .../cyclic/app/config/config_fed_server.conf | 25 +++ .../jobs/cyclic/app/config_fed_server.conf | 39 ++++ .../jobs/cyclic/app/custom/cifar10.py | 130 ++++++++++++++ .../jobs/cyclic/app/custom/fed_cyclic.py | 152 ++++++++++++++++ .../jobs/cyclic/app/custom/net.py | 37 ++++ .../hello-cyclic-pt/jobs/cyclic/meta.conf | 7 + .../hello-cyclic-pt/requirements.txt | 0 .../jobs/fedavg/app/custom/fedavg.py | 17 +- nvflare/app_common/hub/hub_controller.py | 2 +- .../workflows/wf_comm/wf_comm_api.py | 19 +- .../workflows/wf_comm/wf_comm_api_spec.py | 12 +- nvflare/app_common/workflows/wf_controller.py | 33 +++- 14 files changed, 713 insertions(+), 18 deletions(-) create mode 100644 examples/hello-world/hello-cyclic-pt/README.md create mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf create mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf create mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf create mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py create mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py create mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py create mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf create mode 100644 examples/hello-world/hello-cyclic-pt/requirements.txt diff --git a/examples/hello-world/hello-cyclic-pt/README.md b/examples/hello-world/hello-cyclic-pt/README.md new file mode 100644 index 0000000000..d13fdaae37 --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/README.md @@ -0,0 +1,168 @@ +# Fed Cyclic Weight Transfer: simplified + +This example illustrates How to use the new Workflow Communication API to contract a workflow: no need to write a controller. + +## FLARE Workflow Communicator API + +The Flare workflow Communicator API only has small set methods + +``` +class WFCommAPISpec(ABC): + @abstractmethod + def broadcast_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def broadcast(self, msg_payload): + pass + + @abstractmethod + def send(self, msg_payload: Dict): + pass + + @abstractmethod + def send_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def get_site_names(self): + pass + + @abstractmethod + def wait(self, min_responses): + pass +``` + + +## Writing a new Workflow + +With this new API writing the new workflow is really simple: + +* Workflow (Server) + +``` + +class FedCyclic(WF): + def __init__( + self, + output_path: str, + num_rounds: int = 5, + start_round: int = 0, + task_name="train", + order: str = RelayOrder.FIXED, + ): + super(FedCyclic, self).__init__() + + + + # (1) instantiate flare_comm + self.flare_comm = WFCommAPI() + + def run(self): + + self.last_model = self.init_model() + + # note: this one must be within run() method, not in the __init__() method + # as some values are injected at runtime during run() + self.part_sites = self.flare_comm.get_site_names() + + if len(self.part_sites) <= 1: + raise ValueError(f"Not enough client sites. sites={self.part_sites}") + + start = self.start_round + end = self.start_round + self.num_rounds + for current_round in range(start, end): + targets = self.get_relay_orders() + relay_result = self.relay_and_wait(self.last_model, targets, current_round) + + self.logger.info(f"target sites ={targets}.") + + task_name, task_result = next(iter(relay_result.items())) + self.last_site, self.last_model = next(iter(task_result.items())) + + self.logger.info(f"ending current round={current_round}.") + gc.collect() + + self.save_model(self.last_model, self.output_path) + self.logger.info("Cyclic ended.") + +``` + +Relay_and_wait + +``` + + def relay_and_wait(self, last_model: FLModel, targets: List[str], current_round): + msg_payload = { + MIN_RESPONSES: 1, + CURRENT_ROUND: current_round, + NUM_ROUNDS: self.num_rounds, + START_ROUND: self.start_round, + DATA: last_model, + TARGET_SITES: targets, + } + # (2) relay_and_wait and wait + results = self.flare_comm.relay_and_wait(msg_payload) + return results +``` + +The base class ```WF``` is define as + +``` +class WF(ABC): + + @abstractmethod + def run(self): + raise NotImplemented +``` +is mainly make sure user define ```run()``` method + +## Configurations + +### client-side configuration + +This is the same as FLARE Client API configuration + +### server-side configuration + + Server side controller is really simple, all we need is to use WFController with newly defined workflow class + + +``` + { + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "fed_avg" + path = "nvflare.app_opt.pt.wf_controller.PTWFController" + args { + comm_msg_pull_interval = 5 + task_name = "train" + wf_class_path = "fed_cyclic.FedCyclic", + wf_args { + num_rounds = 10 + output_path = "/tmp/nvflare/fedavg/mode.pth" + } + } + } + ] + + components = [] + +} + +``` + + +## Run the job + +assume current working directory is at ```hello-cyclic-pt``` directory + +``` + nvflare simulator jobs/cyclic -w /tmp/cyclic -n 3 -t 3 + +``` diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf new file mode 100644 index 0000000000..814025b8ad --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf @@ -0,0 +1,90 @@ +{ + # version of the configuration + format_version = 2 + + # This is the application script which will be invoked. Client can replace this script with user's own training script. + app_script = "cifar10.py" + + # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. + app_config = "" + + # Client Computing Executors. + executors = [ + { + # tasks the executors are defined to handle + tasks = ["train"] + + # This particular executor + executor { + + # This is an executor for pytorch + Client API. The underline data exchange is using Pipe. + path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" + + args { + # launcher_id is used to locate the Launcher object in "components" + launcher_id = "launcher" + + # pipe_id is used to locate the Pipe object in "components" + pipe_id = "pipe" + + # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. + # Please refer to the class docstring for all available arguments + heartbeat_timeout = 60 + + # format of the exchange parameters + params_exchange_format = "pytorch" + + # if the transfer_type is FULL, then it will be sent directly + # if the transfer_type is DIFF, then we will calculate the + # difference VS received parameters and send the difference + params_transfer_type = "DIFF" + + # if train_with_evaluation is true, the executor will expect + # the custom code need to send back both the trained parameters and the evaluation metric + # otherwise only trained parameters are expected + train_with_evaluation = true + } + } + } + ], + + # this defined an array of task data filters. If provided, it will control the data from server controller to client executor + task_data_filters = [] + + # this defined an array of task result filters. If provided, it will control the result from client executor to server controller + task_result_filters = [] + + components = [ + { + # component id is "launcher" + id = "launcher" + + # the class path of this component + path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" + + args { + # the launcher will invoke the script + script = "python3 custom/{app_script} {app_config} " + # if launch_once is true, the SubprocessLauncher will launch once for the whole job + # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server + launch_once = true + } + } + { + id = "pipe" + + path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" + + args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. + mode = "PASSIVE" + + # root_path: is the directory location of the parameters exchange. + # You can also set it to an absolute path in your system. + root_path = "{WORKSPACE}/{JOB_ID}/{SITE_NAME}" + } + } + ] +} diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf new file mode 100644 index 0000000000..357c596408 --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf @@ -0,0 +1,25 @@ +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "fed_avg" + path = "nvflare.app_opt.pt.wf_controller.PTWFController" + args { + comm_msg_pull_interval = 5 + task_name = "train" + wf_class_path = "fed_cyclic.FedCyclic", + wf_args { + num_rounds = 10 + output_path = "/tmp/nvflare/fedavg/mode.pth" + } + } + } + ] + + components = [] + +} diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf new file mode 100644 index 0000000000..b822a67134 --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf @@ -0,0 +1,39 @@ +{ + format_version = 2 + server { + heart_beat_timeout = 600 + } + task_data_filters =[] + task_result_filters = [] + # path to defined PyTorch network + # this assumes that there will be a "net.py" file with class name "Net", please modify accordingly + model_class_path = "net.Net" + components = [ + { + id = "persistor" + path = "nvflare.app_opt.pt.file_model_persistor.PTFileModelPersistor" + args.model.path = "{model_class_path}" + } + { + id = "shareable_generator" + path = "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator" + args = {} + } + ] + workflows = [ + { + # server-side Cyclic Controller for Cyclic Weight Transfer + id = "cyclic_ctl" + path = "nvflare.app_common.workflows.cyclic_ctl.CyclicController" + args { + # see the doc strings for all available arguments + num_rounds = 3 + task_assignment_timeout = 8 + persistor_id = "persistor" + shareable_generator_id = "shareable_generator" + # task name: client side needs to have an executor that handles this task + task_name = "train" + } + } + ] +} diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py new file mode 100644 index 0000000000..33c9272a7a --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py @@ -0,0 +1,130 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.optim as optim +import torchvision +import torchvision.transforms as transforms +from net import Net + +# (1) import nvflare client API +import nvflare.client as flare + +# (optional) set a fix place so we don't need to download everytime +DATASET_PATH = "/tmp/nvflare/data" +# (optional) We change to use GPU to speed things up. +# if you want to use CPU, change DEVICE="cpu" +DEVICE = "cuda:0" + + +def main(): + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + + batch_size = 4 + epochs = 2 + + trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) + trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) + + testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) + testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) + + net = Net() + + # (2) initializes NVFlare client API + flare.init() + + while flare.is_running(): + # (3) receives FLModel from NVFlare + input_model = flare.receive() + print(f"current_round={input_model.current_round}") + + # (4) loads model from NVFlare + net.load_state_dict(input_model.params) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) + + # (optional) use GPU to speed things up + net.to(DEVICE) + # (optional) calculate total steps + steps = epochs * len(trainloader) + for epoch in range(epochs): # loop over the dataset multiple times + + running_loss = 0.0 + for i, data in enumerate(trainloader, 0): + # get the inputs; data is a list of [inputs, labels] + # (optional) use GPU to speed things up + inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) + + # zero the parameter gradients + optimizer.zero_grad() + + # forward + backward + optimize + outputs = net(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # print statistics + running_loss += loss.item() + if i % 2000 == 1999: # print every 2000 mini-batches + print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") + running_loss = 0.0 + + print("Finished Training") + + PATH = "./cifar_net.pth" + torch.save(net.state_dict(), PATH) + + # (5) wraps evaluation logic into a method to re-use for + # evaluation on both trained and received model + def evaluate(input_weights): + net = Net() + net.load_state_dict(input_weights) + # (optional) use GPU to speed things up + net.to(DEVICE) + + correct = 0 + total = 0 + # since we're not training, we don't need to calculate the gradients for our outputs + with torch.no_grad(): + for data in testloader: + # (optional) use GPU to speed things up + images, labels = data[0].to(DEVICE), data[1].to(DEVICE) + # calculate outputs by running images through the network + outputs = net(images) + # the class with the highest energy is what we choose as prediction + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") + return 100 * correct // total + + # (6) evaluate on received model for model selection + accuracy = evaluate(input_model.params) + # (7) construct trained FL model + output_model = flare.FLModel( + params=net.cpu().state_dict(), + metrics={"accuracy": accuracy}, + meta={"NUM_STEPS_CURRENT_ROUND": steps}, + ) + # (8) send model back to NVFlare + flare.send(output_model) + + +if __name__ == "__main__": + main() diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py new file mode 100644 index 0000000000..3f0bc20154 --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -0,0 +1,152 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import gc +import logging +import random +from typing import Dict, List, Optional + +import torch +from net import Net + +from nvflare.app_common.abstract.fl_model import FLModel, ParamsType +from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( + CURRENT_ROUND, + DATA, + MIN_RESPONSES, + NUM_ROUNDS, + START_ROUND, + TARGET_SITES, +) +from nvflare.app_common.workflows.wf_comm.wf_spec import WF + +update_model = FLModelUtils.update_model + + +# Fed Cyclic Weight Transfer Workflow + + +class RelayOrder: + FIXED = "FIXED" + RANDOM = "RANDOM" + RANDOM_WITHOUT_SAME_IN_A_ROW = "RANDOM_WITHOUT_SAME_IN_A_ROW" + + +SUPPORTED_ORDERS = (RelayOrder.FIXED, RelayOrder.RANDOM, RelayOrder.RANDOM_WITHOUT_SAME_IN_A_ROW) + + +class FedCyclic(WF): + def __init__( + self, + output_path: str, + num_rounds: int = 5, + start_round: int = 0, + task_name="train", + order: str = RelayOrder.FIXED, + ): + super(FedCyclic, self).__init__() + self.logger = logging.getLogger(self.__class__.__name__) + + self.output_path = output_path + self.num_rounds = num_rounds + self.start_round = start_round + self.task_name = task_name + self.order = order + self.last_site: Optional[str] = None + self.last_model: Optional[FLModel] = None + self.part_sites = None + + self.check_inputs() + + # (1) instantiate flare_comm + self.flare_comm = WFCommAPI() + + def init_model(self): + net = Net() + model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) + return model + + def run(self): + + self.last_model = self.init_model() + + # note: this one must be within run() method, not in the __init__() method + # as some values are injected at runtime during run() + self.part_sites = self.flare_comm.get_site_names() + + if len(self.part_sites) <= 1: + raise ValueError(f"Not enough client sites. sites={self.part_sites}") + + start = self.start_round + end = self.start_round + self.num_rounds + for current_round in range(start, end): + targets = self.get_relay_orders() + relay_result = self.relay_and_wait(self.last_model, targets, current_round) + + self.logger.info(f"target sites ={targets}.") + + task_name, task_result = next(iter(relay_result.items())) + self.last_site, self.last_model = next(iter(task_result.items())) + + self.logger.info(f"ending current round={current_round}.") + gc.collect() + + self.save_model(self.last_model, self.output_path) + self.logger.info("Cyclic ended.") + + def relay_and_wait(self, last_model: FLModel, targets: List[str], current_round): + msg_payload = { + MIN_RESPONSES: 1, + CURRENT_ROUND: current_round, + NUM_ROUNDS: self.num_rounds, + START_ROUND: self.start_round, + DATA: last_model, + TARGET_SITES: targets, + } + # (2) relay_and_wait and wait + results = self.flare_comm.relay_and_wait(msg_payload) + return results + + def check_inputs(self): + if not isinstance(self.num_rounds, int): + raise TypeError("num_rounds must be int but got {}".format(type(self.num_rounds))) + if not isinstance(self.task_name, str): + raise TypeError("task_name must be a string but got {}".format(type(self.task_name))) + if self.order not in SUPPORTED_ORDERS: + raise ValueError(f"order must be in {SUPPORTED_ORDERS}") + + def get_relay_orders(self): + targets = list(self.part_sites) + if len(targets) <= 1: + raise ValueError("Not enough client sites.") + + if self.order == RelayOrder.RANDOM: + random.shuffle(targets) + elif self.order == RelayOrder.RANDOM_WITHOUT_SAME_IN_A_ROW: + random.shuffle(targets) + if self.last_site == targets[0]: + targets = targets.append(targets.pop(0)) + self.last_site = targets[-1] + return targets + + def save_model(self, model: FLModel, file_path: str): + if not file_path: + raise ValueError("invalid file path") + + dir_name = os.path.dirname(file_path) + os.makedirs(dir_name, exist_ok=True) + + self.logger.info(f"save best model to {file_path} \n") + torch.save(model.params, file_path) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py new file mode 100644 index 0000000000..031f84f432 --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Net(nn.Module): + def __init__(self): + super().__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = torch.flatten(x, 1) # flatten all dimensions except batch + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf new file mode 100644 index 0000000000..ed1e0f364b --- /dev/null +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf @@ -0,0 +1,7 @@ +{ + name = "cyclic_pt" + deploy_map { + app = ["@ALL"] + } + min_clients = 2 +} diff --git a/examples/hello-world/hello-cyclic-pt/requirements.txt b/examples/hello-world/hello-cyclic-pt/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index 58c24ba07b..2136164117 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -17,6 +17,7 @@ from typing import Callable, Dict, Optional from net import Net + from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils @@ -40,13 +41,13 @@ class FedAvg(WF): def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - model_selection_rule: str = None, + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + model_selection_rule: str = None, ): super(FedAvg, self).__init__() self.logger = logging.getLogger(self.__class__.__name__) @@ -202,7 +203,7 @@ def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[st return op_fn(value, target) def is_curr_mode_better( - self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable + self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable ) -> bool: curr_metrics = curr_model.metrics if curr_metrics is None: diff --git a/nvflare/app_common/hub/hub_controller.py b/nvflare/app_common/hub/hub_controller.py index d7699ed2b0..e9d0392072 100644 --- a/nvflare/app_common/hub/hub_controller.py +++ b/nvflare/app_common/hub/hub_controller.py @@ -134,7 +134,7 @@ def process_result_of_unknown_task( class RelayOperator(OperatorSpec, FLComponent): - _PROP_LAST_RESULT = "last_result" + _PROP_LAST_RESULT = "last_model" _PROP_SHAREABLE_GEN = "shareable_generator" def __init__(self): diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py index 065624ac8b..9ac3fc1767 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py @@ -22,6 +22,7 @@ CMD, CMD_ABORT, CMD_BROADCAST, + CMD_RELAY, CMD_SEND, CMD_STOP, MIN_RESPONSES, @@ -74,6 +75,7 @@ def send_and_wait(self, msg_payload: Dict): return self.wait(min_responses) def get_site_names(self): + print(f"{self.meta =}") return self.meta.get(SITE_NAMES) def wait(self, min_responses): @@ -89,6 +91,19 @@ def wait(self, min_responses): # self.logger.info(f"no result available, sleep {self.result_pull_interval} sec") time.sleep(self.result_pull_interval) + def relay_and_wait(self, msg_payload: Dict): + self.relay(msg_payload) + min_responses = msg_payload.get(MIN_RESPONSES, 1) + return self.wait(min_responses) + + def relay(self, msg_payload: Dict): + self._check_wf_queue() + message = { + CMD: CMD_RELAY, + PAYLOAD: msg_payload, + } + self.wf_queue.put_ctrl_msg(message) + def _get_results(self) -> dict: items_size = self.wf_queue.result_size() batch_result: Dict = {} @@ -111,7 +126,7 @@ def _get_results(self) -> dict: one_site_result = item.get(PAYLOAD) for task, site_result in one_site_result.items(): task_result = batch_result.get(task, {}) - self.check_result(site_result) + self._check_result(site_result) rc = site_result.get(STATUS) if rc == ReturnCode.OK: result = site_result.get(RESULT, {}) @@ -126,7 +141,7 @@ def _get_results(self) -> dict: return batch_result - def check_result(self, site_result): + def _check_result(self, site_result): if site_result is None: raise RuntimeError("expecting site_result to be dictionary, but get None") diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py index b11130c9a2..5455da34eb 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py @@ -20,6 +20,7 @@ CMD_STOP = "STOP" CMD_ABORT = "ABORT" CMD_BROADCAST = "BROADCAST" +CMD_RELAY = "RELAY" PAYLOAD = "PAYLOAD" SITE_NAMES = "SITE_NAMES" @@ -34,6 +35,7 @@ STATUS = "status" RESULT = "result" DATA = "data" +TARGET_SITES = "target_sizes" class WFCommAPISpec(ABC): @@ -41,6 +43,14 @@ class WFCommAPISpec(ABC): def broadcast_and_wait(self, msg_payload: Dict): pass + @abstractmethod + def send_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def relay_and_wait(self, msg_payload: Dict): + pass + @abstractmethod def broadcast(self, msg_payload): pass @@ -50,7 +60,7 @@ def send(self, msg_payload: Dict): pass @abstractmethod - def send_and_wait(self, msg_payload: Dict): + def relay(self, msg_payload: Dict): pass @abstractmethod diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index 731ddea760..6bcf5d90bd 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -15,7 +15,7 @@ import time from concurrent.futures import ThreadPoolExecutor from queue import Queue -from typing import Dict, Tuple +from typing import Dict, List, Tuple from nvflare.apis.client import Client from nvflare.apis.controller_spec import ClientTask, OperatorMethod, Task, TaskOperatorKey @@ -34,6 +34,8 @@ CMD, CMD_ABORT, CMD_BROADCAST, + CMD_RELAY, + CMD_SEND, CMD_STOP, DATA, MIN_RESPONSES, @@ -41,6 +43,7 @@ RESULT, SITE_NAMES, STATUS, + TARGET_SITES, ) from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue from nvflare.app_common.workflows.wf_comm.wf_spec import WF @@ -88,6 +91,8 @@ def setup_wf_queue(self): wf_comm_api.set_result_pull_interval(self.comm_msg_pull_interval) wf_comm_api.meta.update({SITE_NAMES: self.get_site_names()}) + print(f"\n \n {wf_comm_api.meta=}") + def find_wf_comm_in_wf(self): attr_objs = [getattr(self.wf, attr_name, None) for attr_name in dir(self.wf)] wf_comm_attrs = [attr for attr in attr_objs if isinstance(attr, WFCommAPI)] @@ -166,10 +171,11 @@ def ctrl_msg_loop(self, fl_ctx: FLContext, abort_signal: Signal): pay_load = item.get(PAYLOAD) current_round = self.prepare_round_info(fl_ctx, pay_load) - task, min_responses = self.get_payload_task(pay_load) + task, min_responses, targets = self.get_payload_task(pay_load) self.broadcast_and_wait( task=task, + targets=targets, min_responses=min_responses, wait_time_after_min_received=0, fl_ctx=fl_ctx, @@ -178,7 +184,21 @@ def ctrl_msg_loop(self, fl_ctx: FLContext, abort_signal: Signal): self.fire_event(AppEventType.ROUND_DONE, fl_ctx) self.log_info(fl_ctx, f"Round {current_round} finished.") - elif cmd == "SEND": + elif cmd == CMD_RELAY: + pay_load = item.get(PAYLOAD) + current_round = self.prepare_round_info(fl_ctx, pay_load) + task, min_responses, targets = self.get_payload_task(pay_load) + + self.relay_and_wait( + task=task, + targets=targets, + fl_ctx=fl_ctx, + abort_signal=abort_signal, + ) + self.fire_event(AppEventType.ROUND_DONE, fl_ctx) + self.log_info(fl_ctx, f"Round {current_round} finished.") + + elif cmd == CMD_SEND: raise NotImplementedError else: abort_signal.trigger(f"Unknown command '{cmd}'") @@ -205,11 +225,12 @@ def prepare_round_info(self, fl_ctx, pay_load): self.fire_event(AppEventType.ROUND_STARTED, fl_ctx) return current_round - def get_payload_task(self, pay_load) -> Tuple[Task, int]: + def get_payload_task(self, pay_load) -> Tuple[Task, int, List[str]]: min_responses = pay_load.get(MIN_RESPONSES) current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) start_round = pay_load.get(AppConstants.START_ROUND, 0) num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) + targets = pay_load.get(TARGET_SITES, self.get_site_names()) data = pay_load.get(DATA, {}) data_shareable = self.get_shareable(data) @@ -234,7 +255,7 @@ def get_payload_task(self, pay_load) -> Tuple[Task, int]: result_received_cb=self._result_received_cb, ) - return task, min_responses + return task, min_responses, targets def get_shareable(self, data): if isinstance(data, FLModel): @@ -266,7 +287,7 @@ def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): self.wf_queue.ask_abort(f"error code {rc} occurred") self.handle_client_errors(rc, client_task, fl_ctx) else: - self.log_warning(f"ignore result with return code: {rc}") + self.log_warning(fl_ctx, f"ignore result with return code: {rc}") # Cleanup task result client_task.result = None From f1ad53d6a96c6648d06d5c107d440d14514eeb54 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Thu, 28 Dec 2023 21:48:41 -0800 Subject: [PATCH 06/41] Add Fed Cyclic example --- .../jobs/cyclic/app/config_fed_server.conf | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf deleted file mode 100644 index b822a67134..0000000000 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config_fed_server.conf +++ /dev/null @@ -1,39 +0,0 @@ -{ - format_version = 2 - server { - heart_beat_timeout = 600 - } - task_data_filters =[] - task_result_filters = [] - # path to defined PyTorch network - # this assumes that there will be a "net.py" file with class name "Net", please modify accordingly - model_class_path = "net.Net" - components = [ - { - id = "persistor" - path = "nvflare.app_opt.pt.file_model_persistor.PTFileModelPersistor" - args.model.path = "{model_class_path}" - } - { - id = "shareable_generator" - path = "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator" - args = {} - } - ] - workflows = [ - { - # server-side Cyclic Controller for Cyclic Weight Transfer - id = "cyclic_ctl" - path = "nvflare.app_common.workflows.cyclic_ctl.CyclicController" - args { - # see the doc strings for all available arguments - num_rounds = 3 - task_assignment_timeout = 8 - persistor_id = "persistor" - shareable_generator_id = "shareable_generator" - # task name: client side needs to have an executor that handles this task - task_name = "train" - } - } - ] -} From b1fa51be1e32e88ba042c729128931cd527e774c Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 29 Dec 2023 09:34:04 -0800 Subject: [PATCH 07/41] addres PR comments --- .../jobs/cyclic/app/config/config_fed_server.conf | 2 +- .../jobs/cyclic/app/custom/fed_cyclic.py | 15 ++++++++------- .../app_common/workflows/wf_comm/wf_comm_api.py | 1 - nvflare/app_common/workflows/wf_comm/wf_queue.py | 8 ++++++-- nvflare/app_common/workflows/wf_controller.py | 2 -- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf index 357c596408..96d938cc57 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf @@ -13,7 +13,7 @@ task_name = "train" wf_class_path = "fed_cyclic.FedCyclic", wf_args { - num_rounds = 10 + num_rounds = 2 output_path = "/tmp/nvflare/fedavg/mode.pth" } } diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index 3f0bc20154..ed3896ab61 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -13,8 +13,9 @@ # limitations under the License. import gc import logging +import os import random -from typing import Dict, List, Optional +from typing import List, Optional import torch from net import Net @@ -73,11 +74,6 @@ def __init__( # (1) instantiate flare_comm self.flare_comm = WFCommAPI() - def init_model(self): - net = Net() - model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) - return model - def run(self): self.last_model = self.init_model() @@ -104,7 +100,7 @@ def run(self): gc.collect() self.save_model(self.last_model, self.output_path) - self.logger.info("Cyclic ended.") + self.logger.info("\n fed cyclic ended \n") def relay_and_wait(self, last_model: FLModel, targets: List[str], current_round): msg_payload = { @@ -119,6 +115,11 @@ def relay_and_wait(self, last_model: FLModel, targets: List[str], current_round) results = self.flare_comm.relay_and_wait(msg_payload) return results + def init_model(self): + net = Net() + model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) + return model + def check_inputs(self): if not isinstance(self.num_rounds, int): raise TypeError("num_rounds must be int but got {}".format(type(self.num_rounds))) diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py index 9ac3fc1767..e718c13539 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py @@ -75,7 +75,6 @@ def send_and_wait(self, msg_payload: Dict): return self.wait(min_responses) def get_site_names(self): - print(f"{self.meta =}") return self.meta.get(SITE_NAMES) def wait(self, min_responses): diff --git a/nvflare/app_common/workflows/wf_comm/wf_queue.py b/nvflare/app_common/workflows/wf_comm/wf_queue.py index 6287f2f907..ac64365136 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_queue.py +++ b/nvflare/app_common/workflows/wf_comm/wf_queue.py @@ -43,10 +43,14 @@ def result_size(self) -> int: return self.result_queue.qsize() def get_ctrl_msg(self): - return self.ctrl_queue.get() + item = self.ctrl_queue.get() + self.ctrl_queue.task_done() + return item def get_result(self): - return self.result_queue.get() + item = self.result_queue.get() + self.result_queue.task_done() + return item def stop(self, msg: Optional[str] = None): msg = msg if msg else {} diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index 6bcf5d90bd..41eb428b04 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -91,8 +91,6 @@ def setup_wf_queue(self): wf_comm_api.set_result_pull_interval(self.comm_msg_pull_interval) wf_comm_api.meta.update({SITE_NAMES: self.get_site_names()}) - print(f"\n \n {wf_comm_api.meta=}") - def find_wf_comm_in_wf(self): attr_objs = [getattr(self.wf, attr_name, None) for attr_name in dir(self.wf)] wf_comm_attrs = [attr for attr in attr_objs if isinstance(attr, WFCommAPI)] From d9ad8b2533403b47170aa98dcfc5a7661752df4e Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 16:24:46 -0800 Subject: [PATCH 08/41] 1. Remove Base Class ErrorHandleController, instead move the function into some utils class 2. Refactoring the WFController class into BaseWFController class and WFController class. The BaseWFController has majority of the logics and implement ControllerSpec. But not implement Responder, which is needed for Server-side controller. WFController Inherited both BaseWFController and Controller. This allows BaseWFController be able to be used on the client-side controller logics without implementing control_flow() method --- .../app_common/common_workflows/__init__.py | 0 .../common_workflows/base_wf_controller.py | 309 ++++++++++++++++++ .../workflows/error_handle_utils.py | 10 + nvflare/app_common/workflows/wf_controller.py | 282 +--------------- 4 files changed, 331 insertions(+), 270 deletions(-) create mode 100644 nvflare/app_common/common_workflows/__init__.py create mode 100644 nvflare/app_common/common_workflows/base_wf_controller.py create mode 100644 nvflare/app_common/workflows/error_handle_utils.py diff --git a/nvflare/app_common/common_workflows/__init__.py b/nvflare/app_common/common_workflows/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py new file mode 100644 index 0000000000..372559597d --- /dev/null +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -0,0 +1,309 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from abc import ABC +from concurrent.futures import ThreadPoolExecutor +from queue import Queue +from typing import Dict, List, Tuple + +from nvflare.apis.client import Client +from nvflare.apis.controller_spec import ClientTask, OperatorMethod, Task, TaskOperatorKey, ControllerSpec +from nvflare.apis.dxo import DXO, DataKind +from nvflare.apis.fl_component import FLComponent +from nvflare.apis.fl_constant import ReturnCode +from nvflare.apis.fl_context import FLContext +from nvflare.apis.shareable import Shareable +from nvflare.apis.signal import Signal +from nvflare.app_common.abstract.fl_model import FLModel +from nvflare.app_common.app_constant import AppConstants +from nvflare.app_common.app_event_type import AppEventType +from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.workflows.error_handle_utils import ABORT_WHEN_IN_ERROR +from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( + CMD, + CMD_ABORT, + CMD_BROADCAST, + CMD_RELAY, + CMD_SEND, + CMD_STOP, + DATA, + MIN_RESPONSES, + PAYLOAD, + RESULT, + SITE_NAMES, + STATUS, + TARGET_SITES, +) +from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue +from nvflare.app_common.workflows.wf_comm.wf_spec import WF +from nvflare.fuel.utils import class_utils +from nvflare.security.logging import secure_format_traceback + + +class BaseWFController(FLComponent, ControllerSpec, ABC): + def __init__( + self, + task_name: str, + wf_class_path: str, + wf_args: Dict, + task_timeout: int = 0, + comm_msg_pull_interval: float = 0.2, + ): + super().__init__() + + self.clients = None + self.task_timeout = task_timeout + self.task_name = task_name + self.comm_msg_pull_interval = comm_msg_pull_interval + self.wf_class_path = wf_class_path + self.wf_args = wf_args + self.wf_queue: WFQueue = WFQueue(ctrl_queue=Queue(), result_queue=Queue()) + self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) + self._thread_pool_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=self.__class__.__name__) + + self.engine = None + self.fl_ctx = None + + def start_controller(self, fl_ctx: FLContext): + self.fl_ctx = fl_ctx + self.log_info(fl_ctx, "Initializing controller workflow.") + self.engine = self.fl_ctx.get_engine() + self.clients = self.engine.get_clients() + + self.setup_wf_queue() + + self.log_info(fl_ctx, "workflow controller started") + + def setup_wf_queue(self): + wf_comm_api = self.find_wf_comm_in_wf() + wf_comm_api.set_queue(self.wf_queue) + wf_comm_api.set_result_pull_interval(self.comm_msg_pull_interval) + wf_comm_api.meta.update({SITE_NAMES: self.get_site_names()}) + + def find_wf_comm_in_wf(self): + attr_objs = [getattr(self.wf, attr_name, None) for attr_name in dir(self.wf)] + wf_comm_attrs = [attr for attr in attr_objs if isinstance(attr, WFCommAPI)] + if wf_comm_attrs: + return wf_comm_attrs[0] + else: + raise RuntimeError(f"missing required attribute with type of 'WFCommAPI' in {self.wf.__class__.__name__}") + + def start_workflow(self, abort_signal, fl_ctx): + try: + future = self._thread_pool_executor.submit(self.ctrl_msg_loop, fl_ctx=fl_ctx, abort_signal=abort_signal) + self.wf.run() + self.stop_msg_queue("job completed", fl_ctx) + future.result() + except Exception as e: + error_msg = secure_format_traceback() + self.log_error(fl_ctx, error_msg) + self.system_panic(error_msg, fl_ctx=fl_ctx) + finally: + wait_time = self.comm_msg_pull_interval + 0.05 + self.stop_msg_queue("job finished", fl_ctx, wait_time) + + def stop_msg_queue(self, stop_message, fl_ctx, wait_time: float = 0): + self.wf_queue.stop(stop_message) + self.log_info(fl_ctx, stop_message) + + if wait_time > 0: + self.log_info(fl_ctx, f"wait for {wait_time} sec") + time.sleep(wait_time) + + def stop_controller(self, fl_ctx: FLContext): + self.stop_msg_queue("job completed", fl_ctx) + if self._thread_pool_executor: + self._thread_pool_executor.shutdown() + + def process_result_of_unknown_task( + self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext + ): + pass + + def ctrl_msg_loop(self, fl_ctx: FLContext, abort_signal: Signal): + + if self.wf_queue is None: + raise ValueError("WFQueue must provided") + + try: + while True: + if abort_signal.triggered: + break + if not self.wf_queue.has_ctrl_msg(): + time.sleep(self.comm_msg_pull_interval) + else: + item = self.wf_queue.get_ctrl_msg() + if item is None: + self.log_warning(fl_ctx, "Ignore 'None' ctrl comm message") + continue + + cmd = item.get(CMD, None) + + if cmd is None: + msg = f"get None command, expecting {CMD} key'" + self.log_error(fl_ctx, msg) + raise ValueError(msg) + + elif cmd == CMD_STOP: + msg = item.get(PAYLOAD) + self.log_info(fl_ctx, f"receive {CMD_STOP} command, {msg}") + break + + elif cmd == CMD_ABORT: + msg = item.get(PAYLOAD) + self.log_info(fl_ctx, f"receive {CMD_ABORT} command, {msg}") + raise RuntimeError(msg) + + elif cmd == CMD_BROADCAST: + pay_load = item.get(PAYLOAD) + + current_round = self.prepare_round_info(fl_ctx, pay_load) + task, min_responses, targets = self.get_payload_task(pay_load) + + self.broadcast_and_wait( + task=task, + targets=targets, + min_responses=min_responses, + wait_time_after_min_received=0, + fl_ctx=fl_ctx, + abort_signal=abort_signal, + ) + self.fire_event(AppEventType.ROUND_DONE, fl_ctx) + self.log_info(fl_ctx, f"Round {current_round} finished.") + + elif cmd == CMD_RELAY: + pay_load = item.get(PAYLOAD) + current_round = self.prepare_round_info(fl_ctx, pay_load) + task, min_responses, targets = self.get_payload_task(pay_load) + + self.relay_and_wait( + task=task, + targets=targets, + fl_ctx=fl_ctx, + abort_signal=abort_signal, + ) + self.fire_event(AppEventType.ROUND_DONE, fl_ctx) + self.log_info(fl_ctx, f"Round {current_round} finished.") + + elif cmd == CMD_SEND: + raise NotImplementedError + else: + abort_signal.trigger(f"Unknown command '{cmd}'") + raise ValueError(f"Unknown command '{cmd}'") + + if abort_signal.triggered: + self.log_debug(self.fl_ctx, f"task {self.task_name} aborted") + break + except Exception as e: + error_msg = secure_format_traceback() + self.wf_queue.ask_abort(error_msg) + self.log_error(fl_ctx, error_msg) + self.system_panic(error_msg, fl_ctx=fl_ctx) + + def prepare_round_info(self, fl_ctx, pay_load): + current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) + start_round = pay_load.get(AppConstants.START_ROUND, 0) + num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) + + fl_ctx.set_prop(AppConstants.CURRENT_ROUND, current_round, private=True, sticky=True) + fl_ctx.set_prop(AppConstants.NUM_ROUNDS, num_rounds, private=True, sticky=True) + fl_ctx.set_prop(AppConstants.START_ROUND, start_round, private=True, sticky=True) + if current_round == start_round: + self.fire_event(AppEventType.ROUND_STARTED, fl_ctx) + return current_round + + def get_payload_task(self, pay_load) -> Tuple[Task, int, List[str]]: + min_responses = pay_load.get(MIN_RESPONSES) + current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) + start_round = pay_load.get(AppConstants.START_ROUND, 0) + num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) + targets = pay_load.get(TARGET_SITES, self.get_site_names()) + + data = pay_load.get(DATA, {}) + data_shareable = self.get_shareable(data) + data_shareable.set_header(AppConstants.START_ROUND, start_round) + data_shareable.set_header(AppConstants.CURRENT_ROUND, current_round) + data_shareable.set_header(AppConstants.NUM_ROUNDS, num_rounds) + data_shareable.add_cookie(AppConstants.CONTRIBUTION_ROUND, current_round) + + operator = { + TaskOperatorKey.OP_ID: self.task_name, + TaskOperatorKey.METHOD: OperatorMethod.BROADCAST, + TaskOperatorKey.TIMEOUT: self.task_timeout, + } + + task = Task( + name=self.task_name, + data=data_shareable, + operator=operator, + props={}, + timeout=self.task_timeout, + before_task_sent_cb=None, + result_received_cb=self._result_received_cb, + ) + + return task, min_responses, targets + + def get_shareable(self, data): + if isinstance(data, FLModel): + data_shareable: Shareable = FLModelUtils.to_shareable(data) + elif data is None: + data_shareable = Shareable() + else: + dxo = DXO(DataKind.RAW, data=data, meta={}) + data_shareable = dxo.to_shareable() + return data_shareable + + def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): + + self.log_info(fl_ctx, f"{client_task.client.name} task:'{client_task.task.name}' result callback received.\n") + + client_name = client_task.client.name + task_name = client_task.task.name + result = client_task.result + rc = result.get_return_code() + results: Dict[str, any] = {STATUS: rc} + + if rc == ReturnCode.OK: + self.log_info(fl_ctx, f"Received result entries from client:{client_name} for task {task_name}") + fl_model = FLModelUtils.from_shareable(result) + results[RESULT] = {client_name: fl_model} + payload = {CMD: RESULT, PAYLOAD: {task_name: results}} + self.wf_queue.put_result(payload) + else: + self.handle_client_errors(rc, client_task, fl_ctx) + + # Cleanup task result + client_task.result = None + + def get_site_names(self): + return [client.name for client in self.clients] + + def handle_client_errors(self, + rc: str, + client_task: ClientTask, + fl_ctx: FLContext): + abort = ABORT_WHEN_IN_ERROR[rc] + if abort: + self.wf_queue.ask_abort(f"error code {rc} occurred") + self.log_error(fl_ctx, f"error code = {rc}") + self.system_panic( + f"Failed in client-site for {client_task.client.name} during task {client_task.task.name}.", + fl_ctx=fl_ctx, + ) + self.log_error(fl_ctx, f"Execution failed for {client_task.client.name}") + else: + raise ValueError(f"Execution result is not received for {client_task.client.name}") diff --git a/nvflare/app_common/workflows/error_handle_utils.py b/nvflare/app_common/workflows/error_handle_utils.py new file mode 100644 index 0000000000..f7fd16bf97 --- /dev/null +++ b/nvflare/app_common/workflows/error_handle_utils.py @@ -0,0 +1,10 @@ +from nvflare.apis.fl_constant import ReturnCode + +ABORT_WHEN_IN_ERROR = { + ReturnCode.EXECUTION_EXCEPTION: True, + ReturnCode.TASK_UNKNOWN: True, + ReturnCode.EXECUTION_RESULT_ERROR: False, + ReturnCode.TASK_DATA_FILTER_ERROR: True, + ReturnCode.TASK_RESULT_FILTER_ERROR: True, +} + diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index 41eb428b04..92dfd09929 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -12,283 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time -from concurrent.futures import ThreadPoolExecutor -from queue import Queue -from typing import Dict, List, Tuple +from typing import Dict -from nvflare.apis.client import Client -from nvflare.apis.controller_spec import ClientTask, OperatorMethod, Task, TaskOperatorKey -from nvflare.apis.dxo import DXO, DataKind -from nvflare.apis.fl_constant import ReturnCode from nvflare.apis.fl_context import FLContext -from nvflare.apis.shareable import Shareable +from nvflare.apis.impl.controller import Controller from nvflare.apis.signal import Signal -from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.app_constant import AppConstants -from nvflare.app_common.app_event_type import AppEventType -from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.workflows.error_handling_controller import ErrorHandlingController -from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( - CMD, - CMD_ABORT, - CMD_BROADCAST, - CMD_RELAY, - CMD_SEND, - CMD_STOP, - DATA, - MIN_RESPONSES, - PAYLOAD, - RESULT, - SITE_NAMES, - STATUS, - TARGET_SITES, -) -from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue -from nvflare.app_common.workflows.wf_comm.wf_spec import WF -from nvflare.fuel.utils import class_utils -from nvflare.security.logging import secure_format_traceback +from nvflare.app_common.common_workflows.base_wf_controller import BaseWFController -class WFController(ErrorHandlingController): +class WFController(BaseWFController, Controller): def __init__( - self, - task_name: str, - wf_class_path: str, - wf_args: Dict, - task_timeout: int = 0, - comm_msg_pull_interval: float = 0.2, + self, + task_name: str, + wf_class_path: str, + wf_args: Dict, + task_timeout: int = 0, + comm_msg_pull_interval: float = 0.2, ): - super().__init__() - - self.clients = None - self.task_timeout = task_timeout - self.task_name = task_name - self.comm_msg_pull_interval = comm_msg_pull_interval - self.wf_class_path = wf_class_path - self.wf_args = wf_args - self.wf_queue: WFQueue = WFQueue(ctrl_queue=Queue(), result_queue=Queue()) - self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) - self._thread_pool_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=self.__class__.__name__) - - self.engine = None - self.fl_ctx = None - - def start_controller(self, fl_ctx: FLContext): - self.fl_ctx = fl_ctx - self.log_info(fl_ctx, "Initializing controller workflow.") - self.engine = self.fl_ctx.get_engine() - self.clients = self.engine.get_clients() - - self.setup_wf_queue() - - self.log_info(fl_ctx, "workflow controller started") - - def setup_wf_queue(self): - wf_comm_api = self.find_wf_comm_in_wf() - wf_comm_api.set_queue(self.wf_queue) - wf_comm_api.set_result_pull_interval(self.comm_msg_pull_interval) - wf_comm_api.meta.update({SITE_NAMES: self.get_site_names()}) - - def find_wf_comm_in_wf(self): - attr_objs = [getattr(self.wf, attr_name, None) for attr_name in dir(self.wf)] - wf_comm_attrs = [attr for attr in attr_objs if isinstance(attr, WFCommAPI)] - if wf_comm_attrs: - return wf_comm_attrs[0] - else: - raise RuntimeError(f"missing required attribute with type of 'WFCommAPI' in {self.wf.__class__.__name__}") + super().__init__(task_name, wf_class_path, wf_args, task_timeout, comm_msg_pull_interval) def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): - try: - future = self._thread_pool_executor.submit(self.ctrl_msg_loop, fl_ctx=fl_ctx, abort_signal=abort_signal) - self.wf.run() - self.stop_msg_queue("job completed", fl_ctx) - future.result() - except Exception as e: - error_msg = secure_format_traceback() - self.log_error(fl_ctx, error_msg) - self.system_panic(error_msg, fl_ctx=fl_ctx) - finally: - wait_time = self.comm_msg_pull_interval + 0.05 - self.stop_msg_queue("job finished", fl_ctx, wait_time) - - def stop_msg_queue(self, stop_message, fl_ctx, wait_time: float = 0): - self.wf_queue.stop(stop_message) - self.log_info(fl_ctx, stop_message) - - if wait_time > 0: - self.log_info(fl_ctx, f"wait for {wait_time} sec") - time.sleep(wait_time) - - def stop_controller(self, fl_ctx: FLContext): - self.stop_msg_queue("job completed", fl_ctx) - if self._thread_pool_executor: - self._thread_pool_executor.shutdown() - - def process_result_of_unknown_task( - self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext - ): - pass - - def ctrl_msg_loop(self, fl_ctx: FLContext, abort_signal: Signal): - - if self.wf_queue is None: - raise ValueError("WFQueue must provided") - - try: - while True: - if abort_signal.triggered: - break - if not self.wf_queue.has_ctrl_msg(): - time.sleep(self.comm_msg_pull_interval) - else: - item = self.wf_queue.get_ctrl_msg() - if item is None: - self.log_warning(fl_ctx, "Ignore 'None' ctrl comm message") - continue - - cmd = item.get(CMD, None) - - if cmd is None: - msg = f"get None command, expecting {CMD} key'" - self.log_error(fl_ctx, msg) - raise ValueError(msg) - - elif cmd == CMD_STOP: - msg = item.get(PAYLOAD) - self.log_info(fl_ctx, f"receive {CMD_STOP} command, {msg}") - break - - elif cmd == CMD_ABORT: - msg = item.get(PAYLOAD) - self.log_info(fl_ctx, f"receive {CMD_ABORT} command, {msg}") - raise RuntimeError(msg) - - elif cmd == CMD_BROADCAST: - pay_load = item.get(PAYLOAD) - - current_round = self.prepare_round_info(fl_ctx, pay_load) - task, min_responses, targets = self.get_payload_task(pay_load) - - self.broadcast_and_wait( - task=task, - targets=targets, - min_responses=min_responses, - wait_time_after_min_received=0, - fl_ctx=fl_ctx, - abort_signal=abort_signal, - ) - self.fire_event(AppEventType.ROUND_DONE, fl_ctx) - self.log_info(fl_ctx, f"Round {current_round} finished.") - - elif cmd == CMD_RELAY: - pay_load = item.get(PAYLOAD) - current_round = self.prepare_round_info(fl_ctx, pay_load) - task, min_responses, targets = self.get_payload_task(pay_load) - - self.relay_and_wait( - task=task, - targets=targets, - fl_ctx=fl_ctx, - abort_signal=abort_signal, - ) - self.fire_event(AppEventType.ROUND_DONE, fl_ctx) - self.log_info(fl_ctx, f"Round {current_round} finished.") - - elif cmd == CMD_SEND: - raise NotImplementedError - else: - abort_signal.trigger(f"Unknown command '{cmd}'") - raise ValueError(f"Unknown command '{cmd}'") - - if abort_signal.triggered: - self.log_debug(self.fl_ctx, f"task {self.task_name} aborted") - break - except Exception as e: - error_msg = secure_format_traceback() - self.wf_queue.ask_abort(error_msg) - self.log_error(fl_ctx, error_msg) - self.system_panic(error_msg, fl_ctx=fl_ctx) - - def prepare_round_info(self, fl_ctx, pay_load): - current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) - start_round = pay_load.get(AppConstants.START_ROUND, 0) - num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) - - fl_ctx.set_prop(AppConstants.CURRENT_ROUND, current_round, private=True, sticky=True) - fl_ctx.set_prop(AppConstants.NUM_ROUNDS, num_rounds, private=True, sticky=True) - fl_ctx.set_prop(AppConstants.START_ROUND, start_round, private=True, sticky=True) - if current_round == start_round: - self.fire_event(AppEventType.ROUND_STARTED, fl_ctx) - return current_round - - def get_payload_task(self, pay_load) -> Tuple[Task, int, List[str]]: - min_responses = pay_load.get(MIN_RESPONSES) - current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) - start_round = pay_load.get(AppConstants.START_ROUND, 0) - num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) - targets = pay_load.get(TARGET_SITES, self.get_site_names()) - - data = pay_load.get(DATA, {}) - data_shareable = self.get_shareable(data) - data_shareable.set_header(AppConstants.START_ROUND, start_round) - data_shareable.set_header(AppConstants.CURRENT_ROUND, current_round) - data_shareable.set_header(AppConstants.NUM_ROUNDS, num_rounds) - data_shareable.add_cookie(AppConstants.CONTRIBUTION_ROUND, current_round) - - operator = { - TaskOperatorKey.OP_ID: self.task_name, - TaskOperatorKey.METHOD: OperatorMethod.BROADCAST, - TaskOperatorKey.TIMEOUT: self.task_timeout, - } - - task = Task( - name=self.task_name, - data=data_shareable, - operator=operator, - props={}, - timeout=self.task_timeout, - before_task_sent_cb=None, - result_received_cb=self._result_received_cb, - ) - - return task, min_responses, targets - - def get_shareable(self, data): - if isinstance(data, FLModel): - data_shareable: Shareable = FLModelUtils.to_shareable(data) - elif data is None: - data_shareable = Shareable() - else: - dxo = DXO(DataKind.RAW, data=data, meta={}) - data_shareable = dxo.to_shareable() - return data_shareable - - def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): - - self.log_info(fl_ctx, f"{client_task.client.name} task:'{client_task.task.name}' result callback received.\n") - - client_name = client_task.client.name - task_name = client_task.task.name - result = client_task.result - rc = result.get_return_code() - results: Dict[str, any] = {STATUS: rc} - - if rc == ReturnCode.OK: - self.log_info(fl_ctx, f"Received result entries from client:{client_name} for task {task_name}") - fl_model = FLModelUtils.from_shareable(result) - results[RESULT] = {client_name: fl_model} - payload = {CMD: RESULT, PAYLOAD: {task_name: results}} - self.wf_queue.put_result(payload) - elif rc in self.abort_job_in_error.keys(): - self.wf_queue.ask_abort(f"error code {rc} occurred") - self.handle_client_errors(rc, client_task, fl_ctx) - else: - self.log_warning(fl_ctx, f"ignore result with return code: {rc}") - - # Cleanup task result - client_task.result = None + self.start_workflow(abort_signal, fl_ctx) - def get_site_names(self): - return [client.name for client in self.clients] From 432260253d09817c6caac4b0d0257f129ea7cf8a Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 16:26:19 -0800 Subject: [PATCH 09/41] add header --- nvflare/app_common/workflows/error_handle_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nvflare/app_common/workflows/error_handle_utils.py b/nvflare/app_common/workflows/error_handle_utils.py index f7fd16bf97..94575f2860 100644 --- a/nvflare/app_common/workflows/error_handle_utils.py +++ b/nvflare/app_common/workflows/error_handle_utils.py @@ -1,3 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from nvflare.apis.fl_constant import ReturnCode ABORT_WHEN_IN_ERROR = { From 3cab712a185a3c77a392892d2b6640d198e997ea Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 16:27:09 -0800 Subject: [PATCH 10/41] add header --- nvflare/app_common/common_workflows/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nvflare/app_common/common_workflows/__init__.py b/nvflare/app_common/common_workflows/__init__.py index e69de29bb2..4fc50543f1 100644 --- a/nvflare/app_common/common_workflows/__init__.py +++ b/nvflare/app_common/common_workflows/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 6221dff9b81643e0af64f57eba3554b3016fa7c9 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 16:28:05 -0800 Subject: [PATCH 11/41] code style format --- .../common_workflows/base_wf_controller.py | 25 ++++++++----------- .../workflows/error_handle_utils.py | 1 - nvflare/app_common/workflows/wf_controller.py | 13 +++++----- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 372559597d..679a15d822 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -19,7 +19,7 @@ from typing import Dict, List, Tuple from nvflare.apis.client import Client -from nvflare.apis.controller_spec import ClientTask, OperatorMethod, Task, TaskOperatorKey, ControllerSpec +from nvflare.apis.controller_spec import ClientTask, ControllerSpec, OperatorMethod, Task, TaskOperatorKey from nvflare.apis.dxo import DXO, DataKind from nvflare.apis.fl_component import FLComponent from nvflare.apis.fl_constant import ReturnCode @@ -55,12 +55,12 @@ class BaseWFController(FLComponent, ControllerSpec, ABC): def __init__( - self, - task_name: str, - wf_class_path: str, - wf_args: Dict, - task_timeout: int = 0, - comm_msg_pull_interval: float = 0.2, + self, + task_name: str, + wf_class_path: str, + wf_args: Dict, + task_timeout: int = 0, + comm_msg_pull_interval: float = 0.2, ): super().__init__() @@ -129,7 +129,7 @@ def stop_controller(self, fl_ctx: FLContext): self._thread_pool_executor.shutdown() def process_result_of_unknown_task( - self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext + self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext ): pass @@ -292,17 +292,14 @@ def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): def get_site_names(self): return [client.name for client in self.clients] - def handle_client_errors(self, - rc: str, - client_task: ClientTask, - fl_ctx: FLContext): + def handle_client_errors(self, rc: str, client_task: ClientTask, fl_ctx: FLContext): abort = ABORT_WHEN_IN_ERROR[rc] if abort: self.wf_queue.ask_abort(f"error code {rc} occurred") self.log_error(fl_ctx, f"error code = {rc}") self.system_panic( - f"Failed in client-site for {client_task.client.name} during task {client_task.task.name}.", - fl_ctx=fl_ctx, + f"Failed in client-site for {client_task.client.name} during task {client_task.task.name}.", + fl_ctx=fl_ctx, ) self.log_error(fl_ctx, f"Execution failed for {client_task.client.name}") else: diff --git a/nvflare/app_common/workflows/error_handle_utils.py b/nvflare/app_common/workflows/error_handle_utils.py index 94575f2860..18f7515f47 100644 --- a/nvflare/app_common/workflows/error_handle_utils.py +++ b/nvflare/app_common/workflows/error_handle_utils.py @@ -21,4 +21,3 @@ ReturnCode.TASK_DATA_FILTER_ERROR: True, ReturnCode.TASK_RESULT_FILTER_ERROR: True, } - diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index 92dfd09929..cb75a8b2f4 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -22,15 +22,14 @@ class WFController(BaseWFController, Controller): def __init__( - self, - task_name: str, - wf_class_path: str, - wf_args: Dict, - task_timeout: int = 0, - comm_msg_pull_interval: float = 0.2, + self, + task_name: str, + wf_class_path: str, + wf_args: Dict, + task_timeout: int = 0, + comm_msg_pull_interval: float = 0.2, ): super().__init__(task_name, wf_class_path, wf_args, task_timeout, comm_msg_pull_interval) def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) - From f535c62c6b1da064185690e2ee7a699916497ffc Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 17:00:08 -0800 Subject: [PATCH 12/41] make better user experience --- .../hello-world/hello-cyclic-pt/README.md | 17 +++++++++++++---- .../jobs/cyclic/app/custom/fed_cyclic.py | 17 +++++++---------- examples/hello-world/hello-fedavg/README.md | 19 +++++++++++++------ .../jobs/fedavg/app/custom/fedavg.py | 3 --- examples/hello-world/hello-km/README.md | 17 ++++++++++++++--- .../kaplan-meier/app/custom/kaplan_meier.py | 1 - .../common_workflows/base_wf_controller.py | 12 +++++++----- .../app_common/workflows/wf_comm/wf_spec.py | 10 ++++++++++ 8 files changed, 64 insertions(+), 32 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/README.md b/examples/hello-world/hello-cyclic-pt/README.md index d13fdaae37..5d0b4f3cde 100644 --- a/examples/hello-world/hello-cyclic-pt/README.md +++ b/examples/hello-world/hello-cyclic-pt/README.md @@ -55,8 +55,6 @@ class FedCyclic(WF): - # (1) instantiate flare_comm - self.flare_comm = WFCommAPI() def run(self): @@ -109,13 +107,24 @@ Relay_and_wait The base class ```WF``` is define as ``` + class WF(ABC): + def __init__(self): + self.flare_comm: Optional[WFCommAPI] = None + + def setup_wf_comm_api(self, flare_comm: WFCommAPI): + self.flare_comm = flare_comm + @abstractmethod def run(self): - raise NotImplemented + raise NotImplementedError + ``` -is mainly make sure user define ```run()``` method +has two expectations: +* Make sure user define ```run()``` method +* make sure a class field of WFCommAPI and be able to dynamically populated at runtime +via setup_wf_comm_api() method ## Configurations diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index ed3896ab61..d991c0fa81 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -18,8 +18,8 @@ from typing import List, Optional import torch -from net import Net +from net import Net from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI @@ -50,12 +50,12 @@ class RelayOrder: class FedCyclic(WF): def __init__( - self, - output_path: str, - num_rounds: int = 5, - start_round: int = 0, - task_name="train", - order: str = RelayOrder.FIXED, + self, + output_path: str, + num_rounds: int = 5, + start_round: int = 0, + task_name="train", + order: str = RelayOrder.FIXED, ): super(FedCyclic, self).__init__() self.logger = logging.getLogger(self.__class__.__name__) @@ -71,9 +71,6 @@ def __init__( self.check_inputs() - # (1) instantiate flare_comm - self.flare_comm = WFCommAPI() - def run(self): self.last_model = self.init_model() diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md index 8fb07cdf51..a7267f2acc 100644 --- a/examples/hello-world/hello-fedavg/README.md +++ b/examples/hello-world/hello-fedavg/README.md @@ -55,9 +55,6 @@ class FedAvg(WF): super(FedAvg, self).__init__() - - # (1) init flare_comm - self.flare_comm = WFCommAPI() def run(self): self.logger.info("start Fed Avg Workflow\n \n") @@ -103,17 +100,27 @@ SAG is simply ask WFController to broadcast the model to all clients results = self.flare_comm.broadcast_and_wait(msg_payload) return results ``` - The base class ```WF``` is define as ``` + class WF(ABC): + def __init__(self): + self.flare_comm: Optional[WFCommAPI] = None + + def setup_wf_comm_api(self, flare_comm: WFCommAPI): + self.flare_comm = flare_comm + @abstractmethod def run(self): - raise NotImplemented + raise NotImplementedError + ``` -is mainly make sure user define ```run()``` method +has two expectations: +* Make sure user define ```run()``` method +* make sure a class field of WFCommAPI and be able to dynamically populated at runtime + via setup_wf_comm_api() method ## Configurations diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index 2136164117..4ea8ef7ef6 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -68,9 +68,6 @@ def __init__( else: self.metric_comp_rule = None - # (1) init flare_comm - self.flare_comm = WFCommAPI() - def run(self): self.logger.info("start Fed Avg Workflow\n \n") diff --git a/examples/hello-world/hello-km/README.md b/examples/hello-world/hello-km/README.md index 79956e2a4b..f5582eddbb 100644 --- a/examples/hello-world/hello-km/README.md +++ b/examples/hello-world/hello-km/README.md @@ -51,7 +51,6 @@ class KM(WF): self.output_path = output_path self.min_clients = min_clients self.num_rounds = 1 - self.flare_comm = WFCommAPI() def run(self): results = self.start_km_analysis() @@ -63,13 +62,25 @@ class KM(WF): The base class ```WF``` is define as ``` + class WF(ABC): + def __init__(self): + self.flare_comm: Optional[WFCommAPI] = None + + def setup_wf_comm_api(self, flare_comm: WFCommAPI): + self.flare_comm = flare_comm + @abstractmethod def run(self): - raise NotImplemented + raise NotImplementedError + ``` -is mainly make sure user define ```run()``` method +has two expectations: +* Make sure user define ```run()``` method +* make sure a class field of WFCommAPI and be able to dynamically populated at runtime + via setup_wf_comm_api() method + for Kaplan-Meier analysis, it literal involves diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py index 0ca1b24d94..a093947743 100644 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py @@ -40,7 +40,6 @@ def __init__(self, min_clients: int, output_path: str): self.output_path = output_path self.min_clients = min_clients self.num_rounds = 1 - self.flare_comm = WFCommAPI() def run(self): results = self.start_km_analysis() diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 679a15d822..a4f6fe2bcb 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -64,6 +64,7 @@ def __init__( ): super().__init__() + self.wf = None self.clients = None self.task_timeout = task_timeout self.task_name = task_name @@ -71,7 +72,6 @@ def __init__( self.wf_class_path = wf_class_path self.wf_args = wf_args self.wf_queue: WFQueue = WFQueue(ctrl_queue=Queue(), result_queue=Queue()) - self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) self._thread_pool_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=self.__class__.__name__) self.engine = None @@ -88,10 +88,12 @@ def start_controller(self, fl_ctx: FLContext): self.log_info(fl_ctx, "workflow controller started") def setup_wf_queue(self): - wf_comm_api = self.find_wf_comm_in_wf() - wf_comm_api.set_queue(self.wf_queue) - wf_comm_api.set_result_pull_interval(self.comm_msg_pull_interval) - wf_comm_api.meta.update({SITE_NAMES: self.get_site_names()}) + self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) + comm_api = WFCommAPI() + comm_api.set_queue(self.wf_queue) + comm_api.set_result_pull_interval(self.comm_msg_pull_interval) + comm_api.meta.update({SITE_NAMES: self.get_site_names()}) + self.wf.setup_wf_comm_api(comm_api) def find_wf_comm_in_wf(self): attr_objs = [getattr(self.wf, attr_name, None) for attr_name in dir(self.wf)] diff --git a/nvflare/app_common/workflows/wf_comm/wf_spec.py b/nvflare/app_common/workflows/wf_comm/wf_spec.py index 9954e8a8c4..d4d1cd2df4 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_spec.py +++ b/nvflare/app_common/workflows/wf_comm/wf_spec.py @@ -13,9 +13,19 @@ # limitations under the License. from abc import ABC, abstractmethod +from typing import Optional + +from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI class WF(ABC): + + def __init__(self): + self.flare_comm: Optional[WFCommAPI] = None + + def setup_wf_comm_api(self, flare_comm: WFCommAPI): + self.flare_comm = flare_comm + @abstractmethod def run(self): raise NotImplementedError From c6e19c78a6ab1011338bb598e8ab98d646e0cc41 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 17:06:00 -0800 Subject: [PATCH 13/41] code format and import --- .../jobs/cyclic/app/custom/fed_cyclic.py | 14 +++++++------- .../hello-fedavg/jobs/fedavg/app/custom/fedavg.py | 8 ++------ .../jobs/kaplan-meier/app/custom/kaplan_meier.py | 1 - nvflare/app_common/workflows/wf_comm/wf_spec.py | 1 - 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index d991c0fa81..de78f285d5 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -20,9 +20,9 @@ import torch from net import Net + from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, @@ -50,12 +50,12 @@ class RelayOrder: class FedCyclic(WF): def __init__( - self, - output_path: str, - num_rounds: int = 5, - start_round: int = 0, - task_name="train", - order: str = RelayOrder.FIXED, + self, + output_path: str, + num_rounds: int = 5, + start_round: int = 0, + task_name="train", + order: str = RelayOrder.FIXED, ): super(FedCyclic, self).__init__() self.logger = logging.getLogger(self.__class__.__name__) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index 4ea8ef7ef6..20591ab403 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -13,16 +13,13 @@ # limitations under the License. import logging -import traceback from typing import Callable, Dict, Optional from net import Net - from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.utils.math_utils import parse_compare_criteria, parse_compare_operator -from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, @@ -31,7 +28,7 @@ START_ROUND, ) from nvflare.app_common.workflows.wf_comm.wf_spec import WF -from nvflare.security.logging import secure_format_exception +from nvflare.security.logging import secure_format_traceback update_model = FLModelUtils.update_model @@ -167,8 +164,7 @@ def aggr_fn(self, sag_result: Dict[str, Dict[str, FLModel]]) -> FLModel: ) return aggr_result except Exception as e: - traceback_str = traceback.format_exc() - raise RuntimeError(f"Exception in aggregate call: {secure_format_exception(e, traceback_str)}") + raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") def select_best_model(self, curr_model: FLModel): if self.best_model is None: diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py index a093947743..2d8ad11bc0 100644 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py @@ -20,7 +20,6 @@ from km_analysis import kaplan_meier_analysis from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, diff --git a/nvflare/app_common/workflows/wf_comm/wf_spec.py b/nvflare/app_common/workflows/wf_comm/wf_spec.py index d4d1cd2df4..22785a7587 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_spec.py +++ b/nvflare/app_common/workflows/wf_comm/wf_spec.py @@ -19,7 +19,6 @@ class WF(ABC): - def __init__(self): self.flare_comm: Optional[WFCommAPI] = None From a3fb099b81faff73033e44e997803a2f35791452 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 17:06:39 -0800 Subject: [PATCH 14/41] remove comment --- .../hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index de78f285d5..1ccfee2ac0 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -75,8 +75,6 @@ def run(self): self.last_model = self.init_model() - # note: this one must be within run() method, not in the __init__() method - # as some values are injected at runtime during run() self.part_sites = self.flare_comm.get_site_names() if len(self.part_sites) <= 1: From 6b57dc926867e1930134a9b83bc73b936e1d8258 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 30 Dec 2023 17:08:43 -0800 Subject: [PATCH 15/41] remove used method --- nvflare/app_common/common_workflows/base_wf_controller.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index a4f6fe2bcb..f69b98a9ef 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -95,14 +95,6 @@ def setup_wf_queue(self): comm_api.meta.update({SITE_NAMES: self.get_site_names()}) self.wf.setup_wf_comm_api(comm_api) - def find_wf_comm_in_wf(self): - attr_objs = [getattr(self.wf, attr_name, None) for attr_name in dir(self.wf)] - wf_comm_attrs = [attr for attr in attr_objs if isinstance(attr, WFCommAPI)] - if wf_comm_attrs: - return wf_comm_attrs[0] - else: - raise RuntimeError(f"missing required attribute with type of 'WFCommAPI' in {self.wf.__class__.__name__}") - def start_workflow(self, abort_signal, fl_ctx): try: future = self._thread_pool_executor.submit(self.ctrl_msg_loop, fl_ctx=fl_ctx, abort_signal=abort_signal) From eb2272f56e25a4fce2e88d085867cd625836c4e8 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Tue, 2 Jan 2024 09:56:31 -0800 Subject: [PATCH 16/41] 1. add intime aggregate version of fedavg 2. add late result handling 3. change the API spec to allow get result one at a time --- .../jobs/cyclic/app/custom/fed_cyclic.py | 1 - .../jobs/fedavg/app/custom/fedavg.py | 20 +- .../jobs/fedavg/app/custom/fedavg_intime.py | 247 ++++++++++++++++++ .../jobs/fedavg/app/custom/fedavg_pt.py | 3 + .../common_workflows/base_wf_controller.py | 4 +- .../workflows/wf_comm/wf_comm_api.py | 120 ++++++--- .../workflows/wf_comm/wf_comm_api_spec.py | 37 ++- .../app_common/workflows/wf_comm/wf_queue.py | 8 +- 8 files changed, 385 insertions(+), 55 deletions(-) create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index 1ccfee2ac0..77d876066a 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -18,7 +18,6 @@ from typing import List, Optional import torch - from net import Net from nvflare.app_common.abstract.fl_model import FLModel, ParamsType diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index 20591ab403..e112615a76 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -13,9 +13,11 @@ # limitations under the License. import logging +import sys from typing import Callable, Dict, Optional from net import Net + from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils @@ -25,6 +27,7 @@ DATA, MIN_RESPONSES, NUM_ROUNDS, + RESP_MAX_WAIT_TIME, START_ROUND, ) from nvflare.app_common.workflows.wf_comm.wf_spec import WF @@ -45,12 +48,14 @@ def __init__( start_round: int = 1, stop_cond: str = None, model_selection_rule: str = None, + resp_max_wait_time: float = 5, ): super(FedAvg, self).__init__() self.logger = logging.getLogger(self.__class__.__name__) self.output_path = output_path self.min_clients = min_clients + self.resp_max_wait_time = resp_max_wait_time self.num_rounds = num_rounds self.start_round = start_round self.current_round = start_round @@ -80,13 +85,14 @@ def run(self): if self.should_stop(model.metrics, self.stop_criteria): self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") break - sag_results = self.scatter_and_gather(model, current_round) aggr_result = self.aggr_fn(sag_results) self.logger.info(f"aggregate metrics = {aggr_result.metrics}") + print("model size =", sys.getsizeof(model.params)) + model = update_model(model, aggr_result) self.select_best_model(model) @@ -101,8 +107,10 @@ def init_model(self): return model def scatter_and_gather(self, model: FLModel, current_round): + msg_payload = { MIN_RESPONSES: self.min_clients, + RESP_MAX_WAIT_TIME: self.resp_max_wait_time, CURRENT_ROUND: current_round, NUM_ROUNDS: self.num_rounds, START_ROUND: self.start_round, @@ -111,6 +119,7 @@ def scatter_and_gather(self, model: FLModel, current_round): # (2) broadcast and wait results = self.flare_comm.broadcast_and_wait(msg_payload) + print(f"{results=}") return results def aggr_fn(self, sag_result: Dict[str, Dict[str, FLModel]]) -> FLModel: @@ -120,11 +129,9 @@ def aggr_fn(self, sag_result: Dict[str, Dict[str, FLModel]]) -> FLModel: if not sag_result: raise RuntimeError("input is None or empty") + # we only have one task task_name, task_result = next(iter(sag_result.items())) - if not task_result: - raise RuntimeError("task_result is None or empty ") - self.logger.info(f"aggregating {len(task_result)} update(s) at round {self.current_round}") try: @@ -160,7 +167,10 @@ def aggr_fn(self, sag_result: Dict[str, Dict[str, FLModel]]) -> FLModel: params=aggr_params, params_type=params_type, metrics=aggr_metrics, - meta={"num_rounds_aggregated": len(task_result), "current_round": self.current_round}, + meta={ + "num_rounds_aggregated": 1 + (self.current_round - self.start_round), + "current_round": self.current_round, + }, ) return aggr_result except Exception as e: diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py new file mode 100644 index 0000000000..4d3faeae2e --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py @@ -0,0 +1,247 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import time +import traceback +from typing import Callable, Dict, Optional, Tuple + +from net import Net + +from nvflare.app_common.abstract.fl_model import FLModel, ParamsType +from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper +from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.utils.math_utils import parse_compare_criteria, parse_compare_operator +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( + CURRENT_ROUND, + DATA, + MIN_RESPONSES, + NUM_ROUNDS, + RESP_MAX_WAIT_TIME, + START_ROUND, +) +from nvflare.app_common.workflows.wf_comm.wf_spec import WF +from nvflare.security.logging import secure_format_traceback + +update_model = FLModelUtils.update_model + + +# FedAvg Workflow + + +class FedAvg(WF): + def __init__( + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + model_selection_rule: str = None, + resp_max_wait_time: float = 5.0, + ): + super(FedAvg, self).__init__() + self.logger = logging.getLogger(self.__class__.__name__) + + self.output_path = output_path + self.min_clients = min_clients + self.num_rounds = num_rounds + self.start_round = start_round + self.current_round = start_round + self.resp_max_wait_time = resp_max_wait_time + self.best_model: Optional[FLModel] = None + if stop_cond: + self.stop_criteria = parse_compare_criteria(stop_cond) + else: + self.stop_criteria = None + + if model_selection_rule: + self.metric_comp_rule = parse_compare_operator(model_selection_rule) + else: + self.metric_comp_rule = None + + def run(self): + self.logger.info("start Fed Avg Workflow\n \n") + + start = self.start_round + end = self.start_round + self.num_rounds + + model = self.init_model() + for current_round in range(start, end): + + self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") + self.current_round = current_round + + if self.should_stop(model.metrics, self.stop_criteria): + self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") + break + + self.scatter(model, current_round) + + self.logger.info("gather and aggregate") + aggr_result = self.gather_and_aggr(self.in_time_aggr_fn) + + self.logger.info(f"aggregate metrics = {aggr_result.metrics}") + + model = update_model(model, aggr_result) + + self.select_best_model(model) + + self.save_model(self.best_model, self.output_path) + + self.logger.info("end Fed Avg Workflow\n \n") + + def init_model(self): + net = Net() + model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) + return model + + def scatter(self, model: FLModel, current_round): + msg_payload = { + MIN_RESPONSES: self.min_clients, + RESP_MAX_WAIT_TIME: self.resp_max_wait_time, + CURRENT_ROUND: current_round, + NUM_ROUNDS: self.num_rounds, + START_ROUND: self.start_round, + DATA: model, + } + + # (2) broadcast and wait + self.flare_comm.broadcast(msg_payload) + + def gather_and_aggr(self, in_time_aggr_fn: Callable): + responses = 0 + start = None + aggr_model = None + aggr_params_helper = WeightedAggregationHelper() + aggr_metrics_helper = WeightedAggregationHelper() + helpers = (aggr_params_helper, aggr_metrics_helper) + + while True: + if responses >= self.min_clients: + return aggr_model + else: + time.sleep(0.2) + + max_timeout = self.resp_max_wait_time if start else None + self.logger.warning(f"{max_timeout=}, {responses=}, {self.min_clients=}") + try: + item = self.flare_comm.wait_one(max_timeout) + except RuntimeError as e: + self.logger.error(traceback.format_exc()) + break + + task_name, site_name, model = item + aggr_model = in_time_aggr_fn(helpers, aggr_model, site_name, model) + start = time.time() if start is None else start + responses += 1 + + if responses < self.min_clients: + raise RuntimeError( + f"not enough responses {responses} compare with min responses requirement {self.min_clients} within the max allowed time {self.resp_max_wait_time} seconds" + ) + else: + return aggr_model + + def in_time_aggr_fn(self, helpers: Tuple, prev_mode: FLModel, site_name: str, fl_model: FLModel) -> FLModel: + + self.logger.info("Fed Avg in time aggregate \n") + if not fl_model: + raise RuntimeError("model must not be None") + + self.logger.info(f"aggregating update at round {self.current_round}") + + self.logger.info(f"site={site_name} {fl_model.metrics=}") + + if prev_mode is None: + self.logger.info(f"aggr_metrics={fl_model.metrics}") + return fl_model + + try: + params_type = fl_model.params_type + aggr_params_helper, aggr_metrics_helper = helpers + + aggr_params_helper.add( + data=fl_model.params, + weight=self.current_round, + contributor_name=site_name, + contribution_round=self.current_round, + ) + + aggr_metrics_helper.add( + data=fl_model.metrics, + weight=self.current_round, + contributor_name=site_name, + contribution_round=self.current_round, + ) + + aggr_params = aggr_params_helper.get_result() + aggr_metrics = aggr_metrics_helper.get_result() + + self.logger.info(f"{aggr_metrics=}") + + aggr_result = FLModel( + params=aggr_params, + params_type=params_type, + metrics=aggr_metrics, + meta={ + "num_rounds_aggregated": 1 + (self.current_round - self.start_round), + "current_round": self.current_round, + }, + ) + return aggr_result + + except Exception as e: + raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") + + def select_best_model(self, curr_model: FLModel): + if self.best_model is None: + self.best_model = curr_model + return + + if self.metric_comp_rule is None: + return + metric, op_fn = self.metric_comp_rule + + self.logger.info("compare models") + if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): + self.best_model = curr_model + + def save_model(self, model: FLModel, file_path: str): + pass + + def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[str] = None): + self.logger.info(f"stop_criteria, metrics = {stop_criteria=}, {metrics=}") + if stop_criteria is None or metrics is None: + return False + + key, target, op_fn = stop_criteria + value = metrics.get(key, None) + + if value is None: + raise RuntimeError(f"stop criteria key '{key}' doesn't exists in metrics") + + return op_fn(value, target) + + def is_curr_mode_better( + self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable + ) -> bool: + curr_metrics = curr_model.metrics + if curr_metrics is None: + return False + if target_metric not in curr_metrics: + return False + + best_metrics = best_model.metrics + return op_fn(curr_metrics.get(target_metric), best_metrics.get(target_metric)) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py index beea8e7e68..d605c564bd 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py @@ -18,6 +18,9 @@ from nvflare.app_common.abstract.fl_model import FLModel +# to use in_time aggregate version of FedAvg +# you change the import to 'from fedavg_intime import FedAvg' + class PTFedAvg(FedAvg): def __init__( diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index f69b98a9ef..b939856e70 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -97,13 +97,13 @@ def setup_wf_queue(self): def start_workflow(self, abort_signal, fl_ctx): try: - future = self._thread_pool_executor.submit(self.ctrl_msg_loop, fl_ctx=fl_ctx, abort_signal=abort_signal) + self._thread_pool_executor.submit(self.ctrl_msg_loop, fl_ctx=fl_ctx, abort_signal=abort_signal) self.wf.run() self.stop_msg_queue("job completed", fl_ctx) - future.result() except Exception as e: error_msg = secure_format_traceback() self.log_error(fl_ctx, error_msg) + self.wf_queue.ask_abort(error_msg) self.system_panic(error_msg, fl_ctx=fl_ctx) finally: wait_time = self.comm_msg_pull_interval + 0.05 diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py index e718c13539..d39bc629fc 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py @@ -15,9 +15,11 @@ import logging import time -from typing import Dict, Optional +from queue import Empty +from typing import Dict, Optional, Tuple from nvflare.apis.fl_constant import ReturnCode +from nvflare.app_common.abstract.fl_model import FLModel from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CMD, CMD_ABORT, @@ -27,6 +29,7 @@ CMD_STOP, MIN_RESPONSES, PAYLOAD, + RESP_MAX_WAIT_TIME, RESULT, SITE_NAMES, STATUS, @@ -51,7 +54,8 @@ def set_queue(self, wf_queue: WFQueue): def broadcast_and_wait(self, msg_payload: Dict): self.broadcast(msg_payload) min_responses = msg_payload.get(MIN_RESPONSES, 0) - return self.wait(min_responses) + resp_max_wait_time = msg_payload.get(RESP_MAX_WAIT_TIME, 5) + return self.wait_all(min_responses, resp_max_wait_time) def broadcast(self, msg_payload): self._check_wf_queue() @@ -72,28 +76,41 @@ def send(self, msg_payload: Dict): def send_and_wait(self, msg_payload: Dict): self.send(msg_payload) min_responses = msg_payload.get(MIN_RESPONSES, 0) - return self.wait(min_responses) + return self.wait_all(min_responses) def get_site_names(self): return self.meta.get(SITE_NAMES) - def wait(self, min_responses): + def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float] = None) -> Dict[str, Dict[str, FLModel]]: + acc_size = 0 + start = None while True: if self.wf_queue.has_result(): + start = time.time() if start is None else start items_size = self.wf_queue.result_size() - if items_size >= min_responses: + acc_size = items_size + acc_size + time_waited = time.time() - start + self.logger.info( + f"\n\n {items_size=}, {acc_size=}, {min_responses=}, {time_waited=}, {resp_max_wait_time=}" + ) + if time_waited < resp_max_wait_time and acc_size >= min_responses: return self._get_results() else: - self.logger.info(f" wait for more results, sleep {self.result_pull_interval} sec") - time.sleep(self.result_pull_interval) + if time_waited < resp_max_wait_time: + self.logger.info(f" wait for more results, sleep {self.result_pull_interval} sec") + time.sleep(self.result_pull_interval) + else: + msg = f"not enough responses {acc_size} compare with min responses requirement {min_responses} within the max allowed time {resp_max_wait_time} seconds" + self.logger.info(msg) + raise RuntimeError(msg) + else: - # self.logger.info(f"no result available, sleep {self.result_pull_interval} sec") time.sleep(self.result_pull_interval) def relay_and_wait(self, msg_payload: Dict): self.relay(msg_payload) min_responses = msg_payload.get(MIN_RESPONSES, 1) - return self.wait(min_responses) + return self.wait_all(min_responses) def relay(self, msg_payload: Dict): self._check_wf_queue() @@ -103,42 +120,67 @@ def relay(self, msg_payload: Dict): } self.wf_queue.put_ctrl_msg(message) - def _get_results(self) -> dict: + def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: + try: + item = self.wf_queue.get_result(resp_max_wait_time) + if item: + return self._process_one_result(item) + except Empty as e: + raise RuntimeError(f"failed to get result within the given timeout {resp_max_wait_time} sec.") + + def _process_one_result(self, item) -> Tuple[str, str, FLModel]: + cmd = item.get(CMD, None) + + if cmd is None: + msg = f"get None command, expecting {CMD} key'" + self.logger.error(msg) + raise RuntimeError(msg) + + elif cmd == CMD_STOP or cmd == CMD_ABORT: + msg = item.get(PAYLOAD) + self.logger.info(f"receive {cmd} command, {msg}") + raise RuntimeError(msg) + + elif cmd == RESULT: + payload = item.get(PAYLOAD) + task_result = None + task, site_result = next(iter(payload.items())) + self._check_result(site_result) + rc = site_result.get(STATUS) + if rc == ReturnCode.OK: + result = site_result.get(RESULT, {}) + site_name, data = next(iter(result.items())) + task_result = (task, site_name, data) + else: + msg = f"task {task} failed with '{rc}' status" + self.wf_queue.ask_abort(msg) + raise RuntimeError(msg) + + return task_result + else: + raise RuntimeError(f"Unknown command {cmd}") + + def _get_results(self) -> Dict[str, Dict[str, FLModel]]: items_size = self.wf_queue.result_size() batch_result: Dict = {} - for i in range(items_size): item = self.wf_queue.get_result() - cmd = item.get(CMD, None) - - if cmd is None: - msg = f"get None command, expecting {CMD} key'" - self.logger.error(msg) - raise RuntimeError(msg) - - elif cmd == CMD_STOP or cmd == CMD_ABORT: - msg = item.get(PAYLOAD) - self.logger.info(f"receive {cmd} command, {msg}") - raise RuntimeError(msg) + task, site_name, data = self._process_one_result(item) + task_result = batch_result.get(task, {}) + task_result.update({site_name: data}) + batch_result[task] = task_result + return batch_result - elif cmd == RESULT: - one_site_result = item.get(PAYLOAD) - for task, site_result in one_site_result.items(): - task_result = batch_result.get(task, {}) - self._check_result(site_result) - rc = site_result.get(STATUS) - if rc == ReturnCode.OK: - result = site_result.get(RESULT, {}) - task_result.update(result) - batch_result[task] = task_result - else: - msg = f"task {task} failed with '{rc}' status" - self.wf_queue.ask_abort(msg) - raise RuntimeError(msg) + def wait_for_responses(self, items_size, min_responses, resp_max_wait_time): + start = time.time() + while items_size < min_responses: + time_waited = time.time() - start + if time_waited < resp_max_wait_time: + time.sleep(1) + items_size = self.wf_queue.result_size() else: - raise RuntimeError(f"Unknown command {cmd}") - - return batch_result + break + return items_size def _check_result(self, site_result): diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py index 5455da34eb..b152e2681d 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py @@ -13,7 +13,9 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import Dict +from typing import Dict, List, Optional, Tuple + +from nvflare.app_common.abstract.fl_model import FLModel CMD = "COMMAND" CMD_SEND = "SEND" @@ -26,6 +28,7 @@ # note same as app_constant constant (todo: we only need one constant definition) MIN_RESPONSES = "min_responses" +RESP_MAX_WAIT_TIME = "resp_max_wait_time" START_ROUND = "start_round" CURRENT_ROUND = "current_round" CONTRIBUTION_ROUND = "contribution_round" @@ -52,7 +55,7 @@ def relay_and_wait(self, msg_payload: Dict): pass @abstractmethod - def broadcast(self, msg_payload): + def broadcast(self, msg_payload: Dict): pass @abstractmethod @@ -64,9 +67,35 @@ def relay(self, msg_payload: Dict): pass @abstractmethod - def get_site_names(self): + def get_site_names(self) -> List[str]: pass @abstractmethod - def wait(self, min_responses): + def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: + """ + wait for result + Args: + min_responses: if min_responses or more sites are received, then the result will return + resp_max_wait_time: the max wait time after the 1st site response is received. This is used to deal + with really late site result arrival, instead of waiting forever, we set a timeout. + if resp_max_wait_time is None, it will not timeout + + Returns: + all results with min_response + """ + pass + + @abstractmethod + def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: + """ + wait for result + Args: + resp_max_wait_time: the max wait time after the 1st site response is received. This is used to deal + with really late site result arrival, instead of waiting forever, we set a timeout. + if resp_max_wait_time is None, it will not timeout + + Returns: + Tuple of task_name, site_name, FLModel + """ + pass diff --git a/nvflare/app_common/workflows/wf_comm/wf_queue.py b/nvflare/app_common/workflows/wf_comm/wf_queue.py index ac64365136..dd0fc462ac 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_queue.py +++ b/nvflare/app_common/workflows/wf_comm/wf_queue.py @@ -14,7 +14,7 @@ from queue import Queue -from typing import Optional +from typing import Dict, Optional from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import CMD, CMD_ABORT, CMD_STOP, PAYLOAD @@ -42,13 +42,13 @@ def ctrl_msg_size(self) -> int: def result_size(self) -> int: return self.result_queue.qsize() - def get_ctrl_msg(self): + def get_ctrl_msg(self) -> Dict: item = self.ctrl_queue.get() self.ctrl_queue.task_done() return item - def get_result(self): - item = self.result_queue.get() + def get_result(self, timeout: Optional[float] = None) -> Dict: + item = self.result_queue.get(timeout=timeout) self.result_queue.task_done() return item From 373ce195509d6309ebd17c3d722c7db42b417dc9 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Tue, 2 Jan 2024 10:03:40 -0800 Subject: [PATCH 17/41] update README.md --- .../hello-world/hello-cyclic-pt/README.md | 6 ++++- examples/hello-world/hello-fedavg/README.md | 22 +++++++++++++++---- examples/hello-world/hello-km/README.md | 6 ++++- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/README.md b/examples/hello-world/hello-cyclic-pt/README.md index 5d0b4f3cde..6fb09a080e 100644 --- a/examples/hello-world/hello-cyclic-pt/README.md +++ b/examples/hello-world/hello-cyclic-pt/README.md @@ -29,7 +29,11 @@ class WFCommAPISpec(ABC): pass @abstractmethod - def wait(self, min_responses): + def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: + pass + + @abstractmethod + def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: pass ``` diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md index a7267f2acc..05c4b1952e 100644 --- a/examples/hello-world/hello-fedavg/README.md +++ b/examples/hello-world/hello-fedavg/README.md @@ -7,13 +7,22 @@ This example illustrates How to use the new Workflow Communication API to contr The Flare workflow Communicator API only has small set methods ``` + class WFCommAPISpec(ABC): @abstractmethod def broadcast_and_wait(self, msg_payload: Dict): pass @abstractmethod - def broadcast(self, msg_payload): + def send_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def relay_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def broadcast(self, msg_payload: Dict): pass @abstractmethod @@ -21,16 +30,21 @@ class WFCommAPISpec(ABC): pass @abstractmethod - def send_and_wait(self, msg_payload: Dict): + def relay(self, msg_payload: Dict): + pass + + @abstractmethod + def get_site_names(self) -> List[str]: pass @abstractmethod - def get_site_names(self): + def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: pass @abstractmethod - def wait(self, min_responses): + def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: pass + ``` diff --git a/examples/hello-world/hello-km/README.md b/examples/hello-world/hello-km/README.md index f5582eddbb..d395636b15 100644 --- a/examples/hello-world/hello-km/README.md +++ b/examples/hello-world/hello-km/README.md @@ -31,7 +31,11 @@ class WFCommAPISpec(ABC): pass @abstractmethod - def wait(self, min_responses): + def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: + pass + + @abstractmethod + def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: pass ``` From 6ac08885bc35b8936799b3b20d63f48a30befdf7 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Tue, 2 Jan 2024 18:37:53 -0800 Subject: [PATCH 18/41] add ask all clients to end run when server in exception --- .../common_workflows/base_wf_controller.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index b939856e70..646923eaf0 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -22,7 +22,7 @@ from nvflare.apis.controller_spec import ClientTask, ControllerSpec, OperatorMethod, Task, TaskOperatorKey from nvflare.apis.dxo import DXO, DataKind from nvflare.apis.fl_component import FLComponent -from nvflare.apis.fl_constant import ReturnCode +from nvflare.apis.fl_constant import ReturnCode, ReservedTopic from nvflare.apis.fl_context import FLContext from nvflare.apis.shareable import Shareable from nvflare.apis.signal import Signal @@ -105,6 +105,16 @@ def start_workflow(self, abort_signal, fl_ctx): self.log_error(fl_ctx, error_msg) self.wf_queue.ask_abort(error_msg) self.system_panic(error_msg, fl_ctx=fl_ctx) + # ask all clients to end run! + self.engine.send_aux_request( + targets=None, + topic=ReservedTopic.END_RUN, + request=Shareable(), + timeout=0.0, + fl_ctx=fl_ctx, + optional=True, + secure=False, + ) finally: wait_time = self.comm_msg_pull_interval + 0.05 self.stop_msg_queue("job finished", fl_ctx, wait_time) From c5994c3b3fc493280ff472fd19b93ade910eda34 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 5 Jan 2024 12:33:58 -0800 Subject: [PATCH 19/41] rebase and remove extra command --- .../app_common/common_workflows/base_wf_controller.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 646923eaf0..622be24dcc 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -105,16 +105,6 @@ def start_workflow(self, abort_signal, fl_ctx): self.log_error(fl_ctx, error_msg) self.wf_queue.ask_abort(error_msg) self.system_panic(error_msg, fl_ctx=fl_ctx) - # ask all clients to end run! - self.engine.send_aux_request( - targets=None, - topic=ReservedTopic.END_RUN, - request=Shareable(), - timeout=0.0, - fl_ctx=fl_ctx, - optional=True, - secure=False, - ) finally: wait_time = self.comm_msg_pull_interval + 0.05 self.stop_msg_queue("job finished", fl_ctx, wait_time) From 4b843105457fc68e4773ee0173d385c116b57fff Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 12 Jan 2024 12:17:20 -0800 Subject: [PATCH 20/41] wip --- .../kaplan-meier/app/custom/kaplan_meier.py | 7 +- .../common_workflows/base_wf_controller.py | 5 +- .../app_common/workflows/wf_comm/__init__.py | 10 +++ nvflare/fuel/message/__init__.py | 13 ++++ nvflare/fuel/message/event_manger.py | 21 ++++++ nvflare/fuel/message/message_bus.py | 52 ++++++++++++++ tests/unit_test/fuel/message/__init__.py | 13 ++++ .../fuel/message/message_bus_test.py | 67 +++++++++++++++++++ 8 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 nvflare/fuel/message/__init__.py create mode 100644 nvflare/fuel/message/event_manger.py create mode 100644 nvflare/fuel/message/message_bus.py create mode 100644 tests/unit_test/fuel/message/__init__.py create mode 100644 tests/unit_test/fuel/message/message_bus_test.py diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py index 2d8ad11bc0..961935d5d2 100644 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py @@ -25,20 +25,22 @@ DATA, MIN_RESPONSES, NUM_ROUNDS, - START_ROUND, + START_ROUND, WFCommAPISpec, ) from nvflare.app_common.workflows.wf_comm.wf_spec import WF +from nvflare.app_common.workflows import wf_comm as flare # Controller Workflow -class KM(WF): +class KM: def __init__(self, min_clients: int, output_path: str): super(KM, self).__init__() self.logger = logging.getLogger(self.__class__.__name__) self.output_path = output_path self.min_clients = min_clients self.num_rounds = 1 + self.flare_comm: WFCommAPISpec = flare.get_wf_comm_api() def run(self): results = self.start_km_analysis() @@ -55,7 +57,6 @@ def start_km_analysis(self): START_ROUND: 1, DATA: {}, } - results = self.flare_comm.broadcast_and_wait(msg_payload) return results diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 622be24dcc..88730c4b03 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -49,6 +49,7 @@ ) from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue from nvflare.app_common.workflows.wf_comm.wf_spec import WF +from nvflare.fuel.message.message_bus import MessageBus from nvflare.fuel.utils import class_utils from nvflare.security.logging import secure_format_traceback @@ -73,6 +74,7 @@ def __init__( self.wf_args = wf_args self.wf_queue: WFQueue = WFQueue(ctrl_queue=Queue(), result_queue=Queue()) self._thread_pool_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=self.__class__.__name__) + self.message_bus = MessageBus() self.engine = None self.fl_ctx = None @@ -93,7 +95,8 @@ def setup_wf_queue(self): comm_api.set_queue(self.wf_queue) comm_api.set_result_pull_interval(self.comm_msg_pull_interval) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) - self.wf.setup_wf_comm_api(comm_api) + # self.wf.setup_wf_comm_api(comm_api) + self.message_bus.send_message("wf_comm_api", comm_api) def start_workflow(self, abort_signal, fl_ctx): try: diff --git a/nvflare/app_common/workflows/wf_comm/__init__.py b/nvflare/app_common/workflows/wf_comm/__init__.py index 4fc50543f1..ef067b9723 100644 --- a/nvflare/app_common/workflows/wf_comm/__init__.py +++ b/nvflare/app_common/workflows/wf_comm/__init__.py @@ -11,3 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI +from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import WFCommAPISpec +from nvflare.fuel.message.message_bus import MessageBus + +message_bus = MessageBus() + + +def get_wf_comm_api() -> WFCommAPISpec: + return message_bus.receive_messages("wf_comm_api") \ No newline at end of file diff --git a/nvflare/fuel/message/__init__.py b/nvflare/fuel/message/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/nvflare/fuel/message/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/fuel/message/event_manger.py b/nvflare/fuel/message/event_manger.py new file mode 100644 index 0000000000..8eb3910a7b --- /dev/null +++ b/nvflare/fuel/message/event_manger.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class EventManager: + def __init__(self, message_bus): + self.message_bus = message_bus + + def fire_event(self, event_name, event_data=None): + self.message_bus.publish(event_name, event_data) diff --git a/nvflare/fuel/message/message_bus.py b/nvflare/fuel/message/message_bus.py new file mode 100644 index 0000000000..fc187e9579 --- /dev/null +++ b/nvflare/fuel/message/message_bus.py @@ -0,0 +1,52 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + + +class MessageBus: + _instance = None + _lock = threading.Lock() + + def __new__(cls): + with cls._lock: + if not cls._instance: + cls._instance = super(MessageBus, cls).__new__(cls) + # Initialize the message bus here + return cls._instance + + def __init__(self): + self.subscribers = {} + self.message_store = {} + + def subscribe(self, topic, callback): + if topic not in self.subscribers: + self.subscribers[topic] = [] + self.subscribers[topic].append(callback) + + def publish(self, topic, message): + if topic in self.subscribers: + for callback in self.subscribers[topic]: + callback(message) + + def send_message(self, key, message, topic: str = "default"): + if topic not in self.message_store: + self.message_store[topic] = {} + + self.message_store[topic][key] = message + + self.publish(key, message) # Notify subscribers about the new message + + def receive_messages(self, key, topic: str = "default"): + return self.message_store.get(topic, {}).get(key) diff --git a/tests/unit_test/fuel/message/__init__.py b/tests/unit_test/fuel/message/__init__.py new file mode 100644 index 0000000000..4fc50543f1 --- /dev/null +++ b/tests/unit_test/fuel/message/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit_test/fuel/message/message_bus_test.py b/tests/unit_test/fuel/message/message_bus_test.py new file mode 100644 index 0000000000..3bc0da6671 --- /dev/null +++ b/tests/unit_test/fuel/message/message_bus_test.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from nvflare.fuel.message.event_manger import EventManager +from nvflare.fuel.message.message_bus import MessageBus + + +class TestMessageBus(unittest.TestCase): + def setUp(self): + self.message_bus = MessageBus() + self.event_manager = EventManager(self.message_bus) + + def test_subscribe_and_publish(self): + result = {"count": 0} + + def callback_function(message): + result["count"] += 1 + + self.message_bus.subscribe("test_topic", callback_function) + self.message_bus.publish("test_topic", "Test Message 1") + self.message_bus.publish("test_topic", "Test Message 2") + + self.assertEqual(result["count"], 2) + + def test_send_message_and_receive_messages(self): + self.message_bus.send_message("user_1", "Hello from User 1!") + self.message_bus.send_message("user_2", "Greetings from User 2!") + + user_1_message = self.message_bus.receive_messages("user_1") + user_2_message = self.message_bus.receive_messages("user_2") + + self.assertEqual(user_1_message, "Hello from User 1!") + self.assertEqual(user_2_message, "Greetings from User 2!") + + self.message_bus.send_message("user_1", "2nd greetings from User 1!") + user_1_message = self.message_bus.receive_messages("user_1") + self.assertEqual(user_1_message, "2nd greetings from User 1!") + + self.message_bus.send_message("user_1", "3rd greetings from User 1!", topic="channel-3") + user_1_message = self.message_bus.receive_messages("user_1") + self.assertEqual(user_1_message, "2nd greetings from User 1!") + + user_1_message = self.message_bus.receive_messages("user_1", topic="channel-3") + self.assertEqual(user_1_message, "3rd greetings from User 1!") + + def test_fire_event(self): + result = {"event_received": False} + + def event_handler(data): + result["event_received"] = True + + self.message_bus.subscribe("test_event", event_handler) + self.event_manager.fire_event("test_event", {"key": "value"}) + + self.assertTrue(result["event_received"]) From 4f3abe5d5625c05de7101843a9ad7451b1b84be6 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 12 Jan 2024 16:24:36 -0800 Subject: [PATCH 21/41] remove WF dependency --- .../jobs/cyclic/app/custom/cifar10.py | 2 +- .../jobs/cyclic/app/custom/fed_cyclic.py | 6 +++-- .../fedavg/app/config/config_fed_server.conf | 2 +- .../jobs/fedavg/app/custom/fedavg.py | 6 +++-- .../app/config/config_fed_server.conf | 1 + .../kaplan-meier/app/custom/kaplan_meier.py | 1 - .../common_workflows/base_wf_controller.py | 26 ++++++++++--------- .../workflows/wf_comm/wf_comm_api.py | 22 ++++++++++------ nvflare/app_common/workflows/wf_controller.py | 3 ++- nvflare/app_opt/pt/wf_controller.py | 3 ++- .../fuel/message/message_bus_test.py | 7 +++++ 11 files changed, 50 insertions(+), 29 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py index 33c9272a7a..4a030dd07d 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py @@ -49,7 +49,7 @@ def main(): while flare.is_running(): # (3) receives FLModel from NVFlare input_model = flare.receive() - print(f"current_round={input_model.current_round}") + print(f"current_round={input_model.current_round} at site = {flare.get_site_name()}") # (4) loads model from NVFlare net.load_state_dict(input_model.params) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index 77d876066a..5c9142d75b 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -19,7 +19,7 @@ import torch from net import Net - +from nvflare.app_common.workflows import wf_comm as flare from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( @@ -47,7 +47,7 @@ class RelayOrder: SUPPORTED_ORDERS = (RelayOrder.FIXED, RelayOrder.RANDOM, RelayOrder.RANDOM_WITHOUT_SAME_IN_A_ROW) -class FedCyclic(WF): +class FedCyclic: def __init__( self, output_path: str, @@ -68,6 +68,8 @@ def __init__( self.last_model: Optional[FLModel] = None self.part_sites = None + self.flare_comm = flare.get_wf_comm_api() + self.check_inputs() def run(self): diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf index 65a23423b3..f57d217771 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf @@ -14,7 +14,7 @@ wf_class_path = "fedavg_pt.PTFedAvg", wf_args { min_clients = 2 - num_rounds = 10 + num_rounds = 2 output_path = "/tmp/nvflare/fedavg/mode.pth" stop_cond = "accuracy >= 55" model_selection_rule = "accuracy >=" diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index e112615a76..6ccf74898d 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -30,7 +30,7 @@ RESP_MAX_WAIT_TIME, START_ROUND, ) -from nvflare.app_common.workflows.wf_comm.wf_spec import WF +from nvflare.app_common.workflows import wf_comm as flare from nvflare.security.logging import secure_format_traceback update_model = FLModelUtils.update_model @@ -39,7 +39,7 @@ # FedAvg Workflow -class FedAvg(WF): +class FedAvg: def __init__( self, min_clients: int, @@ -70,6 +70,8 @@ def __init__( else: self.metric_comp_rule = None + self.flare_comm = flare.get_wf_comm_api() + def run(self): self.logger.info("start Fed Avg Workflow\n \n") diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf index 7765396787..8161e93e79 100644 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf @@ -15,6 +15,7 @@ min_clients = 2 output_path = "/tmp/nvflare/km/km.json" } + wf_fn_name = "run", } } ] diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py index 961935d5d2..ae1e1405dd 100644 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py @@ -27,7 +27,6 @@ NUM_ROUNDS, START_ROUND, WFCommAPISpec, ) -from nvflare.app_common.workflows.wf_comm.wf_spec import WF from nvflare.app_common.workflows import wf_comm as flare # Controller Workflow diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 88730c4b03..0743d0dbc5 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import threading import time from abc import ABC from concurrent.futures import ThreadPoolExecutor @@ -60,6 +60,7 @@ def __init__( task_name: str, wf_class_path: str, wf_args: Dict, + wf_fn_name: str = "run", task_timeout: int = 0, comm_msg_pull_interval: float = 0.2, ): @@ -72,8 +73,8 @@ def __init__( self.comm_msg_pull_interval = comm_msg_pull_interval self.wf_class_path = wf_class_path self.wf_args = wf_args + self.wf_fn_name = wf_fn_name self.wf_queue: WFQueue = WFQueue(ctrl_queue=Queue(), result_queue=Queue()) - self._thread_pool_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=self.__class__.__name__) self.message_bus = MessageBus() self.engine = None @@ -81,28 +82,31 @@ def __init__( def start_controller(self, fl_ctx: FLContext): self.fl_ctx = fl_ctx + self.fl_ctx.set_prop("task_name", self.task_name) self.log_info(fl_ctx, "Initializing controller workflow.") self.engine = self.fl_ctx.get_engine() self.clients = self.engine.get_clients() - - self.setup_wf_queue() + self.publish_comm_api() + self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) self.log_info(fl_ctx, "workflow controller started") - def setup_wf_queue(self): - self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) - comm_api = WFCommAPI() + def publish_comm_api(self): + comm_api = WFCommAPI(self) comm_api.set_queue(self.wf_queue) comm_api.set_result_pull_interval(self.comm_msg_pull_interval) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) - # self.wf.setup_wf_comm_api(comm_api) self.message_bus.send_message("wf_comm_api", comm_api) def start_workflow(self, abort_signal, fl_ctx): try: - self._thread_pool_executor.submit(self.ctrl_msg_loop, fl_ctx=fl_ctx, abort_signal=abort_signal) - self.wf.run() + wf_thread = threading.Thread(target= self.ctrl_msg_loop, args = (fl_ctx, abort_signal)) + wf_thread.start() + func = getattr(self.wf, self.wf_fn_name) + func() self.stop_msg_queue("job completed", fl_ctx) + wf_thread.join() + except Exception as e: error_msg = secure_format_traceback() self.log_error(fl_ctx, error_msg) @@ -122,8 +126,6 @@ def stop_msg_queue(self, stop_message, fl_ctx, wait_time: float = 0): def stop_controller(self, fl_ctx: FLContext): self.stop_msg_queue("job completed", fl_ctx) - if self._thread_pool_executor: - self._thread_pool_executor.shutdown() def process_result_of_unknown_task( self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py index d39bc629fc..554ad28926 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py @@ -93,17 +93,23 @@ def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float] = Non self.logger.info( f"\n\n {items_size=}, {acc_size=}, {min_responses=}, {time_waited=}, {resp_max_wait_time=}" ) - if time_waited < resp_max_wait_time and acc_size >= min_responses: - return self._get_results() + if resp_max_wait_time is not None: + if time_waited < resp_max_wait_time and acc_size >= min_responses: + return self._get_results() + else: + if time_waited < resp_max_wait_time: + self.logger.info(f" wait for more results, sleep {self.result_pull_interval} sec") + time.sleep(self.result_pull_interval) + else: + msg = f"not enough responses {acc_size} compare with min responses requirement {min_responses} within the max allowed time {resp_max_wait_time} seconds" + self.logger.info(msg) + raise RuntimeError(msg) else: - if time_waited < resp_max_wait_time: + if acc_size >= min_responses: + return self._get_results() + else: self.logger.info(f" wait for more results, sleep {self.result_pull_interval} sec") time.sleep(self.result_pull_interval) - else: - msg = f"not enough responses {acc_size} compare with min responses requirement {min_responses} within the max allowed time {resp_max_wait_time} seconds" - self.logger.info(msg) - raise RuntimeError(msg) - else: time.sleep(self.result_pull_interval) diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index cb75a8b2f4..34a446fc06 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -26,10 +26,11 @@ def __init__( task_name: str, wf_class_path: str, wf_args: Dict, + wf_fn_name: str = "run", task_timeout: int = 0, comm_msg_pull_interval: float = 0.2, ): - super().__init__(task_name, wf_class_path, wf_args, task_timeout, comm_msg_pull_interval) + super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, comm_msg_pull_interval) def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) diff --git a/nvflare/app_opt/pt/wf_controller.py b/nvflare/app_opt/pt/wf_controller.py index 36663799fb..3e6869391f 100644 --- a/nvflare/app_opt/pt/wf_controller.py +++ b/nvflare/app_opt/pt/wf_controller.py @@ -24,9 +24,10 @@ def __init__( task_name: str, wf_class_path: str, wf_args: Dict, + wf_fn_name: str = "run", task_timeout: int = 0, comm_msg_pull_interval: float = 0.2, ): - super().__init__(task_name, wf_class_path, wf_args, task_timeout, comm_msg_pull_interval) + super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, comm_msg_pull_interval) fobs.register(TensorDecomposer) diff --git a/tests/unit_test/fuel/message/message_bus_test.py b/tests/unit_test/fuel/message/message_bus_test.py index 3bc0da6671..026b71a1e7 100644 --- a/tests/unit_test/fuel/message/message_bus_test.py +++ b/tests/unit_test/fuel/message/message_bus_test.py @@ -55,6 +55,13 @@ def test_send_message_and_receive_messages(self): user_1_message = self.message_bus.receive_messages("user_1", topic="channel-3") self.assertEqual(user_1_message, "3rd greetings from User 1!") + def test_send_message_and_receive_messages_abnormal(self): + user_1_message = self.message_bus.receive_messages("user_1") + self.assertEqual(user_1_message, None) + + user_1_message = self.message_bus.receive_messages("user_1", topic="channel") + self.assertEqual(user_1_message, None) + def test_fire_event(self): result = {"event_received": False} From e635ea45b5fe96b26cd482b6eeabea2da35e865a Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 12 Jan 2024 20:56:32 -0800 Subject: [PATCH 22/41] 1. remove ctrl_msg_Queue, use controller directly. 2. use messagebus to avoid strong coupling --- .../jobs/cyclic/app/custom/fed_cyclic.py | 4 +- .../jobs/fedavg/app/custom/cifar10.py | 1 + .../jobs/fedavg/app/custom/fedavg.py | 2 +- .../jobs/fedavg/app/custom/fedavg_intime.py | 7 +- .../jobs/fedavg/app/custom/fedavg_pt.py | 4 +- .../kaplan-meier/app/custom/kaplan_meier.py | 5 +- .../common_workflows/base_wf_controller.py | 157 ++++++++---------- .../app_common/workflows/wf_comm/__init__.py | 2 +- .../workflows/wf_comm/wf_comm_api.py | 70 ++++---- .../workflows/wf_comm/wf_comm_api_spec.py | 4 +- .../app_common/workflows/wf_comm/wf_queue.py | 20 +-- nvflare/app_common/workflows/wf_controller.py | 7 + 12 files changed, 123 insertions(+), 160 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index 5c9142d75b..199a8ee64b 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -19,9 +19,10 @@ import torch from net import Net -from nvflare.app_common.workflows import wf_comm as flare + from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.workflows import wf_comm as flare from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, @@ -30,7 +31,6 @@ START_ROUND, TARGET_SITES, ) -from nvflare.app_common.workflows.wf_comm.wf_spec import WF update_model = FLModelUtils.update_model diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py index 9e8cfd1c39..274142432f 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py @@ -30,6 +30,7 @@ # (optional) We change to use GPU to speed things up. # if you want to use CPU, change DEVICE="cpu" DEVICE = "cuda:0" +DEVICE = "cpu" def main(): diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index 6ccf74898d..9857479a9b 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -22,6 +22,7 @@ from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.utils.math_utils import parse_compare_criteria, parse_compare_operator +from nvflare.app_common.workflows import wf_comm as flare from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, @@ -30,7 +31,6 @@ RESP_MAX_WAIT_TIME, START_ROUND, ) -from nvflare.app_common.workflows import wf_comm as flare from nvflare.security.logging import secure_format_traceback update_model = FLModelUtils.update_model diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py index 4d3faeae2e..c92fc2c3cd 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py @@ -23,6 +23,7 @@ from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.utils.math_utils import parse_compare_criteria, parse_compare_operator +from nvflare.app_common.workflows import wf_comm as flare from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, @@ -31,7 +32,6 @@ RESP_MAX_WAIT_TIME, START_ROUND, ) -from nvflare.app_common.workflows.wf_comm.wf_spec import WF from nvflare.security.logging import secure_format_traceback update_model = FLModelUtils.update_model @@ -40,7 +40,7 @@ # FedAvg Workflow -class FedAvg(WF): +class FedAvg: def __init__( self, min_clients: int, @@ -71,6 +71,8 @@ def __init__( else: self.metric_comp_rule = None + self.flare_comm = flare.get_wf_comm_api() + def run(self): self.logger.info("start Fed Avg Workflow\n \n") @@ -141,7 +143,6 @@ def gather_and_aggr(self, in_time_aggr_fn: Callable): except RuntimeError as e: self.logger.error(traceback.format_exc()) break - task_name, site_name, model = item aggr_model = in_time_aggr_fn(helpers, aggr_model, site_name, model) start = time.time() if start is None else start diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py index d605c564bd..e8ff96d6d0 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py @@ -14,7 +14,9 @@ import os import torch -from fedavg import FedAvg + +# from fedavg import FedAvg +from fedavg_intime import FedAvg from nvflare.app_common.abstract.fl_model import FLModel diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py index ae1e1405dd..98e1a48935 100644 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py @@ -20,14 +20,15 @@ from km_analysis import kaplan_meier_analysis from nvflare.app_common.abstract.fl_model import FLModel +from nvflare.app_common.workflows import wf_comm as flare from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, MIN_RESPONSES, NUM_ROUNDS, - START_ROUND, WFCommAPISpec, + START_ROUND, + WFCommAPISpec, ) -from nvflare.app_common.workflows import wf_comm as flare # Controller Workflow diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 0743d0dbc5..c5535adc49 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -11,21 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import threading import time from abc import ABC -from concurrent.futures import ThreadPoolExecutor from queue import Queue from typing import Dict, List, Tuple from nvflare.apis.client import Client -from nvflare.apis.controller_spec import ClientTask, ControllerSpec, OperatorMethod, Task, TaskOperatorKey +from nvflare.apis.controller_spec import ClientTask, ControllerSpec, OperatorMethod, SendOrder, Task, TaskOperatorKey from nvflare.apis.dxo import DXO, DataKind from nvflare.apis.fl_component import FLComponent -from nvflare.apis.fl_constant import ReturnCode, ReservedTopic +from nvflare.apis.fl_constant import ReturnCode from nvflare.apis.fl_context import FLContext from nvflare.apis.shareable import Shareable -from nvflare.apis.signal import Signal from nvflare.app_common.abstract.fl_model import FLModel from nvflare.app_common.app_constant import AppConstants from nvflare.app_common.app_event_type import AppEventType @@ -34,11 +31,6 @@ from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CMD, - CMD_ABORT, - CMD_BROADCAST, - CMD_RELAY, - CMD_SEND, - CMD_STOP, DATA, MIN_RESPONSES, PAYLOAD, @@ -74,7 +66,7 @@ def __init__( self.wf_class_path = wf_class_path self.wf_args = wf_args self.wf_fn_name = wf_fn_name - self.wf_queue: WFQueue = WFQueue(ctrl_queue=Queue(), result_queue=Queue()) + self.wf_queue: WFQueue = WFQueue(result_queue=Queue()) self.message_bus = MessageBus() self.engine = None @@ -92,7 +84,7 @@ def start_controller(self, fl_ctx: FLContext): self.log_info(fl_ctx, "workflow controller started") def publish_comm_api(self): - comm_api = WFCommAPI(self) + comm_api = WFCommAPI() comm_api.set_queue(self.wf_queue) comm_api.set_result_pull_interval(self.comm_msg_pull_interval) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) @@ -100,12 +92,10 @@ def publish_comm_api(self): def start_workflow(self, abort_signal, fl_ctx): try: - wf_thread = threading.Thread(target= self.ctrl_msg_loop, args = (fl_ctx, abort_signal)) - wf_thread.start() + fl_ctx.set_prop("abort_signal", abort_signal) func = getattr(self.wf, self.wf_fn_name) func() self.stop_msg_queue("job completed", fl_ctx) - wf_thread.join() except Exception as e: error_msg = secure_format_traceback() @@ -132,85 +122,68 @@ def process_result_of_unknown_task( ): pass - def ctrl_msg_loop(self, fl_ctx: FLContext, abort_signal: Signal): + def broadcast_to_peers_and_wait(self, pay_load): + abort_signal = self.fl_ctx.get_prop("abort_signal") + current_round = self.prepare_round_info(self.fl_ctx, pay_load) + task, min_responses, targets = self.get_payload_task(pay_load) + self.broadcast_and_wait( + task=task, + targets=targets, + min_responses=min_responses, + wait_time_after_min_received=0, + fl_ctx=self.fl_ctx, + abort_signal=abort_signal, + ) + self.fire_event(AppEventType.ROUND_DONE, self.fl_ctx) + self.log_info(self.fl_ctx, f"Round {current_round} finished.") - if self.wf_queue is None: - raise ValueError("WFQueue must provided") + def broadcast_to_peers(self, pay_load): + task, min_responses, targets = self.get_payload_task(pay_load) + self.broadcast( + task=task, fl_ctx=self.fl_ctx, targets=targets, min_responses=min_responses, wait_time_after_min_received=0 + ) - try: - while True: - if abort_signal.triggered: - break - if not self.wf_queue.has_ctrl_msg(): - time.sleep(self.comm_msg_pull_interval) - else: - item = self.wf_queue.get_ctrl_msg() - if item is None: - self.log_warning(fl_ctx, "Ignore 'None' ctrl comm message") - continue - - cmd = item.get(CMD, None) - - if cmd is None: - msg = f"get None command, expecting {CMD} key'" - self.log_error(fl_ctx, msg) - raise ValueError(msg) - - elif cmd == CMD_STOP: - msg = item.get(PAYLOAD) - self.log_info(fl_ctx, f"receive {CMD_STOP} command, {msg}") - break - - elif cmd == CMD_ABORT: - msg = item.get(PAYLOAD) - self.log_info(fl_ctx, f"receive {CMD_ABORT} command, {msg}") - raise RuntimeError(msg) - - elif cmd == CMD_BROADCAST: - pay_load = item.get(PAYLOAD) - - current_round = self.prepare_round_info(fl_ctx, pay_load) - task, min_responses, targets = self.get_payload_task(pay_load) - - self.broadcast_and_wait( - task=task, - targets=targets, - min_responses=min_responses, - wait_time_after_min_received=0, - fl_ctx=fl_ctx, - abort_signal=abort_signal, - ) - self.fire_event(AppEventType.ROUND_DONE, fl_ctx) - self.log_info(fl_ctx, f"Round {current_round} finished.") - - elif cmd == CMD_RELAY: - pay_load = item.get(PAYLOAD) - current_round = self.prepare_round_info(fl_ctx, pay_load) - task, min_responses, targets = self.get_payload_task(pay_load) - - self.relay_and_wait( - task=task, - targets=targets, - fl_ctx=fl_ctx, - abort_signal=abort_signal, - ) - self.fire_event(AppEventType.ROUND_DONE, fl_ctx) - self.log_info(fl_ctx, f"Round {current_round} finished.") - - elif cmd == CMD_SEND: - raise NotImplementedError - else: - abort_signal.trigger(f"Unknown command '{cmd}'") - raise ValueError(f"Unknown command '{cmd}'") - - if abort_signal.triggered: - self.log_debug(self.fl_ctx, f"task {self.task_name} aborted") - break - except Exception as e: - error_msg = secure_format_traceback() - self.wf_queue.ask_abort(error_msg) - self.log_error(fl_ctx, error_msg) - self.system_panic(error_msg, fl_ctx=fl_ctx) + def send_to_peers(self, pay_load, send_order: SendOrder = SendOrder.SEQUENTIAL): + task, _, targets = self.get_payload_task(pay_load) + self.send(task=task, fl_ctx=self.fl_ctx, targets=targets, send_order=send_order, task_assignment_timeout=0) + + def send_to_peers_and_wait(self, pay_load, send_order: SendOrder = SendOrder.SEQUENTIAL): + abort_signal = self.fl_ctx.get_prop("abort_signal") + task, _, targets = self.get_payload_task(pay_load) + self.send_and_wait( + task=task, + fl_ctx=self.fl_ctx, + targets=targets, + send_order=send_order, + task_assignment_timeout=0, + abort_signal=abort_signal, + ) + + def relay_to_peers_and_wait(self, pay_load, send_order: SendOrder = SendOrder.SEQUENTIAL): + abort_signal = self.fl_ctx.get_prop("abort_signal") + task, min_responses, targets = self.get_payload_task(pay_load) + self.relay_and_wait( + task=task, + fl_ctx=self.fl_ctx, + targets=targets, + send_order=send_order, + task_assignment_timeout=0, + task_result_timeout=0, + dynamic_targets=True, + abort_signal=abort_signal, + ) + + def relay_to_peers(self, pay_load, send_order: SendOrder = SendOrder.SEQUENTIAL): + task, min_responses, targets = self.get_payload_task(pay_load) + self.relay( + task=task, + fl_ctx=self.fl_ctx, + targets=targets, + send_order=send_order, + task_assignment_timeout=0, + task_result_timeout=0, + dynamic_targets=True, + ) def prepare_round_info(self, fl_ctx, pay_load): current_round = pay_load.get(AppConstants.CURRENT_ROUND, 0) diff --git a/nvflare/app_common/workflows/wf_comm/__init__.py b/nvflare/app_common/workflows/wf_comm/__init__.py index ef067b9723..68ae3c5743 100644 --- a/nvflare/app_common/workflows/wf_comm/__init__.py +++ b/nvflare/app_common/workflows/wf_comm/__init__.py @@ -20,4 +20,4 @@ def get_wf_comm_api() -> WFCommAPISpec: - return message_bus.receive_messages("wf_comm_api") \ No newline at end of file + return message_bus.receive_messages("wf_comm_api") diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py index 554ad28926..cac07dfa30 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py @@ -18,19 +18,16 @@ from queue import Empty from typing import Dict, Optional, Tuple +from nvflare.apis.controller_spec import SendOrder from nvflare.apis.fl_constant import ReturnCode from nvflare.app_common.abstract.fl_model import FLModel from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CMD, CMD_ABORT, - CMD_BROADCAST, - CMD_RELAY, - CMD_SEND, CMD_STOP, - MIN_RESPONSES, PAYLOAD, - RESP_MAX_WAIT_TIME, RESULT, + SEND_ORDER, SITE_NAMES, STATUS, WFCommAPISpec, @@ -41,6 +38,7 @@ class WFCommAPI(WFCommAPISpec): def __init__(self): self.result_pull_interval = 2 + self.ctrl = None self.wf_queue: Optional[WFQueue] = None self.meta = {SITE_NAMES: []} self.logger = logging.getLogger(self.__class__.__name__) @@ -48,38 +46,49 @@ def __init__(self): def set_result_pull_interval(self, pull_interval: float): self.result_pull_interval = pull_interval + def set_ctrl(self, ctrl): + self.ctrl = ctrl + def set_queue(self, wf_queue: WFQueue): self.wf_queue = wf_queue + def get_site_names(self): + return self.meta.get(SITE_NAMES) + def broadcast_and_wait(self, msg_payload: Dict): - self.broadcast(msg_payload) - min_responses = msg_payload.get(MIN_RESPONSES, 0) - resp_max_wait_time = msg_payload.get(RESP_MAX_WAIT_TIME, 5) - return self.wait_all(min_responses, resp_max_wait_time) + self._check_wf_queue() + self.ctrl.broadcast_to_peers_and_wait(msg_payload) + return self._get_results() def broadcast(self, msg_payload): self._check_wf_queue() - message = { - CMD: CMD_BROADCAST, - PAYLOAD: msg_payload, - } - self.wf_queue.put_ctrl_msg(message) + self.ctrl.broadcast_to_peers(pay_load=msg_payload) def send(self, msg_payload: Dict): self._check_wf_queue() - message = { - CMD: CMD_SEND, - PAYLOAD: msg_payload, - } - self.wf_queue.put_ctrl_msg(message) + send_order_name = msg_payload.get(SEND_ORDER) + send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) + self.ctrl.send_to_peers(pay_load=msg_payload, send_order=send_order) def send_and_wait(self, msg_payload: Dict): - self.send(msg_payload) - min_responses = msg_payload.get(MIN_RESPONSES, 0) - return self.wait_all(min_responses) + self._check_wf_queue() + send_order_name = msg_payload.get(SEND_ORDER) + send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) + self.ctrl.send_to_peers_and_wait(msg_payload, send_order=send_order) + return self._get_results() - def get_site_names(self): - return self.meta.get(SITE_NAMES) + def relay_and_wait(self, msg_payload: Dict): + self._check_wf_queue() + send_order_name = msg_payload.get(SEND_ORDER) + send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) + self.ctrl.relay_to_peers_and_wait(msg_payload, send_order) + return self._get_results() + + def relay(self, msg_payload: Dict): + self._check_wf_queue() + send_order_name = msg_payload.get(SEND_ORDER) + send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) + self.ctrl.relay_to_peers(msg_payload, send_order) def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float] = None) -> Dict[str, Dict[str, FLModel]]: acc_size = 0 @@ -113,19 +122,6 @@ def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float] = Non else: time.sleep(self.result_pull_interval) - def relay_and_wait(self, msg_payload: Dict): - self.relay(msg_payload) - min_responses = msg_payload.get(MIN_RESPONSES, 1) - return self.wait_all(min_responses) - - def relay(self, msg_payload: Dict): - self._check_wf_queue() - message = { - CMD: CMD_RELAY, - PAYLOAD: msg_payload, - } - self.wf_queue.put_ctrl_msg(message) - def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: try: item = self.wf_queue.get_result(resp_max_wait_time) diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py index b152e2681d..83da8314b7 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py @@ -18,12 +18,10 @@ from nvflare.app_common.abstract.fl_model import FLModel CMD = "COMMAND" -CMD_SEND = "SEND" CMD_STOP = "STOP" CMD_ABORT = "ABORT" -CMD_BROADCAST = "BROADCAST" -CMD_RELAY = "RELAY" PAYLOAD = "PAYLOAD" +SEND_ORDER = "SEND_ORDER" SITE_NAMES = "SITE_NAMES" # note same as app_constant constant (todo: we only need one constant definition) diff --git a/nvflare/app_common/workflows/wf_comm/wf_queue.py b/nvflare/app_common/workflows/wf_comm/wf_queue.py index dd0fc462ac..926bcbddeb 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_queue.py +++ b/nvflare/app_common/workflows/wf_comm/wf_queue.py @@ -20,33 +20,18 @@ class WFQueue: - def __init__(self, ctrl_queue: Queue, result_queue: Queue): - self.ctrl_queue = ctrl_queue + def __init__(self, result_queue: Queue): self.result_queue = result_queue - def put_ctrl_msg(self, msg): - self.ctrl_queue.put(msg) - def put_result(self, msg): self.result_queue.put(msg) - def has_ctrl_msg(self) -> bool: - return not self.ctrl_queue.empty() - def has_result(self) -> bool: return not self.result_queue.empty() - def ctrl_msg_size(self) -> int: - return self.ctrl_queue.qsize() - def result_size(self) -> int: return self.result_queue.qsize() - def get_ctrl_msg(self) -> Dict: - item = self.ctrl_queue.get() - self.ctrl_queue.task_done() - return item - def get_result(self, timeout: Optional[float] = None) -> Dict: item = self.result_queue.get(timeout=timeout) self.result_queue.task_done() @@ -54,9 +39,8 @@ def get_result(self, timeout: Optional[float] = None) -> Dict: def stop(self, msg: Optional[str] = None): msg = msg if msg else {} - self.put_ctrl_msg({CMD: CMD_STOP, PAYLOAD: msg}) + self.put_result({CMD: CMD_STOP, PAYLOAD: msg}) def ask_abort(self, msg: Optional[str] = None): msg = msg if msg else {} - self.put_ctrl_msg({CMD: CMD_ABORT, PAYLOAD: msg}) self.put_result({CMD: CMD_ABORT, PAYLOAD: msg}) diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index 34a446fc06..01a9068a60 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -18,6 +18,7 @@ from nvflare.apis.impl.controller import Controller from nvflare.apis.signal import Signal from nvflare.app_common.common_workflows.base_wf_controller import BaseWFController +from nvflare.app_common.workflows.wf_comm import WFCommAPI class WFController(BaseWFController, Controller): @@ -34,3 +35,9 @@ def __init__( def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) + + def publish_comm_api(self): + super(WFController, self).publish_comm_api() + comm_api: WFCommAPI = self.message_bus.receive_messages("wf_comm_api") + comm_api.set_ctrl(self) + self.message_bus.send_message("wf_comm_api", comm_api) From c9ee619b57e8ce75c4443cd326a158429da38c77 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 12 Jan 2024 21:21:03 -0800 Subject: [PATCH 23/41] update README.md and cleanup --- .../hello-world/hello-cyclic-pt/README.md | 36 ++++--------------- examples/hello-world/hello-fedavg/README.md | 26 +++----------- examples/hello-world/hello-km/README.md | 31 ++++------------ .../common_workflows/base_wf_controller.py | 3 +- .../app_common/workflows/wf_comm/wf_spec.py | 30 ---------------- 5 files changed, 18 insertions(+), 108 deletions(-) delete mode 100644 nvflare/app_common/workflows/wf_comm/wf_spec.py diff --git a/examples/hello-world/hello-cyclic-pt/README.md b/examples/hello-world/hello-cyclic-pt/README.md index 6fb09a080e..16f68e18d0 100644 --- a/examples/hello-world/hello-cyclic-pt/README.md +++ b/examples/hello-world/hello-cyclic-pt/README.md @@ -45,8 +45,9 @@ With this new API writing the new workflow is really simple: * Workflow (Server) ``` +from nvflare.app_common.workflows import wf_comm as flare -class FedCyclic(WF): +class FedCyclic: def __init__( self, output_path: str, @@ -56,16 +57,16 @@ class FedCyclic(WF): order: str = RelayOrder.FIXED, ): super(FedCyclic, self).__init__() + <... skip init code ...> - + self.flare_comm = flare.get_wf_comm_api() + + self.check_inputs() - def run(self): self.last_model = self.init_model() - # note: this one must be within run() method, not in the __init__() method - # as some values are injected at runtime during run() self.part_sites = self.flare_comm.get_site_names() if len(self.part_sites) <= 1: @@ -86,8 +87,7 @@ class FedCyclic(WF): gc.collect() self.save_model(self.last_model, self.output_path) - self.logger.info("Cyclic ended.") - + self.logger.info("\n fed cyclic ended \n") ``` Relay_and_wait @@ -107,28 +107,6 @@ Relay_and_wait results = self.flare_comm.relay_and_wait(msg_payload) return results ``` - -The base class ```WF``` is define as - -``` - -class WF(ABC): - - def __init__(self): - self.flare_comm: Optional[WFCommAPI] = None - - def setup_wf_comm_api(self, flare_comm: WFCommAPI): - self.flare_comm = flare_comm - - @abstractmethod - def run(self): - raise NotImplementedError - -``` -has two expectations: -* Make sure user define ```run()``` method -* make sure a class field of WFCommAPI and be able to dynamically populated at runtime -via setup_wf_comm_api() method ## Configurations diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md index 05c4b1952e..76d43f16ed 100644 --- a/examples/hello-world/hello-fedavg/README.md +++ b/examples/hello-world/hello-fedavg/README.md @@ -55,8 +55,9 @@ With this new API writing the new workflow is really simple: * Workflow (Server) ``` +from nvflare.app_common.workflows import wf_comm as flare -class FedAvg(WF): +class FedAvg: def __init__( self, min_clients: int, @@ -69,6 +70,8 @@ class FedAvg(WF): super(FedAvg, self).__init__() + + self.flare_comm = flare.get_wf_comm_api() def run(self): self.logger.info("start Fed Avg Workflow\n \n") @@ -114,28 +117,7 @@ SAG is simply ask WFController to broadcast the model to all clients results = self.flare_comm.broadcast_and_wait(msg_payload) return results ``` -The base class ```WF``` is define as - -``` - -class WF(ABC): - - def __init__(self): - self.flare_comm: Optional[WFCommAPI] = None - def setup_wf_comm_api(self, flare_comm: WFCommAPI): - self.flare_comm = flare_comm - - @abstractmethod - def run(self): - raise NotImplementedError - -``` -has two expectations: -* Make sure user define ```run()``` method -* make sure a class field of WFCommAPI and be able to dynamically populated at runtime - via setup_wf_comm_api() method - ## Configurations ### client-side configuration diff --git a/examples/hello-world/hello-km/README.md b/examples/hello-world/hello-km/README.md index d395636b15..55e5f51aff 100644 --- a/examples/hello-world/hello-km/README.md +++ b/examples/hello-world/hello-km/README.md @@ -48,13 +48,17 @@ For example for Kaplan-Meier Analysis, we could write a new workflow like this: ``` -class KM(WF): +from nvflare.app_common.workflows import wf_comm as flare + +class KM: def __init__(self, min_clients: int, output_path: str): super(KM, self).__init__() self.logger = logging.getLogger(self.__class__.__name__) self.output_path = output_path self.min_clients = min_clients self.num_rounds = 1 + + self.flare_comm = flare.get_wf_comm_api() def run(self): results = self.start_km_analysis() @@ -63,30 +67,7 @@ class KM(WF): ``` -The base class ```WF``` is define as - -``` - -class WF(ABC): - - def __init__(self): - self.flare_comm: Optional[WFCommAPI] = None - - def setup_wf_comm_api(self, flare_comm: WFCommAPI): - self.flare_comm = flare_comm - - @abstractmethod - def run(self): - raise NotImplementedError - -``` -has two expectations: -* Make sure user define ```run()``` method -* make sure a class field of WFCommAPI and be able to dynamically populated at runtime - via setup_wf_comm_api() method - - -for Kaplan-Meier analysis, it literal involves +The Kaplan-Meier analysis involves the following steps * start the analysis --> ask all clients to perform local KM analysis, then wait for results * then aggregate the result to obtain gloabl results diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index c5535adc49..593a9b4ad6 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -40,7 +40,6 @@ TARGET_SITES, ) from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue -from nvflare.app_common.workflows.wf_comm.wf_spec import WF from nvflare.fuel.message.message_bus import MessageBus from nvflare.fuel.utils import class_utils from nvflare.security.logging import secure_format_traceback @@ -79,7 +78,7 @@ def start_controller(self, fl_ctx: FLContext): self.engine = self.fl_ctx.get_engine() self.clients = self.engine.get_clients() self.publish_comm_api() - self.wf: WF = class_utils.instantiate_class(self.wf_class_path, self.wf_args) + self.wf = class_utils.instantiate_class(self.wf_class_path, self.wf_args) self.log_info(fl_ctx, "workflow controller started") diff --git a/nvflare/app_common/workflows/wf_comm/wf_spec.py b/nvflare/app_common/workflows/wf_comm/wf_spec.py deleted file mode 100644 index 22785a7587..0000000000 --- a/nvflare/app_common/workflows/wf_comm/wf_spec.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod -from typing import Optional - -from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI - - -class WF(ABC): - def __init__(self): - self.flare_comm: Optional[WFCommAPI] = None - - def setup_wf_comm_api(self, flare_comm: WFCommAPI): - self.flare_comm = flare_comm - - @abstractmethod - def run(self): - raise NotImplementedError From 76c3c43d982bfe14bd716345c0f323fc565eb343 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 12 Jan 2024 21:49:30 -0800 Subject: [PATCH 24/41] change comm_msg_pull_interval to result_pull_interval --- .../jobs/cyclic/app/config/config_fed_server.conf | 2 +- .../jobs/fedavg/app/config/config_fed_server.conf | 2 +- nvflare/app_common/common_workflows/base_wf_controller.py | 6 +++--- nvflare/app_common/workflows/wf_controller.py | 4 ++-- nvflare/app_opt/pt/wf_controller.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf index 96d938cc57..f45d61a39a 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf @@ -9,7 +9,7 @@ id = "fed_avg" path = "nvflare.app_opt.pt.wf_controller.PTWFController" args { - comm_msg_pull_interval = 5 + result_pull_interval = 5 task_name = "train" wf_class_path = "fed_cyclic.FedCyclic", wf_args { diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf index f57d217771..c065a48921 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf @@ -9,7 +9,7 @@ id = "fed_avg" path = "nvflare.app_opt.pt.wf_controller.PTWFController" args { - comm_msg_pull_interval = 5 + result_pull_interval = 5 task_name = "train" wf_class_path = "fedavg_pt.PTFedAvg", wf_args { diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 593a9b4ad6..5a85bf9c48 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -53,7 +53,7 @@ def __init__( wf_args: Dict, wf_fn_name: str = "run", task_timeout: int = 0, - comm_msg_pull_interval: float = 0.2, + result_pull_interval: float = 0.2, ): super().__init__() @@ -61,7 +61,7 @@ def __init__( self.clients = None self.task_timeout = task_timeout self.task_name = task_name - self.comm_msg_pull_interval = comm_msg_pull_interval + self.result_pull_interval = result_pull_interval self.wf_class_path = wf_class_path self.wf_args = wf_args self.wf_fn_name = wf_fn_name @@ -85,7 +85,7 @@ def start_controller(self, fl_ctx: FLContext): def publish_comm_api(self): comm_api = WFCommAPI() comm_api.set_queue(self.wf_queue) - comm_api.set_result_pull_interval(self.comm_msg_pull_interval) + comm_api.set_result_pull_interval(self.result_pull_interval) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) self.message_bus.send_message("wf_comm_api", comm_api) diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index 01a9068a60..d175aa8095 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -29,9 +29,9 @@ def __init__( wf_args: Dict, wf_fn_name: str = "run", task_timeout: int = 0, - comm_msg_pull_interval: float = 0.2, + result_pull_interval: float = 0.2, ): - super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, comm_msg_pull_interval) + super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, result_pull_interval) def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) diff --git a/nvflare/app_opt/pt/wf_controller.py b/nvflare/app_opt/pt/wf_controller.py index 3e6869391f..9847bdd0fb 100644 --- a/nvflare/app_opt/pt/wf_controller.py +++ b/nvflare/app_opt/pt/wf_controller.py @@ -26,8 +26,8 @@ def __init__( wf_args: Dict, wf_fn_name: str = "run", task_timeout: int = 0, - comm_msg_pull_interval: float = 0.2, + result_pull_interval: float = 0.2, ): - super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, comm_msg_pull_interval) + super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, result_pull_interval) fobs.register(TensorDecomposer) From a52d27abdfa95ef06e797fb03f39ea3335d1e45f Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 13 Jan 2024 10:34:06 -0800 Subject: [PATCH 25/41] 1. fix message_bus 2. continue to loose couple via message_bus --- .../fedavg/app/config/config_fed_server.conf | 1 - .../jobs/fedavg/app/custom/fedavg.py | 20 +++++------- .../jobs/fedavg/app/custom/fedavg_intime.py | 2 +- .../common_workflows/base_wf_controller.py | 11 +++++-- nvflare/app_common/utils/math_utils.py | 31 ------------------- .../workflows/wf_comm/wf_comm_api.py | 24 ++++++-------- nvflare/app_common/workflows/wf_controller.py | 10 ++---- nvflare/fuel/message/message_bus.py | 7 ++--- .../fuel/message/message_bus_test.py | 18 ++++++++--- 9 files changed, 44 insertions(+), 80 deletions(-) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf index c065a48921..6836782112 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf @@ -17,7 +17,6 @@ num_rounds = 2 output_path = "/tmp/nvflare/fedavg/mode.pth" stop_cond = "accuracy >= 55" - model_selection_rule = "accuracy >=" } } } diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index 9857479a9b..b07ce5f010 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -21,7 +21,7 @@ from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.utils.math_utils import parse_compare_criteria, parse_compare_operator +from nvflare.app_common.utils.math_utils import parse_compare_criteria from nvflare.app_common.workflows import wf_comm as flare from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, @@ -47,7 +47,6 @@ def __init__( output_path: str, start_round: int = 1, stop_cond: str = None, - model_selection_rule: str = None, resp_max_wait_time: float = 5, ): super(FedAvg, self).__init__() @@ -65,11 +64,6 @@ def __init__( else: self.stop_criteria = None - if model_selection_rule: - self.metric_comp_rule = parse_compare_operator(model_selection_rule) - else: - self.metric_comp_rule = None - self.flare_comm = flare.get_wf_comm_api() def run(self): @@ -183,12 +177,12 @@ def select_best_model(self, curr_model: FLModel): self.best_model = curr_model return - if self.metric_comp_rule is None: - return - metric, op_fn = self.metric_comp_rule - - self.logger.info("compare models") - if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): + if self.stop_criteria: + metric, _, op_fn = self.stop_criteria + self.logger.info("compare models") + if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): + self.best_model = curr_model + else: self.best_model = curr_model def save_model(self, model: FLModel, file_path: str): diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py index c92fc2c3cd..6ffa56abaa 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py @@ -22,7 +22,7 @@ from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.utils.math_utils import parse_compare_criteria, parse_compare_operator +from nvflare.app_common.utils.math_utils import parse_compare_criteria from nvflare.app_common.workflows import wf_comm as flare from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/common_workflows/base_wf_controller.py index 5a85bf9c48..054993fda5 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/common_workflows/base_wf_controller.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import time -from abc import ABC +from abc import ABC, abstractmethod from queue import Queue from typing import Dict, List, Tuple @@ -67,6 +67,7 @@ def __init__( self.wf_fn_name = wf_fn_name self.wf_queue: WFQueue = WFQueue(result_queue=Queue()) self.message_bus = MessageBus() + self.message_bus.send_message("wf_queue", self.wf_queue) self.engine = None self.fl_ctx = None @@ -82,9 +83,13 @@ def start_controller(self, fl_ctx: FLContext): self.log_info(fl_ctx, "workflow controller started") + @abstractmethod + def publish_controller(self): + pass + def publish_comm_api(self): + self.publish_controller() comm_api = WFCommAPI() - comm_api.set_queue(self.wf_queue) comm_api.set_result_pull_interval(self.result_pull_interval) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) self.message_bus.send_message("wf_comm_api", comm_api) @@ -102,7 +107,7 @@ def start_workflow(self, abort_signal, fl_ctx): self.wf_queue.ask_abort(error_msg) self.system_panic(error_msg, fl_ctx=fl_ctx) finally: - wait_time = self.comm_msg_pull_interval + 0.05 + wait_time = self.result_pull_interval + 0.05 self.stop_msg_queue("job finished", fl_ctx, wait_time) def stop_msg_queue(self, stop_message, fl_ctx, wait_time: float = 0): diff --git a/nvflare/app_common/utils/math_utils.py b/nvflare/app_common/utils/math_utils.py index fc590fcb0c..6f5fb02240 100644 --- a/nvflare/app_common/utils/math_utils.py +++ b/nvflare/app_common/utils/math_utils.py @@ -23,37 +23,6 @@ } -def parse_compare_operator(compare_expr: Optional[str] = None) -> Tuple[str, Callable]: - """ - Parse the compare expression into individual components - compare expression is in the format of string literal : " " - such as - accuracy >= - loss > - - meaning accuracy will be compared use >= operator, loss should use "<" - - Args: - compare_expr: string literal in the format of " " - - Returns: Tuple key, value, operator - - """ - tokens = compare_expr.split(" ") - if len(tokens) != 2: - raise ValueError( - f"Invalid early_stop_condition, expecting form of ' value' but got '{compare_expr}'" - ) - - key = tokens[0] - op = tokens[1] - op_fn = operator_mapping.get(op, None) - if op_fn is None: - raise ValueError("Invalid operator symbol: expecting one of <=, =, >=, <, > ") - - return key, op_fn - - def parse_compare_criteria(compare_expr: Optional[str] = None) -> Tuple[str, float, Callable]: """ Parse the compare expression into individual component diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py index cac07dfa30..4e879668c0 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/workflows/wf_comm/wf_comm_api.py @@ -33,59 +33,51 @@ WFCommAPISpec, ) from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue +from nvflare.fuel.message.message_bus import MessageBus class WFCommAPI(WFCommAPISpec): def __init__(self): self.result_pull_interval = 2 - self.ctrl = None - self.wf_queue: Optional[WFQueue] = None self.meta = {SITE_NAMES: []} self.logger = logging.getLogger(self.__class__.__name__) + message_bus = MessageBus() + self.ctrl = message_bus.receive_messages("controller") + self.wf_queue: Optional[WFQueue] = message_bus.receive_messages("wf_queue") + self._check_inputs() + def set_result_pull_interval(self, pull_interval: float): self.result_pull_interval = pull_interval - def set_ctrl(self, ctrl): - self.ctrl = ctrl - - def set_queue(self, wf_queue: WFQueue): - self.wf_queue = wf_queue - def get_site_names(self): return self.meta.get(SITE_NAMES) def broadcast_and_wait(self, msg_payload: Dict): - self._check_wf_queue() self.ctrl.broadcast_to_peers_and_wait(msg_payload) return self._get_results() def broadcast(self, msg_payload): - self._check_wf_queue() self.ctrl.broadcast_to_peers(pay_load=msg_payload) def send(self, msg_payload: Dict): - self._check_wf_queue() send_order_name = msg_payload.get(SEND_ORDER) send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) self.ctrl.send_to_peers(pay_load=msg_payload, send_order=send_order) def send_and_wait(self, msg_payload: Dict): - self._check_wf_queue() send_order_name = msg_payload.get(SEND_ORDER) send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) self.ctrl.send_to_peers_and_wait(msg_payload, send_order=send_order) return self._get_results() def relay_and_wait(self, msg_payload: Dict): - self._check_wf_queue() send_order_name = msg_payload.get(SEND_ORDER) send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) self.ctrl.relay_to_peers_and_wait(msg_payload, send_order) return self._get_results() def relay(self, msg_payload: Dict): - self._check_wf_queue() send_order_name = msg_payload.get(SEND_ORDER) send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) self.ctrl.relay_to_peers(msg_payload, send_order) @@ -197,6 +189,8 @@ def _check_result(self, site_result): if not all_keys_present: raise RuntimeError(f"expecting all keys {keys} present in site_result") - def _check_wf_queue(self): + def _check_inputs(self): if self.wf_queue is None: raise RuntimeError("missing WFQueue") + if self.ctrl is None: + raise RuntimeError("missing Controller") diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/workflows/wf_controller.py index d175aa8095..a578cf73de 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/workflows/wf_controller.py @@ -18,7 +18,6 @@ from nvflare.apis.impl.controller import Controller from nvflare.apis.signal import Signal from nvflare.app_common.common_workflows.base_wf_controller import BaseWFController -from nvflare.app_common.workflows.wf_comm import WFCommAPI class WFController(BaseWFController, Controller): @@ -33,11 +32,8 @@ def __init__( ): super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, result_pull_interval) + def publish_controller(self): + self.message_bus.send_message("controller", self) + def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) - - def publish_comm_api(self): - super(WFController, self).publish_comm_api() - comm_api: WFCommAPI = self.message_bus.receive_messages("wf_comm_api") - comm_api.set_ctrl(self) - self.message_bus.send_message("wf_comm_api", comm_api) diff --git a/nvflare/fuel/message/message_bus.py b/nvflare/fuel/message/message_bus.py index fc187e9579..f2f90e5ea4 100644 --- a/nvflare/fuel/message/message_bus.py +++ b/nvflare/fuel/message/message_bus.py @@ -23,13 +23,10 @@ def __new__(cls): with cls._lock: if not cls._instance: cls._instance = super(MessageBus, cls).__new__(cls) - # Initialize the message bus here + cls._instance.subscribers = {} + cls._instance.message_store = {} return cls._instance - def __init__(self): - self.subscribers = {} - self.message_store = {} - def subscribe(self, topic, callback): if topic not in self.subscribers: self.subscribers[topic] = [] diff --git a/tests/unit_test/fuel/message/message_bus_test.py b/tests/unit_test/fuel/message/message_bus_test.py index 026b71a1e7..6135a74dfb 100644 --- a/tests/unit_test/fuel/message/message_bus_test.py +++ b/tests/unit_test/fuel/message/message_bus_test.py @@ -34,6 +34,16 @@ def callback_function(message): self.assertEqual(result["count"], 2) + def test_singleton_message_bus(self): + message_bus1 = MessageBus() + message_bus1.send_message("user_1", "Hello from User 1!") + user_1_message = message_bus1.receive_messages("user_1") + self.assertEqual(user_1_message, "Hello from User 1!") + + message_bus2 = MessageBus() + user_1_message = message_bus2.receive_messages("user_1") + self.assertEqual(user_1_message, "Hello from User 1!") + def test_send_message_and_receive_messages(self): self.message_bus.send_message("user_1", "Hello from User 1!") self.message_bus.send_message("user_2", "Greetings from User 2!") @@ -56,11 +66,11 @@ def test_send_message_and_receive_messages(self): self.assertEqual(user_1_message, "3rd greetings from User 1!") def test_send_message_and_receive_messages_abnormal(self): - user_1_message = self.message_bus.receive_messages("user_1") - self.assertEqual(user_1_message, None) + user_3_message = self.message_bus.receive_messages("user_3") + self.assertEqual(user_3_message, None) - user_1_message = self.message_bus.receive_messages("user_1", topic="channel") - self.assertEqual(user_1_message, None) + user_3_message = self.message_bus.receive_messages("user_3", topic="channel") + self.assertEqual(user_3_message, None) def test_fire_event(self): result = {"event_received": False} From 83433924754c826ed6eff17ef2f2b7101114bd09 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Wed, 17 Jan 2024 23:07:16 -0800 Subject: [PATCH 26/41] design change, broken commit --- .../jobs/cyclic/app/custom/fed_cyclic.py | 4 +- .../fedavg/app/config/config_fed_server.conf | 28 +- .../jobs/fedavg/app/custom/fedavg.py | 7 +- .../jobs/fedavg/app/custom/fedavg_intime.py | 248 ------------------ .../jobs/fedavg/app/custom/fedavg_pt.py | 10 +- .../kaplan-meier/app/custom/kaplan_meier.py | 4 +- nvflare/apis/utils/fl_context_utils.py | 2 +- nvflare/app_common/ccwf/server_ctl.py | 2 +- .../wf_comm/__init__.py} | 11 +- .../base_wf_comm.py} | 35 +-- .../{workflows => }/wf_comm/wf_comm_api.py | 10 +- .../wf_comm/wf_comm_api_spec.py | 0 .../wf_communicator.py} | 21 +- .../wf_comm/wf_communicator_spec.py | 38 +++ .../{workflows => }/wf_comm/wf_queue.py | 2 +- .../app_common/workflows/wf_comm/__init__.py | 23 -- nvflare/app_opt/pt/wf_controller.py | 33 --- nvflare/fuel/message/__init__.py | 2 +- .../message/{message_bus.py => data_bus.py} | 29 +- .../message/event_manager.py} | 11 +- nvflare/fuel/utils/component_builder.py | 49 +++- .../private/fed/server/server_json_config.py | 61 ++++- nvflare/private/fed/server/server_runner.py | 1 + 23 files changed, 210 insertions(+), 421 deletions(-) delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py rename nvflare/{fuel/message/event_manger.py => app_common/wf_comm/__init__.py} (72%) rename nvflare/app_common/{common_workflows/base_wf_controller.py => wf_comm/base_wf_comm.py} (92%) rename nvflare/app_common/{workflows => }/wf_comm/wf_comm_api.py (96%) rename nvflare/app_common/{workflows => }/wf_comm/wf_comm_api_spec.py (100%) rename nvflare/app_common/{workflows/wf_controller.py => wf_comm/wf_communicator.py} (61%) create mode 100644 nvflare/app_common/wf_comm/wf_communicator_spec.py rename nvflare/app_common/{workflows => }/wf_comm/wf_queue.py (93%) delete mode 100644 nvflare/app_common/workflows/wf_comm/__init__.py delete mode 100644 nvflare/app_opt/pt/wf_controller.py rename nvflare/fuel/message/{message_bus.py => data_bus.py} (61%) rename nvflare/{app_common/common_workflows/__init__.py => fuel/message/event_manager.py} (62%) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py index 199a8ee64b..a94c37032d 100644 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py @@ -22,8 +22,8 @@ from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.workflows import wf_comm as flare -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( +from nvflare.app_common import wf_comm as flare +from nvflare.app_common.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, MIN_RESPONSES, diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf index 6836782112..291d4ad444 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf @@ -5,23 +5,25 @@ task_result_filters = [] workflows = [ - { + { id = "fed_avg" - path = "nvflare.app_opt.pt.wf_controller.PTWFController" - args { - result_pull_interval = 5 - task_name = "train" - wf_class_path = "fedavg_pt.PTFedAvg", - wf_args { - min_clients = 2 - num_rounds = 2 - output_path = "/tmp/nvflare/fedavg/mode.pth" - stop_cond = "accuracy >= 55" - } + communicator { + path = "nvflare.app_common.wf_comm.wf_communicator.WFCommunicator" + args = {} + } + strategy { + path = "fedavg_pt.PTFedAvg" + args { + min_clients = 2 + num_rounds = 2 + output_path = "/tmp/nvflare/fedavg/mode.pth" + stop_cond = "accuracy >= 55" + } + serializers = ["nvflare.app_opt.pt.decomposers.TensorDecomposer"] + } } ] components = [] - } diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index b07ce5f010..e76cdc0a5f 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -22,8 +22,8 @@ from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.utils.math_utils import parse_compare_criteria -from nvflare.app_common.workflows import wf_comm as flare -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( +from nvflare.app_common import wf_comm as flare +from nvflare.app_common.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, MIN_RESPONSES, @@ -64,10 +64,11 @@ def __init__( else: self.stop_criteria = None - self.flare_comm = flare.get_wf_comm_api() + self.flare_comm = None def run(self): self.logger.info("start Fed Avg Workflow\n \n") + self.flare_comm = flare.get_wf_comm_api() start = self.start_round end = self.start_round + self.num_rounds diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py deleted file mode 100644 index 6ffa56abaa..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_intime.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import time -import traceback -from typing import Callable, Dict, Optional, Tuple - -from net import Net - -from nvflare.app_common.abstract.fl_model import FLModel, ParamsType -from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper -from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.utils.math_utils import parse_compare_criteria -from nvflare.app_common.workflows import wf_comm as flare -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( - CURRENT_ROUND, - DATA, - MIN_RESPONSES, - NUM_ROUNDS, - RESP_MAX_WAIT_TIME, - START_ROUND, -) -from nvflare.security.logging import secure_format_traceback - -update_model = FLModelUtils.update_model - - -# FedAvg Workflow - - -class FedAvg: - def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - model_selection_rule: str = None, - resp_max_wait_time: float = 5.0, - ): - super(FedAvg, self).__init__() - self.logger = logging.getLogger(self.__class__.__name__) - - self.output_path = output_path - self.min_clients = min_clients - self.num_rounds = num_rounds - self.start_round = start_round - self.current_round = start_round - self.resp_max_wait_time = resp_max_wait_time - self.best_model: Optional[FLModel] = None - if stop_cond: - self.stop_criteria = parse_compare_criteria(stop_cond) - else: - self.stop_criteria = None - - if model_selection_rule: - self.metric_comp_rule = parse_compare_operator(model_selection_rule) - else: - self.metric_comp_rule = None - - self.flare_comm = flare.get_wf_comm_api() - - def run(self): - self.logger.info("start Fed Avg Workflow\n \n") - - start = self.start_round - end = self.start_round + self.num_rounds - - model = self.init_model() - for current_round in range(start, end): - - self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") - self.current_round = current_round - - if self.should_stop(model.metrics, self.stop_criteria): - self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") - break - - self.scatter(model, current_round) - - self.logger.info("gather and aggregate") - aggr_result = self.gather_and_aggr(self.in_time_aggr_fn) - - self.logger.info(f"aggregate metrics = {aggr_result.metrics}") - - model = update_model(model, aggr_result) - - self.select_best_model(model) - - self.save_model(self.best_model, self.output_path) - - self.logger.info("end Fed Avg Workflow\n \n") - - def init_model(self): - net = Net() - model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) - return model - - def scatter(self, model: FLModel, current_round): - msg_payload = { - MIN_RESPONSES: self.min_clients, - RESP_MAX_WAIT_TIME: self.resp_max_wait_time, - CURRENT_ROUND: current_round, - NUM_ROUNDS: self.num_rounds, - START_ROUND: self.start_round, - DATA: model, - } - - # (2) broadcast and wait - self.flare_comm.broadcast(msg_payload) - - def gather_and_aggr(self, in_time_aggr_fn: Callable): - responses = 0 - start = None - aggr_model = None - aggr_params_helper = WeightedAggregationHelper() - aggr_metrics_helper = WeightedAggregationHelper() - helpers = (aggr_params_helper, aggr_metrics_helper) - - while True: - if responses >= self.min_clients: - return aggr_model - else: - time.sleep(0.2) - - max_timeout = self.resp_max_wait_time if start else None - self.logger.warning(f"{max_timeout=}, {responses=}, {self.min_clients=}") - try: - item = self.flare_comm.wait_one(max_timeout) - except RuntimeError as e: - self.logger.error(traceback.format_exc()) - break - task_name, site_name, model = item - aggr_model = in_time_aggr_fn(helpers, aggr_model, site_name, model) - start = time.time() if start is None else start - responses += 1 - - if responses < self.min_clients: - raise RuntimeError( - f"not enough responses {responses} compare with min responses requirement {self.min_clients} within the max allowed time {self.resp_max_wait_time} seconds" - ) - else: - return aggr_model - - def in_time_aggr_fn(self, helpers: Tuple, prev_mode: FLModel, site_name: str, fl_model: FLModel) -> FLModel: - - self.logger.info("Fed Avg in time aggregate \n") - if not fl_model: - raise RuntimeError("model must not be None") - - self.logger.info(f"aggregating update at round {self.current_round}") - - self.logger.info(f"site={site_name} {fl_model.metrics=}") - - if prev_mode is None: - self.logger.info(f"aggr_metrics={fl_model.metrics}") - return fl_model - - try: - params_type = fl_model.params_type - aggr_params_helper, aggr_metrics_helper = helpers - - aggr_params_helper.add( - data=fl_model.params, - weight=self.current_round, - contributor_name=site_name, - contribution_round=self.current_round, - ) - - aggr_metrics_helper.add( - data=fl_model.metrics, - weight=self.current_round, - contributor_name=site_name, - contribution_round=self.current_round, - ) - - aggr_params = aggr_params_helper.get_result() - aggr_metrics = aggr_metrics_helper.get_result() - - self.logger.info(f"{aggr_metrics=}") - - aggr_result = FLModel( - params=aggr_params, - params_type=params_type, - metrics=aggr_metrics, - meta={ - "num_rounds_aggregated": 1 + (self.current_round - self.start_round), - "current_round": self.current_round, - }, - ) - return aggr_result - - except Exception as e: - raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") - - def select_best_model(self, curr_model: FLModel): - if self.best_model is None: - self.best_model = curr_model - return - - if self.metric_comp_rule is None: - return - metric, op_fn = self.metric_comp_rule - - self.logger.info("compare models") - if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): - self.best_model = curr_model - - def save_model(self, model: FLModel, file_path: str): - pass - - def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[str] = None): - self.logger.info(f"stop_criteria, metrics = {stop_criteria=}, {metrics=}") - if stop_criteria is None or metrics is None: - return False - - key, target, op_fn = stop_criteria - value = metrics.get(key, None) - - if value is None: - raise RuntimeError(f"stop criteria key '{key}' doesn't exists in metrics") - - return op_fn(value, target) - - def is_curr_mode_better( - self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable - ) -> bool: - curr_metrics = curr_model.metrics - if curr_metrics is None: - return False - if target_metric not in curr_metrics: - return False - - best_metrics = best_model.metrics - return op_fn(curr_metrics.get(target_metric), best_metrics.get(target_metric)) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py index e8ff96d6d0..33ff098a75 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py @@ -15,14 +15,9 @@ import torch -# from fedavg import FedAvg -from fedavg_intime import FedAvg - +from fedavg import FedAvg from nvflare.app_common.abstract.fl_model import FLModel -# to use in_time aggregate version of FedAvg -# you change the import to 'from fedavg_intime import FedAvg' - class PTFedAvg(FedAvg): def __init__( @@ -32,9 +27,8 @@ def __init__( output_path: str, start_round: int = 1, stop_cond: str = None, - model_selection_rule: str = None, ): - super().__init__(min_clients, num_rounds, output_path, start_round, stop_cond, model_selection_rule) + super().__init__(min_clients, num_rounds, output_path, start_round, stop_cond) def save_model(self, model: FLModel, file_path: str): if not file_path: diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py index 98e1a48935..ca8d592d47 100644 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py +++ b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py @@ -20,8 +20,8 @@ from km_analysis import kaplan_meier_analysis from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.workflows import wf_comm as flare -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( +from nvflare.app_common import wf_comm as flare +from nvflare.app_common.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, MIN_RESPONSES, diff --git a/nvflare/apis/utils/fl_context_utils.py b/nvflare/apis/utils/fl_context_utils.py index 819e69186b..6882432de9 100644 --- a/nvflare/apis/utils/fl_context_utils.py +++ b/nvflare/apis/utils/fl_context_utils.py @@ -60,7 +60,7 @@ def generate_log_message(fl_ctx: FLContext, msg: str): _task_name = "task_name" _task_id = "task_id" _rc = "peer_rc" - _wf = "wf" + _wf = "strategy" all_kvs = {_identity_: fl_ctx.get_identity_name()} my_run = fl_ctx.get_job_id() diff --git a/nvflare/app_common/ccwf/server_ctl.py b/nvflare/app_common/ccwf/server_ctl.py index 5ec2e868a1..1167df8a8a 100644 --- a/nvflare/app_common/ccwf/server_ctl.py +++ b/nvflare/app_common/ccwf/server_ctl.py @@ -57,7 +57,7 @@ def __init__( self, num_rounds: int, start_round: int = 0, - task_name_prefix: str = "wf", + task_name_prefix: str = "strategy", configure_task_timeout=Constant.CONFIG_TASK_TIMEOUT, end_workflow_timeout=Constant.END_WORKFLOW_TIMEOUT, start_task_timeout=Constant.START_TASK_TIMEOUT, diff --git a/nvflare/fuel/message/event_manger.py b/nvflare/app_common/wf_comm/__init__.py similarity index 72% rename from nvflare/fuel/message/event_manger.py rename to nvflare/app_common/wf_comm/__init__.py index 8eb3910a7b..ae5c6b2d3c 100644 --- a/nvflare/fuel/message/event_manger.py +++ b/nvflare/app_common/wf_comm/__init__.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.app_common.wf_comm.wf_comm_api_spec import WFCommAPISpec +from nvflare.fuel.message.data_bus import DataBus -class EventManager: - def __init__(self, message_bus): - self.message_bus = message_bus +data_bus = DataBus() - def fire_event(self, event_name, event_data=None): - self.message_bus.publish(event_name, event_data) + +def get_wf_comm_api() -> WFCommAPISpec: + return data_bus.receive_messages("wf_comm_api") \ No newline at end of file diff --git a/nvflare/app_common/common_workflows/base_wf_controller.py b/nvflare/app_common/wf_comm/base_wf_comm.py similarity index 92% rename from nvflare/app_common/common_workflows/base_wf_controller.py rename to nvflare/app_common/wf_comm/base_wf_comm.py index 054993fda5..bb314818ff 100644 --- a/nvflare/app_common/common_workflows/base_wf_controller.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import time -from abc import ABC, abstractmethod +from abc import ABC from queue import Queue from typing import Dict, List, Tuple @@ -27,9 +27,8 @@ from nvflare.app_common.app_constant import AppConstants from nvflare.app_common.app_event_type import AppEventType from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.workflows.error_handle_utils import ABORT_WHEN_IN_ERROR -from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( +from nvflare.app_common.wf_comm.wf_comm_api import WFCommAPI +from nvflare.app_common.wf_comm.wf_comm_api_spec import ( CMD, DATA, MIN_RESPONSES, @@ -39,34 +38,29 @@ STATUS, TARGET_SITES, ) -from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue -from nvflare.fuel.message.message_bus import MessageBus -from nvflare.fuel.utils import class_utils +from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec +from nvflare.app_common.wf_comm.wf_queue import WFQueue +from nvflare.app_common.workflows.error_handle_utils import ABORT_WHEN_IN_ERROR +from nvflare.fuel.message.data_bus import DataBus from nvflare.security.logging import secure_format_traceback -class BaseWFController(FLComponent, ControllerSpec, ABC): +class BaseWFCommunicator(FLComponent, WFCommunicatorSpec, ControllerSpec, ABC): def __init__( self, task_name: str, - wf_class_path: str, - wf_args: Dict, - wf_fn_name: str = "run", task_timeout: int = 0, result_pull_interval: float = 0.2, ): super().__init__() - self.wf = None + self.strategy_fn_name = "run" self.clients = None self.task_timeout = task_timeout self.task_name = task_name self.result_pull_interval = result_pull_interval - self.wf_class_path = wf_class_path - self.wf_args = wf_args - self.wf_fn_name = wf_fn_name self.wf_queue: WFQueue = WFQueue(result_queue=Queue()) - self.message_bus = MessageBus() + self.message_bus = DataBus() self.message_bus.send_message("wf_queue", self.wf_queue) self.engine = None @@ -79,16 +73,9 @@ def start_controller(self, fl_ctx: FLContext): self.engine = self.fl_ctx.get_engine() self.clients = self.engine.get_clients() self.publish_comm_api() - self.wf = class_utils.instantiate_class(self.wf_class_path, self.wf_args) - self.log_info(fl_ctx, "workflow controller started") - @abstractmethod - def publish_controller(self): - pass - def publish_comm_api(self): - self.publish_controller() comm_api = WFCommAPI() comm_api.set_result_pull_interval(self.result_pull_interval) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) @@ -97,7 +84,7 @@ def publish_comm_api(self): def start_workflow(self, abort_signal, fl_ctx): try: fl_ctx.set_prop("abort_signal", abort_signal) - func = getattr(self.wf, self.wf_fn_name) + func = getattr(self.get_strategy(), self.strategy_fn_name) func() self.stop_msg_queue("job completed", fl_ctx) diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py similarity index 96% rename from nvflare/app_common/workflows/wf_comm/wf_comm_api.py rename to nvflare/app_common/wf_comm/wf_comm_api.py index 4e879668c0..d666397252 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -21,7 +21,7 @@ from nvflare.apis.controller_spec import SendOrder from nvflare.apis.fl_constant import ReturnCode from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import ( +from nvflare.app_common.wf_comm.wf_comm_api_spec import ( CMD, CMD_ABORT, CMD_STOP, @@ -32,8 +32,8 @@ STATUS, WFCommAPISpec, ) -from nvflare.app_common.workflows.wf_comm.wf_queue import WFQueue -from nvflare.fuel.message.message_bus import MessageBus +from nvflare.app_common.wf_comm.wf_queue import WFQueue +from nvflare.fuel.message.data_bus import DataBus class WFCommAPI(WFCommAPISpec): @@ -42,8 +42,8 @@ def __init__(self): self.meta = {SITE_NAMES: []} self.logger = logging.getLogger(self.__class__.__name__) - message_bus = MessageBus() - self.ctrl = message_bus.receive_messages("controller") + message_bus = DataBus() + self.ctrl = message_bus.receive_messages("communicator") self.wf_queue: Optional[WFQueue] = message_bus.receive_messages("wf_queue") self._check_inputs() diff --git a/nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/wf_comm/wf_comm_api_spec.py similarity index 100% rename from nvflare/app_common/workflows/wf_comm/wf_comm_api_spec.py rename to nvflare/app_common/wf_comm/wf_comm_api_spec.py diff --git a/nvflare/app_common/workflows/wf_controller.py b/nvflare/app_common/wf_comm/wf_communicator.py similarity index 61% rename from nvflare/app_common/workflows/wf_controller.py rename to nvflare/app_common/wf_comm/wf_communicator.py index a578cf73de..cdd1f3abe5 100644 --- a/nvflare/app_common/workflows/wf_controller.py +++ b/nvflare/app_common/wf_comm/wf_communicator.py @@ -12,28 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict - from nvflare.apis.fl_context import FLContext from nvflare.apis.impl.controller import Controller from nvflare.apis.signal import Signal -from nvflare.app_common.common_workflows.base_wf_controller import BaseWFController +from nvflare.app_common.wf_comm.base_wf_comm import BaseWFCommunicator -class WFController(BaseWFController, Controller): +class WFCommunicator(BaseWFCommunicator, Controller): def __init__( - self, - task_name: str, - wf_class_path: str, - wf_args: Dict, - wf_fn_name: str = "run", - task_timeout: int = 0, - result_pull_interval: float = 0.2, + self, + task_timeout: int = 0, + result_pull_interval: float = 0.2, ): - super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, result_pull_interval) - - def publish_controller(self): - self.message_bus.send_message("controller", self) + super().__init__(task_name="train") def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) diff --git a/nvflare/app_common/wf_comm/wf_communicator_spec.py b/nvflare/app_common/wf_comm/wf_communicator_spec.py new file mode 100644 index 0000000000..01a6b0607b --- /dev/null +++ b/nvflare/app_common/wf_comm/wf_communicator_spec.py @@ -0,0 +1,38 @@ +from abc import ABC +from typing import List, Dict, Optional + +from nvflare.fuel.utils.class_utils import instantiate_class +from nvflare.fuel.utils.component_builder import ComponentBuilder +from nvflare.fuel.utils.fobs import fobs + + +class WFCommunicatorSpec(ABC): + + def __init__(self): + self.strategy = None + self.strategy_config: Optional[Dict] = None + + def set_strategy_config(self, strategy_config): + if strategy_config is None: + raise ValueError("strategy_config is None") + + if not isinstance(strategy_config, dict): + raise ValueError(f"strategy_config should be Dict, found '{type(strategy_config)}'") + + self.strategy_config = strategy_config + + def get_strategy(self): + # if self.strategy is None and isinstance(self.strategy_config, dict): + print(f"{self.strategy_config=}") + if isinstance(self.strategy_config, dict): + strategy = ComponentBuilder().build_component(self.strategy_config) + if strategy is None: + raise ValueError("strategy should provided, but get None") + self.strategy = strategy + + return self.strategy + + def set_serializers(self, serializer_class_paths: List[str] = None): + if serializer_class_paths: + for class_path in serializer_class_paths: + fobs.register(instantiate_class(class_path, {})) diff --git a/nvflare/app_common/workflows/wf_comm/wf_queue.py b/nvflare/app_common/wf_comm/wf_queue.py similarity index 93% rename from nvflare/app_common/workflows/wf_comm/wf_queue.py rename to nvflare/app_common/wf_comm/wf_queue.py index 926bcbddeb..6638d5373c 100644 --- a/nvflare/app_common/workflows/wf_comm/wf_queue.py +++ b/nvflare/app_common/wf_comm/wf_queue.py @@ -16,7 +16,7 @@ from queue import Queue from typing import Dict, Optional -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import CMD, CMD_ABORT, CMD_STOP, PAYLOAD +from nvflare.app_common.wf_comm.wf_comm_api_spec import CMD, CMD_ABORT, CMD_STOP, PAYLOAD class WFQueue: diff --git a/nvflare/app_common/workflows/wf_comm/__init__.py b/nvflare/app_common/workflows/wf_comm/__init__.py deleted file mode 100644 index 68ae3c5743..0000000000 --- a/nvflare/app_common/workflows/wf_comm/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from nvflare.app_common.workflows.wf_comm.wf_comm_api import WFCommAPI -from nvflare.app_common.workflows.wf_comm.wf_comm_api_spec import WFCommAPISpec -from nvflare.fuel.message.message_bus import MessageBus - -message_bus = MessageBus() - - -def get_wf_comm_api() -> WFCommAPISpec: - return message_bus.receive_messages("wf_comm_api") diff --git a/nvflare/app_opt/pt/wf_controller.py b/nvflare/app_opt/pt/wf_controller.py deleted file mode 100644 index 9847bdd0fb..0000000000 --- a/nvflare/app_opt/pt/wf_controller.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Dict - -from nvflare.app_common.workflows.wf_controller import WFController -from nvflare.app_opt.pt.decomposers import TensorDecomposer -from nvflare.fuel.utils.fobs import fobs - - -class PTWFController(WFController): - def __init__( - self, - task_name: str, - wf_class_path: str, - wf_args: Dict, - wf_fn_name: str = "run", - task_timeout: int = 0, - result_pull_interval: float = 0.2, - ): - super().__init__(task_name, wf_class_path, wf_args, wf_fn_name, task_timeout, result_pull_interval) - - fobs.register(TensorDecomposer) diff --git a/nvflare/fuel/message/__init__.py b/nvflare/fuel/message/__init__.py index bc443be41c..d9155f923f 100644 --- a/nvflare/fuel/message/__init__.py +++ b/nvflare/fuel/message/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nvflare/fuel/message/message_bus.py b/nvflare/fuel/message/data_bus.py similarity index 61% rename from nvflare/fuel/message/message_bus.py rename to nvflare/fuel/message/data_bus.py index f2f90e5ea4..b1bb265622 100644 --- a/nvflare/fuel/message/message_bus.py +++ b/nvflare/fuel/message/data_bus.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,29 +13,34 @@ # limitations under the License. import threading +from typing import Callable, List -class MessageBus: +class DataBus: _instance = None _lock = threading.Lock() def __new__(cls): with cls._lock: if not cls._instance: - cls._instance = super(MessageBus, cls).__new__(cls) + cls._instance = super(DataBus, cls).__new__(cls) cls._instance.subscribers = {} cls._instance.message_store = {} return cls._instance - def subscribe(self, topic, callback): - if topic not in self.subscribers: - self.subscribers[topic] = [] - self.subscribers[topic].append(callback) + def subscribe(self, topics: List[str], callback: Callable): + if topics: + for topic in topics: + if topic not in self.subscribers: + self.subscribers[topic] = [] + self.subscribers[topic].append(callback) - def publish(self, topic, message): - if topic in self.subscribers: - for callback in self.subscribers[topic]: - callback(message) + def publish(self, topics: List[str], message: any): + if topics: + for topic in topics: + if topic in self.subscribers: + for callback in self.subscribers[topic]: + callback(message, topic) def send_message(self, key, message, topic: str = "default"): if topic not in self.message_store: @@ -43,7 +48,5 @@ def send_message(self, key, message, topic: str = "default"): self.message_store[topic][key] = message - self.publish(key, message) # Notify subscribers about the new message - def receive_messages(self, key, topic: str = "default"): return self.message_store.get(topic, {}).get(key) diff --git a/nvflare/app_common/common_workflows/__init__.py b/nvflare/fuel/message/event_manager.py similarity index 62% rename from nvflare/app_common/common_workflows/__init__.py rename to nvflare/fuel/message/event_manager.py index 4fc50543f1..7b3dbd2623 100644 --- a/nvflare/app_common/common_workflows/__init__.py +++ b/nvflare/fuel/message/event_manager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fuel.message.data_bus import DataBus + + +class EventManager: + def __init__(self, data_bus: DataBus): + self.data_bus = data_bus + + def fire_event(self, event_name, event_data=None): + self.data_bus.publish([event_name], event_data) diff --git a/nvflare/fuel/utils/component_builder.py b/nvflare/fuel/utils/component_builder.py index ca6fc48847..ac39a71ad6 100644 --- a/nvflare/fuel/utils/component_builder.py +++ b/nvflare/fuel/utils/component_builder.py @@ -67,23 +67,46 @@ def build_component(self, config_dict): return None class_args = config_dict.get("args", dict()) - for k, v in class_args.items(): - if isinstance(v, dict) and self.is_class_config(v): - # try to replace the arg with a component - try: - t = self.build_component(v) - class_args[k] = t - except Exception as e: - raise ValueError(f"failed to instantiate class: {secure_format_exception(e)} ") - - class_path = self.get_class_path(config_dict) + lazy_instantiate = config_dict.get("lazy_instantiate", False) + if not lazy_instantiate: + for k, v in class_args.items(): + if isinstance(v, dict) and self.is_class_config(v): + # try to replace the arg with a component + try: + t = self.build_component(v) + class_args[k] = t + except Exception as e: + raise ValueError(f"failed to instantiate class: {secure_format_exception(e)} ") + + class_path = None + if self.is_class_config(config_dict): + class_path = self.get_class_path(config_dict) # Handle the special case, if config pass in the class_attributes, use the user defined class attributes # parameters directly. - if "class_attributes" in class_args: - class_args = class_args["class_attributes"] + if class_path and not lazy_instantiate: + if "class_attributes" in class_args: + class_args = class_args["class_attributes"] - return instantiate_class(class_path, class_args) + return instantiate_class(class_path, class_args) + else: + comp_dict = {} + lazy_instantiate = config_dict.get("lazy_instantiate", False) + if not lazy_instantiate: + for k, v in config_dict.items(): + if isinstance(v, dict) and self.is_class_config(v): + # try to replace the arg with a component + try: + t = self.build_component(v) + comp_dict[k] = t + except Exception as e: + raise ValueError(f"failed to instantiate class: {secure_format_exception(e)} ") + else: + comp_dict[k] = v + else: + comp_dict = config_dict + + return comp_dict def get_class_path(self, config_dict): if "path" in config_dict.keys(): diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index 5685a2e456..ab32d64884 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -17,12 +17,15 @@ from nvflare.apis.fl_component import FLComponent from nvflare.apis.fl_constant import SystemConfigs, SystemVarName from nvflare.apis.responder import Responder +from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec +from nvflare.app_common.wf_comm.wf_communicator import WFCommunicator +from nvflare.fuel.message.data_bus import DataBus from nvflare.fuel.utils.argument_utils import parse_vars +from nvflare.fuel.utils.component_builder import ComponentBuilder from nvflare.fuel.utils.config_service import ConfigService from nvflare.fuel.utils.json_scanner import Node from nvflare.private.fed_json_config import FedJsonConfigurator from nvflare.private.json_configer import ConfigContext, ConfigError - from .server_runner import ServerRunnerConfig FL_PACKAGES = ["nvflare"] @@ -30,15 +33,25 @@ class WorkFlow: - def __init__(self, id, responder: Responder): + def __init__(self, id, responder: Responder, strategy=None): """Workflow is a responder with ID. Args: id: identification - responder (Responder): A responder + responder (Responder): A responder or communicator + strategy: federated learning strategy. If None, the responder will implement the strategy """ self.id = id self.responder = responder + self.strategy = strategy + + +def enhance_workflow_config(element: dict): + if "strategy" in element: + strategy_config = element.get("strategy") + strategy_config["lazy_instantiate"] = True + element["strategy"] = strategy_config + return element class ServerJsonConfigurator(FedJsonConfigurator): @@ -124,15 +137,31 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): return if re.search(r"^workflows\.#[0-9]+$", path): - workflow = self.authorize_and_build_component(element, config_ctx, node) - if not isinstance(workflow, Responder): + element = enhance_workflow_config(element) + component = self.authorize_and_build_component(element, config_ctx, node) + if isinstance(component, dict): + wf_config = component + communicator = wf_config.get("communicator", WFCommunicator()) + if isinstance(communicator, WFCommunicatorSpec): + strategy_config = wf_config.get("strategy") + strategy_config["lazy_instantiate"] = False + communicator.set_strategy_config(strategy_config) + communicator.set_serializers(strategy_config.get("serializers")) + data_bus = DataBus() + data_bus.send_message("communicator", communicator) + responder = communicator + else: + responder = component + + if not isinstance(responder, Responder): raise ConfigError( - '"workflow" must be a Responder or Controller object, but got {}'.format(type(workflow)) + '"workflow" must be a Responder or Controller or has a Responder object, but got {}'.format( + type(responder)) ) cid = element.get("id", None) if not cid: - cid = type(workflow).__name__ + cid = type(responder).__name__ if not isinstance(cid, str): raise ConfigError('"id" must be str but got {}'.format(type(cid))) @@ -143,10 +172,24 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): if cid in self.components: raise ConfigError('duplicate component id "{}"'.format(cid)) - self.workflows.append(WorkFlow(cid, workflow)) - self.components[cid] = workflow + workflow = WorkFlow(cid, responder) + self.workflows.append(workflow) + self.components[cid] = responder return + def get_strategy(self, wf_config): + strategy_comp = wf_config.get("strategy") + strategy_comp["lazy_instantiate"] = False + if isinstance(strategy_comp, dict): + strategy = ComponentBuilder().build_component(strategy_comp) + else: + strategy = strategy_comp + + if strategy is None: + raise ValueError("strategy should provided, but get None") + + return strategy + def _get_all_workflows_ids(self): ids = [] for t in self.workflows: diff --git a/nvflare/private/fed/server/server_runner.py b/nvflare/private/fed/server/server_runner.py index 75bb9b3d7e..c9dd76a3a1 100644 --- a/nvflare/private/fed/server/server_runner.py +++ b/nvflare/private/fed/server/server_runner.py @@ -125,6 +125,7 @@ def _execute_run(self): self.log_info(fl_ctx, "starting workflow {} ({}) ...".format(wf.id, type(wf.responder))) fl_ctx.set_prop(FLContextKey.WORKFLOW, wf.id, sticky=True) + wf.responder.initialize_run(fl_ctx) self.log_info(fl_ctx, "Workflow {} ({}) started".format(wf.id, type(wf.responder))) From d91459244f9043402de8ac4855bd5bfb88f6d55c Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 19 Jan 2024 21:25:16 -0800 Subject: [PATCH 27/41] everything works now --- .../hello-world/hello-cyclic-pt/README.md | 159 ---------- .../cyclic/app/config/config_fed_client.conf | 90 ------ .../cyclic/app/config/config_fed_server.conf | 25 -- .../jobs/cyclic/app/custom/cifar10.py | 130 -------- .../jobs/cyclic/app/custom/fed_cyclic.py | 149 --------- .../jobs/cyclic/app/custom/net.py | 37 --- .../hello-cyclic-pt/jobs/cyclic/meta.conf | 7 - .../hello-cyclic-pt/requirements.txt | 0 examples/hello-world/hello-fedavg/README.md | 172 ----------- .../fedavg/app/config/config_fed_client.conf | 116 ------- .../fedavg/app/config/config_fed_server.conf | 29 -- .../jobs/fedavg/app/custom/cifar10.py | 138 --------- .../jobs/fedavg/app/custom/fedavg.py | 215 ------------- .../jobs/fedavg/app/custom/fedavg_pt.py | 41 --- .../jobs/fedavg/app/custom/net.py | 37 --- .../hello-fedavg/jobs/fedavg/meta.conf | 7 - .../hello-world/hello-fedavg/requirements.txt | 0 examples/hello-world/hello-km/README.md | 161 ---------- examples/hello-world/hello-km/demo/km.ipynb | 101 ------ examples/hello-world/hello-km/demo/km.json | 172 ----------- .../app/config/config_fed_client.conf | 116 ------- .../app/config/config_fed_server.conf | 25 -- .../kaplan-meier/app/custom/kaplan_meier.py | 109 ------- .../kaplan-meier/app/custom/km_analysis.py | 46 --- .../jobs/kaplan-meier/app/custom/km_train.py | 71 ----- .../hello-km/jobs/kaplan-meier/meta.conf | 7 - .../hello-km/km_survival_curve.png | Bin 61673 -> 0 bytes .../hello-world/hello-km/requirements.txt | 1 - nvflare/app_common/abstract/fl_model.py | 2 + nvflare/app_common/wf_comm/__init__.py | 2 +- nvflare/app_common/wf_comm/base_wf_comm.py | 59 ++-- nvflare/app_common/wf_comm/wf_comm_api.py | 292 ++++++++++-------- .../app_common/wf_comm/wf_comm_api_spec.py | 92 +++--- nvflare/app_common/wf_comm/wf_communicator.py | 8 +- .../wf_comm/wf_communicator_spec.py | 64 +++- nvflare/app_common/wf_comm/wf_queue.py | 46 --- .../private/fed/server/server_json_config.py | 13 +- runtest.sh | 2 +- 38 files changed, 304 insertions(+), 2437 deletions(-) delete mode 100644 examples/hello-world/hello-cyclic-pt/README.md delete mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf delete mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf delete mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py delete mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py delete mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py delete mode 100644 examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf delete mode 100644 examples/hello-world/hello-cyclic-pt/requirements.txt delete mode 100644 examples/hello-world/hello-fedavg/README.md delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf delete mode 100644 examples/hello-world/hello-fedavg/requirements.txt delete mode 100644 examples/hello-world/hello-km/README.md delete mode 100644 examples/hello-world/hello-km/demo/km.ipynb delete mode 100644 examples/hello-world/hello-km/demo/km.json delete mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf delete mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf delete mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py delete mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py delete mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py delete mode 100644 examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf delete mode 100644 examples/hello-world/hello-km/km_survival_curve.png delete mode 100644 examples/hello-world/hello-km/requirements.txt delete mode 100644 nvflare/app_common/wf_comm/wf_queue.py diff --git a/examples/hello-world/hello-cyclic-pt/README.md b/examples/hello-world/hello-cyclic-pt/README.md deleted file mode 100644 index 16f68e18d0..0000000000 --- a/examples/hello-world/hello-cyclic-pt/README.md +++ /dev/null @@ -1,159 +0,0 @@ -# Fed Cyclic Weight Transfer: simplified - -This example illustrates How to use the new Workflow Communication API to contract a workflow: no need to write a controller. - -## FLARE Workflow Communicator API - -The Flare workflow Communicator API only has small set methods - -``` -class WFCommAPISpec(ABC): - @abstractmethod - def broadcast_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def broadcast(self, msg_payload): - pass - - @abstractmethod - def send(self, msg_payload: Dict): - pass - - @abstractmethod - def send_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def get_site_names(self): - pass - - @abstractmethod - def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: - pass - - @abstractmethod - def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: - pass -``` - - -## Writing a new Workflow - -With this new API writing the new workflow is really simple: - -* Workflow (Server) - -``` -from nvflare.app_common.workflows import wf_comm as flare - -class FedCyclic: - def __init__( - self, - output_path: str, - num_rounds: int = 5, - start_round: int = 0, - task_name="train", - order: str = RelayOrder.FIXED, - ): - super(FedCyclic, self).__init__() - <... skip init code ...> - - self.flare_comm = flare.get_wf_comm_api() - - self.check_inputs() - - def run(self): - - self.last_model = self.init_model() - - self.part_sites = self.flare_comm.get_site_names() - - if len(self.part_sites) <= 1: - raise ValueError(f"Not enough client sites. sites={self.part_sites}") - - start = self.start_round - end = self.start_round + self.num_rounds - for current_round in range(start, end): - targets = self.get_relay_orders() - relay_result = self.relay_and_wait(self.last_model, targets, current_round) - - self.logger.info(f"target sites ={targets}.") - - task_name, task_result = next(iter(relay_result.items())) - self.last_site, self.last_model = next(iter(task_result.items())) - - self.logger.info(f"ending current round={current_round}.") - gc.collect() - - self.save_model(self.last_model, self.output_path) - self.logger.info("\n fed cyclic ended \n") -``` - -Relay_and_wait - -``` - - def relay_and_wait(self, last_model: FLModel, targets: List[str], current_round): - msg_payload = { - MIN_RESPONSES: 1, - CURRENT_ROUND: current_round, - NUM_ROUNDS: self.num_rounds, - START_ROUND: self.start_round, - DATA: last_model, - TARGET_SITES: targets, - } - # (2) relay_and_wait and wait - results = self.flare_comm.relay_and_wait(msg_payload) - return results -``` - -## Configurations - -### client-side configuration - -This is the same as FLARE Client API configuration - -### server-side configuration - - Server side controller is really simple, all we need is to use WFController with newly defined workflow class - - -``` - { - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "fed_avg" - path = "nvflare.app_opt.pt.wf_controller.PTWFController" - args { - comm_msg_pull_interval = 5 - task_name = "train" - wf_class_path = "fed_cyclic.FedCyclic", - wf_args { - num_rounds = 10 - output_path = "/tmp/nvflare/fedavg/mode.pth" - } - } - } - ] - - components = [] - -} - -``` - - -## Run the job - -assume current working directory is at ```hello-cyclic-pt``` directory - -``` - nvflare simulator jobs/cyclic -w /tmp/cyclic -n 3 -t 3 - -``` diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf deleted file mode 100644 index 814025b8ad..0000000000 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_client.conf +++ /dev/null @@ -1,90 +0,0 @@ -{ - # version of the configuration - format_version = 2 - - # This is the application script which will be invoked. Client can replace this script with user's own training script. - app_script = "cifar10.py" - - # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. - app_config = "" - - # Client Computing Executors. - executors = [ - { - # tasks the executors are defined to handle - tasks = ["train"] - - # This particular executor - executor { - - # This is an executor for pytorch + Client API. The underline data exchange is using Pipe. - path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" - - args { - # launcher_id is used to locate the Launcher object in "components" - launcher_id = "launcher" - - # pipe_id is used to locate the Pipe object in "components" - pipe_id = "pipe" - - # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. - # Please refer to the class docstring for all available arguments - heartbeat_timeout = 60 - - # format of the exchange parameters - params_exchange_format = "pytorch" - - # if the transfer_type is FULL, then it will be sent directly - # if the transfer_type is DIFF, then we will calculate the - # difference VS received parameters and send the difference - params_transfer_type = "DIFF" - - # if train_with_evaluation is true, the executor will expect - # the custom code need to send back both the trained parameters and the evaluation metric - # otherwise only trained parameters are expected - train_with_evaluation = true - } - } - } - ], - - # this defined an array of task data filters. If provided, it will control the data from server controller to client executor - task_data_filters = [] - - # this defined an array of task result filters. If provided, it will control the result from client executor to server controller - task_result_filters = [] - - components = [ - { - # component id is "launcher" - id = "launcher" - - # the class path of this component - path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" - - args { - # the launcher will invoke the script - script = "python3 custom/{app_script} {app_config} " - # if launch_once is true, the SubprocessLauncher will launch once for the whole job - # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server - launch_once = true - } - } - { - id = "pipe" - - path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" - - args { - # Mode of the endpoint. A pipe has two endpoints. - # An endpoint can be either the one that initiates communication or the one listening. - # PASSIVE is the one listening. - mode = "PASSIVE" - - # root_path: is the directory location of the parameters exchange. - # You can also set it to an absolute path in your system. - root_path = "{WORKSPACE}/{JOB_ID}/{SITE_NAME}" - } - } - ] -} diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf deleted file mode 100644 index f45d61a39a..0000000000 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/config/config_fed_server.conf +++ /dev/null @@ -1,25 +0,0 @@ -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "fed_avg" - path = "nvflare.app_opt.pt.wf_controller.PTWFController" - args { - result_pull_interval = 5 - task_name = "train" - wf_class_path = "fed_cyclic.FedCyclic", - wf_args { - num_rounds = 2 - output_path = "/tmp/nvflare/fedavg/mode.pth" - } - } - } - ] - - components = [] - -} diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py deleted file mode 100644 index 4a030dd07d..0000000000 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/cifar10.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn as nn -import torch.optim as optim -import torchvision -import torchvision.transforms as transforms -from net import Net - -# (1) import nvflare client API -import nvflare.client as flare - -# (optional) set a fix place so we don't need to download everytime -DATASET_PATH = "/tmp/nvflare/data" -# (optional) We change to use GPU to speed things up. -# if you want to use CPU, change DEVICE="cpu" -DEVICE = "cuda:0" - - -def main(): - transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - - batch_size = 4 - epochs = 2 - - trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) - trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) - - testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) - testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) - - net = Net() - - # (2) initializes NVFlare client API - flare.init() - - while flare.is_running(): - # (3) receives FLModel from NVFlare - input_model = flare.receive() - print(f"current_round={input_model.current_round} at site = {flare.get_site_name()}") - - # (4) loads model from NVFlare - net.load_state_dict(input_model.params) - - criterion = nn.CrossEntropyLoss() - optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) - - # (optional) use GPU to speed things up - net.to(DEVICE) - # (optional) calculate total steps - steps = epochs * len(trainloader) - for epoch in range(epochs): # loop over the dataset multiple times - - running_loss = 0.0 - for i, data in enumerate(trainloader, 0): - # get the inputs; data is a list of [inputs, labels] - # (optional) use GPU to speed things up - inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) - - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = net(inputs) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # print statistics - running_loss += loss.item() - if i % 2000 == 1999: # print every 2000 mini-batches - print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") - running_loss = 0.0 - - print("Finished Training") - - PATH = "./cifar_net.pth" - torch.save(net.state_dict(), PATH) - - # (5) wraps evaluation logic into a method to re-use for - # evaluation on both trained and received model - def evaluate(input_weights): - net = Net() - net.load_state_dict(input_weights) - # (optional) use GPU to speed things up - net.to(DEVICE) - - correct = 0 - total = 0 - # since we're not training, we don't need to calculate the gradients for our outputs - with torch.no_grad(): - for data in testloader: - # (optional) use GPU to speed things up - images, labels = data[0].to(DEVICE), data[1].to(DEVICE) - # calculate outputs by running images through the network - outputs = net(images) - # the class with the highest energy is what we choose as prediction - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels).sum().item() - - print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") - return 100 * correct // total - - # (6) evaluate on received model for model selection - accuracy = evaluate(input_model.params) - # (7) construct trained FL model - output_model = flare.FLModel( - params=net.cpu().state_dict(), - metrics={"accuracy": accuracy}, - meta={"NUM_STEPS_CURRENT_ROUND": steps}, - ) - # (8) send model back to NVFlare - flare.send(output_model) - - -if __name__ == "__main__": - main() diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py deleted file mode 100644 index a94c37032d..0000000000 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/fed_cyclic.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import gc -import logging -import os -import random -from typing import List, Optional - -import torch -from net import Net - -from nvflare.app_common.abstract.fl_model import FLModel, ParamsType -from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common import wf_comm as flare -from nvflare.app_common.wf_comm.wf_comm_api_spec import ( - CURRENT_ROUND, - DATA, - MIN_RESPONSES, - NUM_ROUNDS, - START_ROUND, - TARGET_SITES, -) - -update_model = FLModelUtils.update_model - - -# Fed Cyclic Weight Transfer Workflow - - -class RelayOrder: - FIXED = "FIXED" - RANDOM = "RANDOM" - RANDOM_WITHOUT_SAME_IN_A_ROW = "RANDOM_WITHOUT_SAME_IN_A_ROW" - - -SUPPORTED_ORDERS = (RelayOrder.FIXED, RelayOrder.RANDOM, RelayOrder.RANDOM_WITHOUT_SAME_IN_A_ROW) - - -class FedCyclic: - def __init__( - self, - output_path: str, - num_rounds: int = 5, - start_round: int = 0, - task_name="train", - order: str = RelayOrder.FIXED, - ): - super(FedCyclic, self).__init__() - self.logger = logging.getLogger(self.__class__.__name__) - - self.output_path = output_path - self.num_rounds = num_rounds - self.start_round = start_round - self.task_name = task_name - self.order = order - self.last_site: Optional[str] = None - self.last_model: Optional[FLModel] = None - self.part_sites = None - - self.flare_comm = flare.get_wf_comm_api() - - self.check_inputs() - - def run(self): - - self.last_model = self.init_model() - - self.part_sites = self.flare_comm.get_site_names() - - if len(self.part_sites) <= 1: - raise ValueError(f"Not enough client sites. sites={self.part_sites}") - - start = self.start_round - end = self.start_round + self.num_rounds - for current_round in range(start, end): - targets = self.get_relay_orders() - relay_result = self.relay_and_wait(self.last_model, targets, current_round) - - self.logger.info(f"target sites ={targets}.") - - task_name, task_result = next(iter(relay_result.items())) - self.last_site, self.last_model = next(iter(task_result.items())) - - self.logger.info(f"ending current round={current_round}.") - gc.collect() - - self.save_model(self.last_model, self.output_path) - self.logger.info("\n fed cyclic ended \n") - - def relay_and_wait(self, last_model: FLModel, targets: List[str], current_round): - msg_payload = { - MIN_RESPONSES: 1, - CURRENT_ROUND: current_round, - NUM_ROUNDS: self.num_rounds, - START_ROUND: self.start_round, - DATA: last_model, - TARGET_SITES: targets, - } - # (2) relay_and_wait and wait - results = self.flare_comm.relay_and_wait(msg_payload) - return results - - def init_model(self): - net = Net() - model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) - return model - - def check_inputs(self): - if not isinstance(self.num_rounds, int): - raise TypeError("num_rounds must be int but got {}".format(type(self.num_rounds))) - if not isinstance(self.task_name, str): - raise TypeError("task_name must be a string but got {}".format(type(self.task_name))) - if self.order not in SUPPORTED_ORDERS: - raise ValueError(f"order must be in {SUPPORTED_ORDERS}") - - def get_relay_orders(self): - targets = list(self.part_sites) - if len(targets) <= 1: - raise ValueError("Not enough client sites.") - - if self.order == RelayOrder.RANDOM: - random.shuffle(targets) - elif self.order == RelayOrder.RANDOM_WITHOUT_SAME_IN_A_ROW: - random.shuffle(targets) - if self.last_site == targets[0]: - targets = targets.append(targets.pop(0)) - self.last_site = targets[-1] - return targets - - def save_model(self, model: FLModel, file_path: str): - if not file_path: - raise ValueError("invalid file path") - - dir_name = os.path.dirname(file_path) - os.makedirs(dir_name, exist_ok=True) - - self.logger.info(f"save best model to {file_path} \n") - torch.save(model.params, file_path) diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py deleted file mode 100644 index 031f84f432..0000000000 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/app/custom/net.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn as nn -import torch.nn.functional as F - - -class Net(nn.Module): - def __init__(self): - super().__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = torch.flatten(x, 1) # flatten all dimensions except batch - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return x diff --git a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf b/examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf deleted file mode 100644 index ed1e0f364b..0000000000 --- a/examples/hello-world/hello-cyclic-pt/jobs/cyclic/meta.conf +++ /dev/null @@ -1,7 +0,0 @@ -{ - name = "cyclic_pt" - deploy_map { - app = ["@ALL"] - } - min_clients = 2 -} diff --git a/examples/hello-world/hello-cyclic-pt/requirements.txt b/examples/hello-world/hello-cyclic-pt/requirements.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md deleted file mode 100644 index 76d43f16ed..0000000000 --- a/examples/hello-world/hello-fedavg/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# FedAvg: simplified - -This example illustrates How to use the new Workflow Communication API to contract a workflow: no need to write a controller. - -## FLARE Workflow Communicator API - -The Flare workflow Communicator API only has small set methods - -``` - -class WFCommAPISpec(ABC): - @abstractmethod - def broadcast_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def send_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def relay_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def broadcast(self, msg_payload: Dict): - pass - - @abstractmethod - def send(self, msg_payload: Dict): - pass - - @abstractmethod - def relay(self, msg_payload: Dict): - pass - - @abstractmethod - def get_site_names(self) -> List[str]: - pass - - @abstractmethod - def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: - pass - - @abstractmethod - def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: - pass - -``` - - -## Writing a new Workflow - -With this new API writing the new workflow is really simple: - -* Workflow (Server) - -``` -from nvflare.app_common.workflows import wf_comm as flare - -class FedAvg: - def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - model_selection_rule: str = None, - ): - super(FedAvg, self).__init__() - - - - self.flare_comm = flare.get_wf_comm_api() - - def run(self): - self.logger.info("start Fed Avg Workflow\n \n") - - start = self.start_round - end = self.start_round + self.num_rounds - - model = self.init_model() - for current_round in range(start, end): - - self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") - self.current_round = current_round - - sag_results = self.scatter_and_gather(model, current_round) - - aggr_result = self.aggr_fn(sag_results) - - self.logger.info(f"aggregate metrics = {aggr_result.metrics}") - - model = update_model(model, aggr_result) - - self.select_best_model(model) - - self.save_model(self.best_model, self.output_path) - - self.logger.info("end Fed Avg Workflow\n \n") - - -``` -Scatter and Gather (SAG): - -SAG is simply ask WFController to broadcast the model to all clients - -``` - def scatter_and_gather(self, model: FLModel, current_round): - msg_payload = {"min_responses": self.min_clients, - "current_round": current_round, - "num_round": self.num_rounds, - "start_round": self.start_round, - "data": model} - - # (2) broadcast and wait - results = self.flare_comm.broadcast_and_wait(msg_payload) - return results -``` - -## Configurations - -### client-side configuration - -This is the same as FLARE Client API configuration - -### server-side configuration - - Server side controller is really simple, all we need is to use WFController with newly defined workflow class - - -``` -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "fed_avg" - path = "nvflare.app_opt.pt.wf_controller.PTWFController" - args { - comm_msg_pull_interval = 5 - task_name = "train" - wf_class_path = "fedavg_pt.PTFedAvg", - wf_args { - min_clients = 2 - num_rounds = 10 - output_path = "/tmp/nvflare/fedavg/mode.pth" - stop_cond = "accuracy >= 55" - model_selection_rule = "accuracy >=" - } - } - } - ] - - components = [] - -} - -``` - - -## Run the job - -assume current working directory is at ```hello-fedavg``` directory - -``` -nvflare simulator -n 2 -t 2 jobs/fedavg -w /tmp/fedavg - -``` diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf deleted file mode 100644 index aaba629957..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf +++ /dev/null @@ -1,116 +0,0 @@ -{ - # version of the configuration - format_version = 2 - - # This is the application script which will be invoked. Client can replace this script with user's own training script. - app_script = "cifar10.py" - - # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. - app_config = "" - - # Client Computing Executors. - executors = [ - { - # tasks the executors are defined to handle - tasks = ["train"] - - # This particular executor - executor { - - # This is an executor for Client API. The underline data exchange is using Pipe. - path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" - - args { - # launcher_id is used to locate the Launcher object in "components" - launcher_id = "launcher" - - # pipe_id is used to locate the Pipe object in "components" - pipe_id = "pipe" - - # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. - # Please refer to the class docstring for all available arguments - heartbeat_timeout = 60 - - # format of the exchange parameters - params_exchange_format = "pytorch" - - # if the transfer_type is FULL, then it will be sent directly - # if the transfer_type is DIFF, then we will calculate the - # difference VS received parameters and send the difference - params_transfer_type = "DIFF" - - # if train_with_evaluation is true, the executor will expect - # the custom code need to send back both the trained parameters and the evaluation metric - # otherwise only trained parameters are expected - train_with_evaluation = true - } - } - } - ], - - # this defined an array of task data filters. If provided, it will control the data from server controller to client executor - task_data_filters = [] - - # this defined an array of task result filters. If provided, it will control the result from client executor to server controller - task_result_filters = [] - - components = [ - { - # component id is "launcher" - id = "launcher" - - # the class path of this component - path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" - - args { - # the launcher will invoke the script - script = "python3 custom/{app_script} {app_config} " - # if launch_once is true, the SubprocessLauncher will launch once for the whole job - # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server - launch_once = true - } - } - { - id = "pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - } - { - id = "metrics_pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - }, - { - id = "metric_relay" - path = "nvflare.app_common.widgets.metric_relay.MetricRelay" - args { - pipe_id = "metrics_pipe" - event_type = "fed.analytix_log_stats" - # how fast should it read from the peer - read_interval = 0.1 - } - }, - { - # we use this component so the client api `flare.init()` can get required information - id = "config_preparer" - path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" - args { - component_ids = ["metric_relay"] - } - } - ] -} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf deleted file mode 100644 index 291d4ad444..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf +++ /dev/null @@ -1,29 +0,0 @@ -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "fed_avg" - communicator { - path = "nvflare.app_common.wf_comm.wf_communicator.WFCommunicator" - args = {} - } - strategy { - path = "fedavg_pt.PTFedAvg" - args { - min_clients = 2 - num_rounds = 2 - output_path = "/tmp/nvflare/fedavg/mode.pth" - stop_cond = "accuracy >= 55" - } - serializers = ["nvflare.app_opt.pt.decomposers.TensorDecomposer"] - - } - } - ] - - components = [] -} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py deleted file mode 100644 index 274142432f..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn as nn -import torch.optim as optim -import torchvision -import torchvision.transforms as transforms -from net import Net - -# (1) import nvflare client API -import nvflare.client as flare - -# (optional) metrics -from nvflare.client.tracking import SummaryWriter - -# (optional) set a fix place so we don't need to download everytime -DATASET_PATH = "/tmp/nvflare/data" -# (optional) We change to use GPU to speed things up. -# if you want to use CPU, change DEVICE="cpu" -DEVICE = "cuda:0" -DEVICE = "cpu" - - -def main(): - transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - - batch_size = 4 - epochs = 2 - - trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) - trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) - - testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) - testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) - - net = Net() - - # (2) initializes NVFlare client API - flare.init() - - summary_writer = SummaryWriter() - while flare.is_running(): - # (3) receives FLModel from NVFlare - input_model = flare.receive() - print(f"current_round={input_model.current_round}") - - # (4) loads model from NVFlare - net.load_state_dict(input_model.params) - - criterion = nn.CrossEntropyLoss() - optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) - - # (optional) use GPU to speed things up - net.to(DEVICE) - # (optional) calculate total steps - steps = epochs * len(trainloader) - for epoch in range(epochs): # loop over the dataset multiple times - - running_loss = 0.0 - for i, data in enumerate(trainloader, 0): - # get the inputs; data is a list of [inputs, labels] - # (optional) use GPU to speed things up - inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) - - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = net(inputs) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # print statistics - running_loss += loss.item() - if i % 2000 == 1999: # print every 2000 mini-batches - print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") - global_step = input_model.current_round * steps + epoch * len(trainloader) + i - - summary_writer.add_scalar(tag="loss_for_each_batch", scalar=running_loss, global_step=global_step) - running_loss = 0.0 - - print("Finished Training") - - PATH = "./cifar_net.pth" - torch.save(net.state_dict(), PATH) - - # (5) wraps evaluation logic into a method to re-use for - # evaluation on both trained and received model - def evaluate(input_weights): - net = Net() - net.load_state_dict(input_weights) - # (optional) use GPU to speed things up - net.to(DEVICE) - - correct = 0 - total = 0 - # since we're not training, we don't need to calculate the gradients for our outputs - with torch.no_grad(): - for data in testloader: - # (optional) use GPU to speed things up - images, labels = data[0].to(DEVICE), data[1].to(DEVICE) - # calculate outputs by running images through the network - outputs = net(images) - # the class with the highest energy is what we choose as prediction - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels).sum().item() - - print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") - return 100 * correct // total - - # (6) evaluate on received model for model selection - accuracy = evaluate(input_model.params) - # (7) construct trained FL model - output_model = flare.FLModel( - params=net.cpu().state_dict(), - metrics={"accuracy": accuracy}, - meta={"NUM_STEPS_CURRENT_ROUND": steps}, - ) - # (8) send model back to NVFlare - flare.send(output_model) - - -if __name__ == "__main__": - main() diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py deleted file mode 100644 index e76cdc0a5f..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import sys -from typing import Callable, Dict, Optional - -from net import Net - -from nvflare.app_common.abstract.fl_model import FLModel, ParamsType -from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper -from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.utils.math_utils import parse_compare_criteria -from nvflare.app_common import wf_comm as flare -from nvflare.app_common.wf_comm.wf_comm_api_spec import ( - CURRENT_ROUND, - DATA, - MIN_RESPONSES, - NUM_ROUNDS, - RESP_MAX_WAIT_TIME, - START_ROUND, -) -from nvflare.security.logging import secure_format_traceback - -update_model = FLModelUtils.update_model - - -# FedAvg Workflow - - -class FedAvg: - def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - resp_max_wait_time: float = 5, - ): - super(FedAvg, self).__init__() - self.logger = logging.getLogger(self.__class__.__name__) - - self.output_path = output_path - self.min_clients = min_clients - self.resp_max_wait_time = resp_max_wait_time - self.num_rounds = num_rounds - self.start_round = start_round - self.current_round = start_round - self.best_model: Optional[FLModel] = None - if stop_cond: - self.stop_criteria = parse_compare_criteria(stop_cond) - else: - self.stop_criteria = None - - self.flare_comm = None - - def run(self): - self.logger.info("start Fed Avg Workflow\n \n") - self.flare_comm = flare.get_wf_comm_api() - - start = self.start_round - end = self.start_round + self.num_rounds - - model = self.init_model() - for current_round in range(start, end): - - self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") - self.current_round = current_round - - if self.should_stop(model.metrics, self.stop_criteria): - self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") - break - sag_results = self.scatter_and_gather(model, current_round) - - aggr_result = self.aggr_fn(sag_results) - - self.logger.info(f"aggregate metrics = {aggr_result.metrics}") - - print("model size =", sys.getsizeof(model.params)) - - model = update_model(model, aggr_result) - - self.select_best_model(model) - - self.save_model(self.best_model, self.output_path) - - self.logger.info("end Fed Avg Workflow\n \n") - - def init_model(self): - net = Net() - model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) - return model - - def scatter_and_gather(self, model: FLModel, current_round): - - msg_payload = { - MIN_RESPONSES: self.min_clients, - RESP_MAX_WAIT_TIME: self.resp_max_wait_time, - CURRENT_ROUND: current_round, - NUM_ROUNDS: self.num_rounds, - START_ROUND: self.start_round, - DATA: model, - } - - # (2) broadcast and wait - results = self.flare_comm.broadcast_and_wait(msg_payload) - print(f"{results=}") - return results - - def aggr_fn(self, sag_result: Dict[str, Dict[str, FLModel]]) -> FLModel: - - self.logger.info("fed avg aggregate \n") - - if not sag_result: - raise RuntimeError("input is None or empty") - - # we only have one task - task_name, task_result = next(iter(sag_result.items())) - - self.logger.info(f"aggregating {len(task_result)} update(s) at round {self.current_round}") - - try: - aggr_params_helper = WeightedAggregationHelper() - aggr_metrics_helper = WeightedAggregationHelper() - params_type = None - for site, fl_model in task_result.items(): - if params_type is None: - params_type = fl_model.params_type - - aggr_params_helper.add( - data=fl_model.params, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) - - self.logger.info(f"site={site} {fl_model.metrics=}") - - aggr_metrics_helper.add( - data=fl_model.metrics, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) - - aggr_params = aggr_params_helper.get_result() - aggr_metrics = aggr_metrics_helper.get_result() - - self.logger.info(f"{aggr_metrics=}") - - aggr_result = FLModel( - params=aggr_params, - params_type=params_type, - metrics=aggr_metrics, - meta={ - "num_rounds_aggregated": 1 + (self.current_round - self.start_round), - "current_round": self.current_round, - }, - ) - return aggr_result - except Exception as e: - raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") - - def select_best_model(self, curr_model: FLModel): - if self.best_model is None: - self.best_model = curr_model - return - - if self.stop_criteria: - metric, _, op_fn = self.stop_criteria - self.logger.info("compare models") - if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): - self.best_model = curr_model - else: - self.best_model = curr_model - - def save_model(self, model: FLModel, file_path: str): - pass - - def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[str] = None): - self.logger.info(f"stop_criteria, metrics = {stop_criteria=}, {metrics=}") - if stop_criteria is None or metrics is None: - return False - - key, target, op_fn = stop_criteria - value = metrics.get(key, None) - - if value is None: - raise RuntimeError(f"stop criteria key '{key}' doesn't exists in metrics") - - return op_fn(value, target) - - def is_curr_mode_better( - self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable - ) -> bool: - curr_metrics = curr_model.metrics - if curr_metrics is None: - return False - if target_metric not in curr_metrics: - return False - - best_metrics = best_model.metrics - return op_fn(curr_metrics.get(target_metric), best_metrics.get(target_metric)) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py deleted file mode 100644 index 33ff098a75..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -import torch - -from fedavg import FedAvg -from nvflare.app_common.abstract.fl_model import FLModel - - -class PTFedAvg(FedAvg): - def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - ): - super().__init__(min_clients, num_rounds, output_path, start_round, stop_cond) - - def save_model(self, model: FLModel, file_path: str): - if not file_path: - raise ValueError("invalid file path") - - dir_name = os.path.dirname(file_path) - os.makedirs(dir_name, exist_ok=True) - - self.logger.info(f"save best model to {file_path} \n") - torch.save(model.params, file_path) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py deleted file mode 100644 index 031f84f432..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn as nn -import torch.nn.functional as F - - -class Net(nn.Module): - def __init__(self): - super().__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = torch.flatten(x, 1) # flatten all dimensions except batch - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return x diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf deleted file mode 100644 index 1c27c4e99c..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf +++ /dev/null @@ -1,7 +0,0 @@ -{ - name = "fedavg" - deploy_map { - app = ["@ALL"] - } - min_clients = 2 -} diff --git a/examples/hello-world/hello-fedavg/requirements.txt b/examples/hello-world/hello-fedavg/requirements.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/hello-world/hello-km/README.md b/examples/hello-world/hello-km/README.md deleted file mode 100644 index 55e5f51aff..0000000000 --- a/examples/hello-world/hello-km/README.md +++ /dev/null @@ -1,161 +0,0 @@ -# Kaplan-Meier Analysis - -This example illustrates two features: -* How to perform Kaplan-Meirer Survival Analysis in federated setting -* How to use the new Flare Communicator API to contract a workflow: no need to write a controller. - -## FLARE Workflow Communicator API - -The Flare workflow Communicator API only has small set methods - -``` -class WFCommAPISpec(ABC): - @abstractmethod - def broadcast_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def broadcast(self, msg_payload): - pass - - @abstractmethod - def send(self, msg_payload: Dict): - pass - - @abstractmethod - def send_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def get_site_names(self): - pass - - @abstractmethod - def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: - pass - - @abstractmethod - def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: - pass -``` - - -## Writing a new Workflow - -With this new API writing the new workflow is really simple: - -For example for Kaplan-Meier Analysis, we could write a new workflow like this: - -``` - -from nvflare.app_common.workflows import wf_comm as flare - -class KM: - def __init__(self, min_clients: int, output_path: str): - super(KM, self).__init__() - self.logger = logging.getLogger(self.__class__.__name__) - self.output_path = output_path - self.min_clients = min_clients - self.num_rounds = 1 - - self.flare_comm = flare.get_wf_comm_api() - - def run(self): - results = self.start_km_analysis() - global_res = self.aggr_km_result(results) - self.save(global_res, self.output_path) - -``` - -The Kaplan-Meier analysis involves the following steps - -* start the analysis --> ask all clients to perform local KM analysis, then wait for results -* then aggregate the result to obtain gloabl results -* save the result - -We only need to one_round trip from server --> client, client --> server - -Let's define the start_km_analysis() - -``` - - def start_km_analysis(self): - self.logger.info("send kaplan-meier analysis command to all sites \n") - - msg_payload = { - MIN_RESPONSES: self.min_clients, - CURRENT_ROUND: 1, - NUM_ROUNDS: self.num_rounds, - START_ROUND: 1, - DATA: {}, - } - - results = self.flare_comm.broadcast_and_wait(msg_payload) - return results - -``` - -looks like to simply call send broadcast command, then just get the results. - -## Configurations - -### client-side configuration - -This is the same as FLARE Client API configuration - -### server-side configuration - - Server side controller is really simple, all we need is to use WFController with newly defined workflow class - - -``` -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "km" - path = "nvflare.app_common.workflows.wf_controller.WFController" - args { - task_name = "train" - wf_class_path = "kaplan_meier.KM", - wf_args { - min_clients = 2 - output_path = "/tmp/nvflare/km/km.json" - } - } - } - ] - - components = [] - -} - - -``` - - -## Run the job - -assume current working directory is at ```hello-km``` directory - -``` -nvflare simulator -n 2 -t 2 jobs/kaplan-meier -w /tmp/km -``` - - -## Display Result - -Once finished the results will be written to the output_path defined about. -We can copy the result to the demo directory and start notebook - -``` -cp /tmp/nvflare/km/km.json demo/. - -jupyter lab demo/km.ipynb - -``` -![KM survival curl](km_survival_curve.png) diff --git a/examples/hello-world/hello-km/demo/km.ipynb b/examples/hello-world/hello-km/demo/km.ipynb deleted file mode 100644 index a76483151b..0000000000 --- a/examples/hello-world/hello-km/demo/km.ipynb +++ /dev/null @@ -1,101 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "16c35505-cb9e-4517-bd85-6262cf881b3a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "41bd1b35-a8bb-4fe4-9619-c369b2201bcb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "with open(\"km.json\", 'r') as json_file:\n", - " data = json.load(json_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "e81f00ed-24c5-434e-942d-9b91a53eb232", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABm9UlEQVR4nO3deVwU9f8H8NdwLsutXIoInqmFiBeiKSQoeJBHKiIpoFmWmkaWV4rmlaampWlqlpUomkeWeRDhkfeBWmqeGJYcInLItRzz+4Mv+3PlcBcXBpfX8/HYx2PnM5+Zz3uGhX3z+XxmRhBFUQQRERGRjtCTOgAiIiIibWJyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckMkoTt37kAQBHz77bdSh1Itvv32WwiCgDt37kgdimRCQ0Ph4uJSrW0IgoA5c+ZUaxtEzxMmN1TnlX4Bnz17VqU8IyMDnTt3hkwmw/79+yWKrnoJggBBEPDGG2+Uu37mzJnKOqmpqTUcneb+/PNPDBkyBM7OzpDJZHB0dESvXr3wxRdfSB1arZGZmYm5c+fCzc0NZmZmMDExwUsvvYSpU6fi3r17UodHpBVMbojKkZmZid69e+PSpUvYtWsX/P39pQ6p2shkMuzYsQMKhaLMui1btkAmk1V53yNHjkRubi6cnZ2fJUS1HD9+HB07dsTFixcxduxYrFq1Cm+88Qb09PSwcuXKam+/IuvXr8e1a9cka/9xt2/fRrt27TBv3jy0adMGixcvxueff45XXnkFX3/9Nby9vaUOkUgrDKQOgKi2ycrKgp+fHy5cuICdO3eiT58+UodUrfz9/bFnzx7s27cPAwYMUJYfP34c8fHxeO2117Bjx44q7VtfXx/6+vraChXZ2dkwNTUtd92CBQtgaWmJM2fOwMrKSmVdSkpKjcRQHkNDQ621/SwKCwsxePBgJCcn49ChQ3j55ZdV1i9YsACLFy/WSlt5eXkwMjKCnh7/fyZp8JNH9JhHjx7B398f58+fx44dO9CvXz+V9T/99BP69euHhg0bwtjYGM2aNcO8efNQVFSkUs/b2xsvvfQSzp07h65du8LExARNmjTB2rVrnxrDpUuXEBoaiqZNm0Imk8HBwQGjR4/GgwcPVOrNmTMHgiDg5s2bCA0NhZWVFSwtLREWFoacnBy1j9nR0RE9evRAZGSkSvnmzZvh6uqKl156qdztTp06BX9/f1haWkIul8PLywvHjh1TqVPRnJt9+/ahe/fuMDU1hbm5Ofr164fLly+r1AkNDYWZmRlu3bqFvn37wtzcHMHBwRUex61bt/Diiy+WSWwAwM7OTvm+snlOT85dKT3HV65cwYgRI2BtbY2XX34ZS5cuhSAI+Oeff8rsY/r06TAyMsLDhw+Vx1E656agoAD16tVDWFhYme0yMzMhk8kwZcoUAIBCocDs2bPRoUMHWFpawtTUFN27d0dsbGyF56AyO3bswMWLFzFz5swyiQ0AWFhYYMGCBcplFxcXhIaGlqnn7e2t0sNz6NAhCIKArVu34qOPPoKjoyPkcjnOnz8PQRCwadOmMvs4cOAABEHAL7/8oiz777//MHr0aNjb28PY2BgvvvgiNm7cWKVjJWJyQ/Q/2dnZ6NOnD86cOYPt27ejf//+Zep8++23MDMzQ3h4OFauXIkOHTpg9uzZmDZtWpm6Dx8+RN++fdGhQwcsWbIEjRo1wttvv/3UP9jR0dG4ffs2wsLC8MUXX2D48OHYunUr+vbtC1EUy9QfNmwYsrKysGjRIgwbNgzffvst5s6dq9GxjxgxAj///DMePXoEoOS//O3bt2PEiBHl1v/999/Ro0cPZGZmIiIiAgsXLkR6ejp69uyJ06dPV9rW999/j379+sHMzAyLFy/GrFmzcOXKFbz88stlkqDCwkL4+fnBzs4OS5cuxWuvvVbhfp2dnXHu3Dn89ddfGh27OoYOHYqcnBwsXLgQY8eOxbBhwyAIArZt21am7rZt29C7d29YW1uXWWdoaIhBgwZh9+7dZYYBd+/ejfz8fAwfPhxASbKzYcMGeHt7Y/HixZgzZw7u37+v7FXU1J49ewCUDBVWh3nz5mHv3r2YMmUKFi5ciDZt2qBp06blnqOoqChYW1vDz88PAJCcnIwuXbrgt99+w4QJE7By5Uo0b94cY8aMwYoVK6olXtJxIlEd980334gARGdnZ9HQ0FDcvXt3hXVzcnLKlL311luiXC4X8/LylGVeXl4iAHHZsmXKsvz8fLFdu3ainZ2dqFAoRFEUxfj4eBGA+M0331TaxpYtW0QA4pEjR5RlERERIgBx9OjRKnUHDRok1q9f/+kHLooiAHH8+PFiWlqaaGRkJH7//feiKIri3r17RUEQxDt37ijbuX//viiKolhcXCy2aNFC9PPzE4uLi1XibtKkidirVy9lWem5jY+PF0VRFLOyskQrKytx7NixKnEkJSWJlpaWKuUhISEiAHHatGlqHcvBgwdFfX19UV9fX/T09BQ//PBD8cCBA8pzXaq8c/74+YiIiFAulx57UFBQmbqenp5ihw4dVMpOnz4tAhC/++47leNwdnZWLh84cEAEIP78888q2/bt21ds2rSpcrmwsFDMz89XqfPw4UPR3t6+zM/8ybjL4+7uLlpaWlZa53HOzs5iSEhImXIvLy/Ry8tLuRwbGysCEJs2bVrmszt9+nTR0NBQTEtLU5bl5+eLVlZWKscwZswYsUGDBmJqaqrK9sOHDxctLS3L/Z0gqgx7boj+Jzk5GTKZDE5OThXWMTExUb7PyspCamoqunfvjpycHPz9998qdQ0MDPDWW28pl42MjPDWW28hJSUF586dU6uNvLw8pKamokuXLgCA8+fPl6k/btw4leXu3bvjwYMHyMzMrLCNJ1lbW8Pf3x9btmwBAERGRqJr167lTgS+cOECbty4gREjRuDBgwdITU1FamoqsrOz4ePjgyNHjqC4uLjcdqKjo5Geno6goCDldqmpqdDX14eHh0e5Qy5vv/22WsfQq1cvnDhxAq+++iouXryIJUuWwM/PD46Ojspei6p68hwDQGBgIM6dO4dbt24py6KiomBsbKwyd+lJPXv2hI2NDaKiopRlDx8+RHR0NAIDA5Vl+vr6MDIyAgAUFxcjLS0NhYWF6NixY7mfg6fJzMyEubm5xtupKyQkROWzC5Sco4KCAuzcuVNZdvDgQaSnpyuPVRRF7NixAwEBARBFUeVz4efnh4yMjCodL9VtTG6I/uerr76CkZER/P39K7y65fLlyxg0aBAsLS1hYWEBW1tbvP766wBKLh1/XMOGDctMPG3ZsiUAVHrfl7S0NEyaNAn29vYwMTGBra0tmjRpUm4bANC4cWOV5dLhkNI5H2lpaUhKSlK+ytsHUDI0FR0djYSEBOzevbvCIakbN24AKPkys7W1VXlt2LAB+fn5FbZRum3Pnj3LbHvw4MEyE38NDAzQqFGjcvdVnk6dOmHnzp14+PAhTp8+jenTpyMrKwtDhgzBlStX1N7Pk0rP/+OGDh0KPT09ZZIiiiK2b9+OPn36wMLCosJ9GRgY4LXXXsNPP/2E/Px8AMDOnTtRUFCgktwAwKZNm9C2bVvIZDLUr18ftra22Lt3b4XntzIWFhbIysrSeDt1lXeO3Nzc0KpVK5VELioqCjY2NujZsycA4P79+0hPT8e6devKfCZK5yZpc0I41Q28Worof9q0aYNff/0VPj4+6NWrF44dO6bSi5Oeng4vLy9YWFjg448/RrNmzSCTyXD+/HlMnTq1wt4KTQ0bNgzHjx/HBx98gHbt2sHMzAzFxcXw9/cvt42KrkYS/zc/Z/DgwTh8+LCyPCQkpNzJtK+++iqMjY0REhKC/Px8DBs2rNz9lsbw6aefol27duXWMTMzq3Tb77//Hg4ODmXWGxio/kkyNjau0hU3RkZG6NSpEzp16oSWLVsiLCwM27dvR0REBARBKHebJyeFP+7JHgmgJHnt3r07tm3bhhkzZuDkyZNISEhQ64qj4cOH46uvvsK+ffswcOBAbNu2Da1atYKbm5uyzg8//IDQ0FAMHDgQH3zwAezs7KCvr49Fixap9Bapq1WrVoiLi8Pdu3cr7Z0sVdl5Ku8zV945Akp6bxYsWIDU1FSYm5tjz549CAoKUv6sSz8Tr7/+OkJCQsrdR9u2bZ8aL9HjmNwQPaZz587YvXs3+vXrh169euHo0aOwtbUFUHJVyIMHD7Bz50706NFDuU18fHy5+7p3716Zy4avX78OABXesfbhw4eIiYnB3LlzMXv2bGV5aY9HVSxbtkzZiwOUfCmXx8TEBAMHDsQPP/yAPn36wMbGptx6zZo1A1DSE+Dr66tRLKXb2tnZabxtVXXs2BEAkJiYCOD/e7bS09NV6pV35dPTBAYG4p133sG1a9cQFRUFuVyOgICAp27Xo0cPNGjQAFFRUXj55Zfx+++/Y+bMmSp1fvzxRzRt2hQ7d+5USTQiIiI0jhMAAgICsGXLFvzwww+YPn36U+tbW1uXOUdAyXlq2rSp2u0GBgZi7ty52LFjB+zt7ZGZmamcNA0Atra2MDc3R1FRUY19Jkj3cViK6Ak+Pj7YsmULbt68CX9/f+XcldL/VsXHrlhSKBT48ssvy91PYWEhvvrqK5W6X331FWxtbdGhQ4dytymvDQDPdMVIhw4d4Ovrq3y1adOmwrpTpkxBREQEZs2aVen+mjVrhqVLlyqvrnrc/fv3K9zWz88PFhYWWLhwIQoKCjTa9mliY2PLvZrs119/BQC88MILAEqSMhsbGxw5ckSlXkU/x8q89tpr0NfXx5YtW5RX2KlzDxw9PT0MGTIEP//8M77//nsUFhaWGZIq77Nw6tQpnDhxQuM4AWDIkCFwdXXFggULyt1HVlaWSoLVrFkznDx5UuWqrl9++QV3797VqN3WrVvD1dUVUVFRiIqKQoMGDVT+OdDX11feS6m8K92e5TNBdRd7bojKMWjQIKxfvx6jR4/Gq6++iv3796Nr166wtrZGSEgI3n33XQiCgO+//77cL1SgpIdk8eLFuHPnDlq2bImoqChcuHAB69atq/DGbhYWFujRoweWLFmCgoICODo64uDBgxX2Dmmbm5ubytBIefT09LBhwwb06dMHL774IsLCwuDo6Ij//vsPsbGxsLCwwM8//1zuthYWFlizZg1GjhyJ9u3bY/jw4bC1tUVCQgL27t2Lbt26YdWqVVWKfeLEicjJycGgQYPQqlUrKBQKHD9+HFFRUXBxcVG5t8wbb7yBTz75BG+88QY6duyII0eOKHvVNGFnZ4dXXnkFy5cvR1ZWVpkEpTKBgYH44osvEBERAVdXV7Ru3Vplff/+/bFz504MGjQI/fr1Q3x8PNauXYs2bdqUm1Q+jaGhIXbu3AlfX1/06NEDw4YNQ7du3WBoaIjLly8jMjIS1tbWynvdvPHGG/jxxx/h7++PYcOG4datW/jhhx+UvW+aCAwMxOzZsyGTyTBmzJgyQ42ffPIJYmNj4eHhgbFjx6JNmzZIS0vD+fPn8dtvvyEtLU3jNqmOk+w6LaJaovRy5TNnzpRZt3TpUhGA2L9/f7GgoEA8duyY2KVLF9HExERs2LCh8nJjAGJsbKxyOy8vL/HFF18Uz549K3p6eooymUx0dnYWV61apbL/8i5L/vfff8VBgwaJVlZWoqWlpTh06FDx3r17FV6mXHqJ9pPHU3r5dWXwv0vBK1NRO3FxceLgwYPF+vXri8bGxqKzs7M4bNgwMSYm5qmxxMbGin5+fqKlpaUok8nEZs2aiaGhoeLZs2eVdUJCQkRTU9OnHkOpffv2iaNHjxZbtWolmpmZiUZGRmLz5s3FiRMnisnJySp1c3JyxDFjxoiWlpaiubm5OGzYMDElJUXtc/y49evXiwBEc3NzMTc3t8z6Jy8FL1VcXCw6OTmJAMT58+eXu37hwoWis7OzaGxsLLq7u4u//PJLuft7Mu7KPHz4UJw9e7bo6uoqyuVyUSaTiS+99JI4ffp0MTExUaXusmXLREdHR9HY2Fjs1q2bePbs2QovBd++fXuFbd64cUMEIAIQ//jjj3LrJCcni+PHjxednJxEQ0ND0cHBQfTx8RHXrVun1nERPU4QxQr+7SSiKvP29kZqamq13FCOiIgqxzk3REREpFOY3BAREZFOYXJDREREOoVzboiIiEinsOeGiIiIdAqTGyIiItIpde4mfsXFxbh37x7Mzc0rfHYKERER1S6iKCIrKwsNGzZ86jPn6lxyc+/ePbUeGkdERES1z927d9GoUaNK69S55Mbc3BxAycmxsLCQOBoiIiJSR2ZmJpycnJTf45Wpc8lN6VCUhYUFkxsiIqLnjDpTSjihmIiIiHQKkxsiIiLSKUxuiIiISKfUuTk3RERSKioqQkFBgdRhENVKRkZGT73MWx1MboiIaoAoikhKSkJ6errUoRDVWnp6emjSpAmMjIyeaT9MboiIakBpYmNnZwe5XM6biBI9ofQmu4mJiWjcuPEz/Y4wuSEiqmZFRUXKxKZ+/fpSh0NUa9na2uLevXsoLCyEoaFhlffDCcVERNWsdI6NXC6XOBKi2q10OKqoqOiZ9sPkhoiohnAoiqhy2vodYXJDREREOkXS5ObIkSMICAhAw4YNIQgCdu/e/dRtDh06hPbt28PY2BjNmzfHt99+W+1xEhFRWaGhoRg4cKDUYdRa1XF+XFxcsGLFCq3uUxdJmtxkZ2fDzc0Nq1evVqt+fHw8+vXrh1deeQUXLlzA5MmT8cYbb+DAgQPVHCkRET1p5cqVKv9gent7Y/Lkyc+834KCAkydOhWurq4wNTVFw4YNMWrUKNy7d0/jfR0+fBg9e/ZEvXr1IJfL0aJFC4SEhEChUDxznE/z5PmpKZmZmZg5cyZatWoFmUwGBwcH+Pr6YufOnRBFscbjkYKkV0v16dMHffr0Ubv+2rVr0aRJEyxbtgwA0Lp1a/zxxx/47LPP4OfnV11hqqW4qAgPs+5LGgMAyGT1IGjhBkhVZWKoz3kFRHWEpaVltew3JycH58+fx6xZs+Dm5oaHDx9i0qRJePXVV3H27Fm193PlyhX4+/tj4sSJ+Pzzz2FiYoIbN25gx44dzzRhVaFQqHUfluo6P5VJT0/Hyy+/jIyMDMyfPx+dOnWCgYEBDh8+jA8//BA9e/aElZVVlfZdUFDwTFcw1aTn6lLwEydOwNfXV6XMz8+v0v8U8vPzkZ+fr1zOzMysltgeZt2H90+9qmXfmmieKyDuzgJI1SnX0dka28d5MsEh0hE//vgj5s6di5s3b0Iul8Pd3R0//fQTTE1NERoaivT0dOzevRuhoaE4fPgwDh8+jJUrVwIo6W13cXHBX3/9hQ8++ABHjx6Fqakpevfujc8++ww2NjbltmlpaYno6GiVslWrVqFz585ISEhA48aN1Yr94MGDcHBwwJIlS5RlzZo1g7+/v3J5zpw52L17Ny5cuKAsW7FiBVasWIE7d+4AgPI4O3XqhNWrV8PY2BhBQUGIiYnBqVOnVNp0c3PDa6+9htmzZ6ucn3Xr1mHOnDn4999/Ve7AO2DAANSvXx8bN27ErVu3EB4ejpMnTyI7OxutW7fGokWLynzvVWbGjBm4c+cOrl+/joYNGyrLW7ZsiaCgIMhkMgAlE3d37dqlMmxmZWWFFStWIDQ0FHfu3EGTJk2wdetWfPnllzh16hSWLFmCqVOnYufOnSodE7t27cKoUaOQnJwMuVyOu3fv4v3338fBgwehp6eH7t27Y+XKlXBxcVH7OJ7VczWhOCkpCfb29ipl9vb2yMzMRG5ubrnbLFq0CJaWlsqXk5NTTYQqmZsmIkyER5K1f/afh8gteLZL+IjqAlEUkaMolOSl7tBEYmIigoKCMHr0aFy9ehWHDh3C4MGDy91+5cqV8PT0xNixY5GYmIjExEQ4OTkhPT0dPXv2hLu7O86ePYv9+/cjOTkZw4YN0+h8ZWRkQBAElV4Hb29vhIaGVriNg4MDEhMTceTIEY3aKk9MTAyuXbuG6Oho/PLLLwgODsbp06dx69YtZZ3Lly/j0qVLGDFiRJnthw4digcPHiA2NlZZlpaWhv379yM4OBgA8OjRI/Tt2xcxMTGIi4uDv78/AgICkJCQoFaMxcXF2Lp1K4KDg1USm1JmZmYwMNCsT2PatGmYNGkSrl69iqFDh6J///6IjIxUqbN582YMHDgQcrkcBQUF8PPzg7m5OY4ePYpjx47BzMwM/v7+NTIUWOq56rmpiunTpyM8PFy5nJmZWS0JjrW5LQ4NiH56xWqSm5eGPgcCAQB/TH0FJvLy/yOqLjmKInSc/1uNtkn0PMstKEKb2dLMF7zysR/kRk//85+YmIjCwkIMHjwYzs7OAABXV9dy61paWsLIyAhyuRwODg7K8lWrVsHd3R0LFy5Ulm3cuBFOTk64fv06WrZs+dQ48vLyMHXqVAQFBcHCwkJZ3rhxYzRo0KDC7YYOHYoDBw7Ay8sLDg4O6NKlC3x8fDBq1CiV/ajD1NQUGzZsUBmOcnNzQ2RkJGbNmgWg5Evew8MDzZs3L7O9tbU1+vTpg8jISPj4+AAo6RWzsbHBK6+8otyfm5ubcpt58+Zh165d2LNnDyZMmPDUGFNTU/Hw4UO0atVKo2OrzOTJkzF48GDlcnBwMEaOHImcnBzI5XJkZmZi79692LVrFwAgKioKxcXF2LBhg7IH/5tvvoGVlRUOHTqE3r17ay22yjxXyY2DgwOSk5NVypKTk2FhYQETE5NytzE2NoaxsXG1x6anr4/6Vg5Pr1hNcnL+/0dpYmSg1h8uIqLKuLm5wcfHB66urvDz80Pv3r0xZMgQWFtbq72PixcvIjY2FmZmZmXW3bp1C2fOnMFbb72lLNu3bx+6d++uXC4oKMCwYcMgiiLWrFmjsv13331Xadv6+vr45ptvMH/+fPz+++84deoUFi5ciMWLF+P06dOVJkZPcnV1LTPPJjg4GBs3bsSsWbMgiiK2bNmi8s/0k4KDgzF27Fh8+eWXMDY2xubNmzF8+HDlMNWjR48wZ84c7N27V5lY5ubmqt1zUx2ThTt27Kiy3LdvXxgaGmLPnj0YPnw4duzYAQsLC+XQ2cWLF3Hz5k2Ym5urbJeXl6fSy1XdnqtvQE9PT/z6668qZdHR0fD09JQoIiKiqjEx1MeVj6W5EMLEUF+tevr6+oiOjsbx48dx8OBBfPHFF5g5cyZOnTqFJk2aqLWPR48eISAgAIsXLy6zrkGDBiguLoaHh4eyzNHRUfm+NLH5559/8Pvvv2vc2/L4PkeOHImRI0di3rx5aNmyJdauXYu5c+dCT0+vTFJQ3lPbTU1Ny5QFBQVh6tSpOH/+PHJzc3H37l0EBgZWGEdAQABEUcTevXvRqVMnHD16FJ999ply/ZQpUxAdHY2lS5eiefPmMDExwZAhQ9QezrG1tYWVlRX+/vvvp9YVBKFKx21kZIQhQ4YgMjISw4cPR2RkJAIDA5XDXY8ePUKHDh2wefPmcuOrKZImN48ePcLNmzeVy/Hx8bhw4QLq1auHxo0bY/r06fjvv/+U2fm4ceOwatUqfPjhhxg9ejR+//13bNu2DXv37pXqEIiIqkQQhOeih1UQBHTr1g3dunXD7Nmz4ezsjF27dpXbQ2FkZFTmKqT27dtjx44dcHFxqXC+x5P/5QP/n9jcuHEDsbGxWnsml7W1NRo0aIDs7GwAJV+4SUlJEEVROYzy+OTiyjRq1AheXl7YvHkzcnNz0atXL9jZ2VVYXyaTYfDgwdi8eTNu3ryJF154Ae3bt1euP3bsGEJDQzFo0CAAJd+RpZOa1aGnp4fhw4fj+++/R0RERJl5N48ePYJMJoOBgQFsbW2RmJioXHfjxg3k5OSo1U5wcDB69eqFy5cv4/fff8f8+fOV69q3b4+oqCjY2dlVORnVBkknFJ89exbu7u5wd3cHAISHh8Pd3R2zZ88GUDLe+3h3XJMmTbB3715ER0fDzc0Ny5Ytw4YNGyS/DJyISBeVDuOcPXsWCQkJ2LlzJ+7fv4/WrVuXW9/FxQWnTp3CnTt3kJqaiuLiYowfPx5paWkICgrCmTNncOvWLRw4cABhYWEVXo5dUFCAIUOG4OzZs9i8eTOKioqQlJSEpKQklV6MUaNGYfr06RXG/9VXX+Htt9/GwYMHcevWLVy+fBlTp07F5cuXERAQAKBkUvL9+/exZMkS3Lp1C6tXr8a+ffvUPkfBwcHYunUrtm/frpwY/LT6e/fuxcaNG8vUb9GiBXbu3IkLFy7g4sWLGDFiBIqLi9WOBQAWLFgAJycneHh44LvvvsOVK1dw48YNbNy4Ee7u7nj0qOSCk549e2LVqlWIi4vD2bNnMW7cOLUv8+7RowccHBwQHByMJk2aqPS8BQcHw8bGBgMGDMDRo0cRHx+PQ4cO4d1338W///6r0bE8C0mTG29vb4iiWOZVetOjb7/9FocOHSqzTVxcHPLz83Hr1q1KZ8oTEVHVWVhY4MiRI+jbty9atmyJjz76CMuWLavw/mRTpkyBvr4+2rRpA1tbWyQkJKBhw4Y4duwYioqK0Lt3b7i6umLy5MmwsrJSuST6cf/99x/27NmDf//9F+3atUODBg2Ur+PHjyvrJSQkqPQ+PKlz58549OgRxo0bhxdffBFeXl44efIkdu/eDS8vLwAl90v78ssvsXr1ari5ueH06dOYMmWK2udoyJAhePDgAXJyctS6G3HpDQWvXbtW5qqq5cuXw9raGl27dkVAQAD8/PxUenbUUa9ePZw8eRKvv/465s+fD3d3d3Tv3h1btmzBp59+qrz3zrJly+Dk5ITu3btjxIgRmDJlitoPdhUEAUFBQbh48WKZBE0ul+PIkSNo3LgxBg8ejNatW2PMmDHIy8ur0Z4cQawrtyv8n8zMTFhaWiIjI0PSLjNty8lJhcf2khn3p4bGQl7jV0sVKq/8UPdKDKK6Ii8vD/Hx8WjSpInyPiNEVFZlvyuafH8/V/e5ISIiInoaJjdERESkU5jcEBERkU5hckNEREQ6hckNERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3RERUJaGhoWo9cqCumjNnDtq1a6fVfXp7e2Py5Mla3acuYnJDRERVsnLlSuWzAAHtfvHu3LkTvXv3Rv369SEIgtpP6n7SxYsX8eqrr8LOzg4ymQwuLi4IDAxESkqKVuKszJQpUxATE1Pt7TxJoVBgyZIlcHNzg1wuh42NDbp164ZvvvkGBQUFNR6PFPgAICIiqpLShzBWh+zsbLz88ssYNmwYxo4dW6V93L9/Hz4+Pujfvz8OHDgAKysr3LlzB3v27EF2dnaVY1MoFDAyMnpqPTMzM5iZmVW5napQKBTw8/PDxYsXMW/ePHTr1g0WFhY4efIkli5dCnd39yr3JhUUFKj95HCpseeGiIgq9OOPP8LV1RUmJiaoX78+fH19lYnB48NSoaGhOHz4MFauXAlBECAIAu7cuQMA+Ouvv9CnTx+YmZnB3t4eI0eORGpqaqXtjhw5ErNnz4avr2+VYz927BgyMjKwYcMGuLu7o0mTJnjllVfw2WefoUmTJgCAb7/9FlZWVirb7d69G4IgKJdLh5c2bNigfKDjunXr0LBhQxQXF6tsO2DAAIwePVplOwA4ePAgZDIZ0tPTVepPmjQJPXv2BAA8ePAAQUFBcHR0hFwuh6urK7Zs2aLRMa9YsQJHjhxBTEwMxo8fj3bt2qFp06YYMWIETp06hRYtWgAAXFxcsGLFCpVt27Vrhzlz5iiXBUHAmjVr8Oqrr8LU1BTz5s1Do0aNsGbNGpXt4uLioKenh3/++QcAkJ6ejjfeeAO2trawsLBAz549cfHiRY2O41kxuSEikoIoAopsaV6iqFaIiYmJCAoKwujRo3H16lUcOnQIgwcPhljO9itXroSnpyfGjh2LxMREJCYmwsnJCenp6ejZsyfc3d1x9uxZ7N+/H8nJyRg2bNgzn8LQ0FB4e3tXuN7BwQGFhYXYtWtXuTFr4ubNm9ixYwd27tyJCxcuYOjQoXjw4AFiY2OVddLS0rB//34EBweX2d7HxwdWVlbYsWOHsqyoqAhRUVHK+nl5eejQoQP27t2Lv/76C2+++SZGjhyJ06dPqx3n5s2b4evrC3d39zLrDA0NYWpqqslhY86cORg0aBD+/PNPvPHGGwgKCkJkZGSZNrt16wZnZ2cAwNChQ5GSkoJ9+/bh3LlzaN++PXx8fJCWlqZR28+Cw1JERFIoyAEWNpSm7Rn3AKOnf8klJiaisLAQgwcPVn5xubq6llvX0tISRkZGkMvlcHBwUJavWrUK7u7uWLhwobJs48aNcHJywvXr19GyZcsqH0aDBg3K9Jw8rkuXLpgxYwZGjBiBcePGoXPnzujZsydGjRoFe3t7jdpSKBT47rvvYGtrqyzr06cPIiMj4ePjA6Ckl8vGxgavvPJKme319fUxfPhwREZGYsyYMQCAmJgYpKen47XXXgMAODo6YsqUKcptJk6ciAMHDmDbtm3o3LmzWnHeuHGj0oRPUyNGjEBYWJhyOTg4GMuWLUNCQgIaN26M4uJibN26FR999BEA4I8//sDp06eRkpICY2NjAMDSpUuxe/du/Pjjj3jzzTe1Fltl2HNDRETlcnNzg4+PD1xdXTF06FCsX78eDx8+1GgfFy9eRGxsrHL+iZmZGVq1agUAuHXrFjZv3qyy7ujRo2rve9GiRfjuu+8qrbNgwQIkJSVh7dq1ePHFF7F27Vq0atUKf/75p0bH4ezsrJLYACVf9Dt27EB+fj6Akh6M4cOHQ0+v/K/W4OBgHDp0CPfu3VPW79evn3JYrKioCPPmzYOrqyvq1asHMzMzHDhwAAkJCWrH+aw9VE/q2LGjynK7du3QunVrZe/N4cOHkZKSgqFDhwIo+Xk/evQI9evXV/m5xsfH49atW1qNrTLsudFBuYW5Jf8V1mibRYCgAMTnY7IZkeQM5SU9KFK1rQZ9fX1ER0fj+PHjOHjwIL744gvMnDkTp06dUs5ZeZpHjx4hICAAixcvLrOutOfFw8NDWebo6KjeMWigfv36GDp0KIYOHYqFCxfC3d0dS5cuxaZNm6Cnp1cmISjviqLyhnMCAgIgiiL27t2LTp064ejRo/jss88qjKNTp05o1qwZtm7dirfffhu7du1Sudrs008/xcqVK7FixQq4urrC1NQUkydPhkKhUPtYW7Zsib///vup9Z7luIODgxEZGYlp06YhMjIS/v7+qF+/PoCSn3eDBg1w6NChMts9ObepOjG50UHeu/pK0q55K6Awxxmi6CdJ+0TPFUFQa2hIaoIgoFu3bujWrRtmz54NZ2dn7Nq1C+Hh4WXqGhkZoaioSKWsffv22LFjB1xcXGBgUP5Xjrm5ebXEXh4jIyM0a9ZMOSna1tYWWVlZyM7OVn6Rq3vZuUwmw+DBg7F582bcvHkTL7zwAtq3b1/pNsHBwdi8eTMaNWoEPT099OvXT7nu2LFjGDBgAF5//XUAQHFxMa5fv442bdqofXwjRozAjBkzEBcXV2beTUFBARQKBUxNTWFra4vExETluszMTMTHx6vdxkcffYRz587hxx9/xNq1a5Xr2rdvj6SkJBgYGMDFxUXtuLWNw1I6wkRfBve8PKnDgIH8H+TlPqj1EyWJ6OlOnTqFhQsX4uzZs0hISMDOnTtx//59tG7dutz6Li4uOHXqFO7cuYPU1FQUFxdj/PjxSEtLQ1BQEM6cOYNbt27hwIEDCAsLK5MIPS4tLQ0XLlzAlStXAADXrl3DhQsXkJSUpKwzffp0jBo1qsJ9/PLLL3j99dfxyy+/4Pr167h27RqWLl2KX3/9FQMGDAAAeHh4QC6XY8aMGbh16xYiIyNVelOeJjg4GHv37sXGjRvLnUhcXv3z589jwYIFGDJkiHJeCgC0aNFC2VN29epVvPXWW0hOTlY7FgCYPHkyunXrBh8fH6xevRoXL17E7du3sW3bNnTp0gU3btwAAPTs2RPff/89jh49ij///BMhISHQ19dXqw0XFxd07doVY8aMQVFREV599VXlOl9fX3h6emLgwIE4ePAg7ty5g+PHj2PmzJk4e/asRsfyLNhzoyMEQcCmxBTkPnb5Yk3KFQR4OzcCAJisbCVdkuHUBRi9v+S/YiJ6JhYWFjhy5AhWrFiBzMxMODs7Y9myZejTp0+59adMmYKQkBC0adMGubm5iI+Ph4uLC44dO4apU6eid+/eyM/Ph7OzM/z9/SucmwIAe/bsUZnIOnz4cABARESE8nLlxMTESuejtGnTBnK5HO+//z7u3r0LY2NjtGjRAhs2bMDIkSMBAPXq1cMPP/yADz74AOvXr4ePjw/mzJmj9sTXnj17ol69erh27RpGjBjx1PrNmzdH586dcfr06TKXYn/00Ue4ffs2/Pz8IJfL8eabb2LgwIHIyMhQKxYAMDY2RnR0ND777DN89dVXmDJlCuRyOVq3bo13330XL730EoCSxDA+Ph79+/eHpaUl5s2bp3bPDVCSpL3zzjsYNWoUTExMlOWCIODXX3/FzJkzERYWhvv378PBwQE9evTQeBL3sxBEbc8+quUyMzNhaWmJjIwMWFhYSB2O9ogisNEfuHtSkuZzBAEeLk4AgFN37kIu5cdKzStBiGpKXl4e4uPjlfdIIaLyVfa7osn3N3tudIUglPRY1PBE4lK5OVnAT71K3k/6G3J5zY2hAwAUOcDS5jXbJhER1UpMbnSJlBMUCx+714SRKXtOiIhIMpxQTERERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBHRM3FxcSnzKIHKzJkzB+3atXvmdgVBwO7du595PzUlNDQUAwcO1Oo+NT33dQWTGyIiqjMOHz6sfB6UXC5HixYtEBISAoVCUe1tr1y5UqOHcmpLZmYmZs6ciVatWkEmk8HBwQG+vr7YuXMndPUJTLxDMRER1QlXrlyBv78/Jk6ciM8//xwmJia4ceMGduzYUekTyp9GoVDAyMjoqfUsLS2r3EZVpaen4+WXX0ZGRgbmz5+PTp06wcDAAIcPH8aHH36Inj17wsrKqkr7LigogKGhoXYD1hL23BARUYWysrIQHBwMU1NTNGjQAJ999hm8vb0xefLkCrdJSEjAgAEDYGZmBgsLCwwbNgzJycll6n311VdwcnKCXC7HsGHDVJ5+febMGfTq1Qs2NjawtLSEl5cXzp8//0zHcvDgQTg4OGDJkiV46aWX0KxZM/j7+2P9+vXKJ1uXN2S2YsUKuLi4KJdLh5cWLFiAhg0b4oUXXsCMGTPg4eFRpk03Nzd8/PHHKtsBwLp169CwYUMUFxer1B8wYABGjx4NALh16xYGDBgAe3t7mJmZoVOnTvjtt980OuYZM2bgzp07OHXqlPKJ7S1btsTYsWNx4cIFmJmZASh/iM/KykrZ03Tnzh0IgoCoqCh4eXlBJpNhzZo1MDExwb59+1S227VrF8zNzZGTU/Ksw7t372LYsGGwsrJCvXr1MGDAANy5c0ej49AUkxsiIgmIooicghxJXpoMRYSHh+PYsWPYs2cPoqOjcfTo0UqTjOLiYgwYMABpaWk4fPgwoqOjcfv2bQQGBqrUu3nzJrZt24aff/4Z+/fvR1xcHN555x3l+qysLISEhOCPP/7AyZMn0aJFC/Tt2xdZWVkVtu3t7Y3Q0NAK1zs4OCAxMRFHjhxR+/grEhMTg2vXriE6Ohq//PILgoODcfr0ady6dUtZ5/Lly7h06RJGjBhRZvuhQ4fiwYMHiI2NVZalpaVh//79CA4OBgA8evQIffv2RUxMDOLi4uDv74+AgAAkJCSoFWNxcTG2bt2K4OBgNGzYsMx6MzMzGBhoNoAzbdo0TJo0CVevXsXQoUPRv39/REZGqtTZvHkzBg4cCLlcjoKCAvj5+cHc3BxHjx7FsWPHYGZmBn9//2odCuSwFBGRBHILc+ERWfY//ZpwasQpyA3lT62XlZWFTZs2ITIyEj4+PgCAb775ptwvylIxMTH4888/ER8fDycnJwDAd999hxdffBFnzpxBp06dAAB5eXn47rvv4OjoCAD44osv0K9fPyxbtgwODg7o2bOnyn7XrVsHKysrHD58GP379y+37caNG6NBgwYVxjZ06FAcOHAAXl5ecHBwQJcuXeDj44NRo0bBwsLiqefjcaamptiwYYPKcJSbmxsiIyMxa9YsACVf8h4eHmjevHmZ7a2trdGnTx+Vc/vjjz/CxsYGr7zyinJ/bm5uym3mzZuHXbt2Yc+ePZgwYcJTY0xNTcXDhw/RqlUrjY6tMpMnT8bgwYOVy8HBwRg5ciRycnIgl8uRmZmJvXv3YteuXQCAqKgoFBcXY8OGDRAEAUDJZ8jKygqHDh1C7969tRbb49hzQ0RE5bp9+zYKCgrQuXNnZZmlpSVeeOGFCre5evUqnJyclIkNALRp0wZWVla4evWqsqxx48bKxAYAPD09UVxcjGvXrgEAkpOTMXbsWLRo0QKWlpawsLDAo0ePKu21+O6777Bo0aIK1+vr6+Obb77Bv//+iyVLlsDR0RELFy7Eiy++iMTExMpPxhNcXV3LzLMJDg5W9mKIoogtW7Yoe2HKExwcjB07diA/Px9ASTI0fPhw6OmVfDU/evQIU6ZMQevWrWFlZQUzMzNcvXpV7Z6b6pgs3LFjR5Xlvn37wtDQEHv27AEA7NixAxYWFvD19QUAXLx4ETdv3oS5uTnMzMxgZmaGevXqIS8vT6WXS9vYc0NEJAETAxOcGnFKsrZru5CQEDx48AArV66Es7MzjI2N4enpqZWhDEdHR4wcORIjR47EvHnz0LJlS6xduxZz586Fnp5emaSgoKCgzD5MTU3LlAUFBWHq1Kk4f/48cnNzcffu3TLDcY8LCAiAKIrYu3cvOnXqhKNHj+Kzzz5Trp8yZQqio6OxdOlSNG/eHCYmJhgyZIja58DW1hZWVlb4+++/n1pXEIQqHbeRkRGGDBmCyMhIDB8+HJGRkQgMDFQOdz169AgdOnTA5s2by42vujC5Ia3LURTBxKCwZhtVFKK0kz1HUQightuvRUwM9ZXdv1R7CYKg1tCQlJo2bQpDQ0OcOXMGjRs3BgBkZGTg+vXr6NGjR7nbtG7dGnfv3sXdu3eVvTdXrlxBeno62rRpo6yXkJCAe/fuKYe4Tp48CT09PWWv0LFjx/Dll1+ib9++AEompaampmr9GK2trdGgQQNkZ2cDKPnCTUpKgiiKyt+jCxcuqLWvRo0awcvLC5s3b0Zubi569eoFOzu7CuvLZDIMHjwYmzdvxs2bN/HCCy+gffv2yvXHjh1DaGgoBg0aBKAkUdBkIq6enh6GDx+O77//HhEREWWGEx89egSZTAYDAwPY2tqq9F7duHFDOSH4aYKDg9GrVy9cvnwZv//+O+bPn69c1759e0RFRcHOzk7job9nweSGtK774lhAfPplkdpkgjxclZW87zD/N+RCVqPt1yYdna2xfZwnExx6Zubm5ggJCcEHH3yAevXqwc7ODhEREdDT06vw8+Xr6wtXV1cEBwdjxYoVKCwsxDvvvAMvLy+VIQ2ZTIaQkBAsXboUmZmZePfddzFs2DA4ODgAAFq0aIHvv/8eHTt2RGZmJj744APlFU0VGTVqFBwdHSscmvrqq69w4cIFDBo0CM2aNVPO+7l8+TK++OILACWTku/fv48lS5ZgyJAh2L9/P/bt26f2F3NwcDAiIiKgUChUemEqq9+/f39cvnwZr7/+usq6Fi1aYOfOnQgICIAgCJg1a1aZq6ueZsGCBTh06BA8PDywYMECdOzYEYaGhjh69CgWLVqEM2fOwMrKCj179sSqVavg6emJoqIiTJ06Ve3LvHv06AEHBwcEBwejSZMmKleNBQcH49NPP8WAAQPw8ccfo1GjRvjnn3+wc+dOfPjhh2jUqJFGx6MuzrkhrZAZ8KNUW5z95yFyC6p+zw6ixy1fvhyenp7o378/fH190a1bN7Ru3RoyWfn/QAiCgJ9++gnW1tbo0aMHfH190bRpU0RFRanUa968OQYPHoy+ffuid+/eaNu2Lb788kvl+q+//hoPHz5E+/btMXLkSLz77ruV9oIAJb1Blc2d6dy5Mx49eoRx48bhxRdfhJeXF06ePIndu3fDy8sLQEnP05dffonVq1fDzc0Np0+fxpQpU9Q9XRgyZAgePHiAnJwcte5GXHpDwWvXrpW5qmr58uWwtrZG165dERAQAD8/P5WeHXXUq1cPJ0+exOuvv4758+fD3d0d3bt3x5YtW/Dpp58q772zbNkyODk5oXv37hgxYgSmTJkCuVy9nkVBEBAUFISLFy+WmWMkl8tx5MgRNG7cGIMHD0br1q0xZswY5OXlVWtPjiDq6u0JK5CZmQlLS0tkZGTUaBeZrsspyFFe+XFo6PGaH9NXZEO+tKTbPGdKAmBUdjxc1+UoitBxfsk9MK587Ae5ETtma4u8vDzEx8ejSZMmFSYFz4vs7Gw4Ojpi2bJlGDNmjNThkI6p7HdFk+9v/vUj7RMUgKBfs23qFSBHEGAiiiVf6vxiJ9KKuLg4/P333+jcuTMyMjKUN6QbMGCAxJERVYzfAKR13tu8pWnYxQnueXnYJIrgbBMi7Vm6dCmuXbsGIyMjdOjQAUePHoWNjY3UYRFViMkNaYWJgQnc7dwRlxInaRxxMhlyi/Igh5mkcRDpCnd3d5w7d07qMIg0wuSGtEIQBGzy34TcwlxJ2s/NfQDvXX0laZuIiGoXJjekNZLet6NAvfsxEEmpjl2/QaQxbf2O8PpdIqJqVnq/EHVvikZUV5XefVlf/9kuSmHPDRFRNdPX14eVlRVSUlIAlNz7gzdZJFJVXFyM+/fvQy6Xa/y08icxuSHdo8gBDLKljqLmKQphgjzkwljqSKgcpXfeLU1wiKgsPT09NG7c+JmTfyY3pHtWtgXq4NwGOYCrMuBMcUtA9JM6HHqCIAho0KAB7Ozsyn0gIRGVPIiz9Knoz4LJDemG5+ApxzWlk9515BTkAMaWUodC5dDX13/m+QREVDkmN6QbHu/C/OBmnUx2crIzIV/ZSuowiIgkx+SGdI+hvORV1ygKpY6AiKhW4KXgREREpFMkT25Wr14NFxcXyGQyeHh44PTp05XWX7FiBV544QWYmJjAyckJ7733HvLy8mooWiIiIqrtJE1uoqKiEB4ejoiICJw/fx5ubm7w8/Or8FLJyMhITJs2DREREbh69Sq+/vprREVFYcaMGTUcOREREdVWkiY3y5cvx9ixYxEWFoY2bdpg7dq1kMvl2LhxY7n1jx8/jm7dumHEiBFwcXFB7969ERQU9NTeHiIiIqo7JEtuFAoFzp07B19f3/8PRk8Pvr6+OHHiRLnbdO3aFefOnVMmM7dv38avv/6Kvn0rfmBifn4+MjMzVV5ERESkuyS7Wio1NRVFRUWwt7dXKbe3t8fff/9d7jYjRoxAamoqXn75ZYiiiMLCQowbN67SYalFixZh7ty5Wo2diIiIai/JJxRr4tChQ1i4cCG+/PJLnD9/Hjt37sTevXsxb968CreZPn06MjIylK+7d+/WYMRERERU0yTrubGxsYG+vj6Sk5NVypOTk5XPYHnSrFmzMHLkSLzxxhsAAFdXV2RnZ+PNN9/EzJkzy71ls7GxMYyN+awdIiKiukKynhsjIyN06NABMTExyrLi4mLExMTA09Oz3G1ycnLKJDCltzEX6+CzhIiIiKgsSe9QHB4ejpCQEHTs2BGdO3fGihUrkJ2djbCwMADAqFGj4OjoiEWLFgEAAgICsHz5cri7u8PDwwM3b97ErFmzEBAQwGe1EBEREQCJk5vAwEDcv38fs2fPRlJSEtq1a4f9+/crJxknJCSo9NR89NFHEAQBH330Ef777z/Y2toiICAACxYskOoQiIiIqJYRxDo2npOZmQlLS0tkZGTAwsJC6nBIS3IKcuAR6QEAODXiFOR18NlSOY8yIF/auOT9lATIzfhUcCLSHZp8fz9XV0sRERERPQ2fCk46J7cwV+oQJJFbmAsIAkzqVmcsEVEZTG5I53hv85Y6BOm4OME9Lw9rmOAQUR3GYSnSCSYGJnC3c5c6jFohTiZDXlGe1GEQEUmGPTekEwRBwCb/TXV2SAoA0jJS0GdvgNRhEBFJjskN6QxBEOrkVVKlcg1kUodARFQrcFiKiIiIdAqTGyIiItIpTG6IiIhIpzC5ISIiIp3C5IaIiIh0CpMbIiIi0ilMboiIiEinMLkhIiIincKb+BHpooIcQJEtTduGckAQpGmbiAhMboh0ksmX7QGpHp7p1AUYvZ8JDhFJhsNSRLqitjx64u7Jkp4jIiKJsOeGSFc81lOSO+lvyOXmNdu+IgdY2rxm2yQiKgeTGyJdZGRa8iIiqoM4LEVEREQ6hT03RDooR1EEE4PCmm1UUYjSWT85ikIANdx+LWFiqA+Bk6mJJMXkhkgHdV8cC4hGNdqmCfJwVVbyvsP835ALWY22X1t0dLbG9nGeTHCIJMRhKSIdITPgr3NtcPafh8gtKJI6DKI6jT03RDri8Z6Cc7N8YWJgUrMBKLKBpf9r/yPfOjehOUdRhI7zf5M6DCICkxsinWRiqA+5YU3/ev9/e3IjA8CIf16ISBrsxyYiIiKdwuSGiIiIdAr7jYl0UG5hbs03WpgLCAJMRBG8ToiIpKRxcuPl5YUxY8Zg6NChMDGp4QmLRKQW723e0jTs4gT3vDxsYoJDRBLSeFjK3d0dU6ZMgYODA8aOHYuTJ09WR1xEpCETAxO427lLHQbiZDLkFuVJHQYR1WEa99ysWLECS5cuxZ49e7Bp0yb06NEDzZs3x+jRozFy5EjY29tXR5xE9BSCIGCT/yZphqQA5OY+gPeuvpK0TUT0uCrNuTEwMMDgwYMxePBgpKSkYN26dZg1axZmzJiBvn374t1330XPnj21HSsRPYUgCJAbyp9esToU5EjTLhHRE57paqnTp08jIiICy5Ytg52dHaZPnw4bGxv0798fU6ZM0VaMRERERGrTuOcmJSUF33//Pb755hvcuHEDAQEB2LJlC/z8/JR3SA0NDYW/vz+WLl2q9YCJiIiIKqNxctOoUSM0a9YMo0ePRmhoKGxtbcvUadu2LTp16qSVAImIiIg0oXFyExMTg+7du1dax8LCArGxsVUOioiIiKiqNJ5zExERgfT09DLlmZmZnERMREREktM4uTl8+DAUCkWZ8ry8PBw9elQrQRERERFVldrDUpcuXQIAiKKIK1euICkpSbmuqKgI+/fvh6Ojo/YjJCIiItKA2slNu3btIAgCBEEod/jJxMQEX3zxhVaDIyIiItKU2slNfHw8RFFE06ZNcfr0aZWrpIyMjGBnZwd9ff1qCZKIiIhIXWonN87OzgCA4uLiaguGiIiI6Fmpldzs2bMHffr0gaGhIfbs2VNp3VdffVUrgRERERFVhVrJzcCBA5GUlAQ7OzsMHDiwwnqCIKCoqEhbsRERERFpTK3k5vGhKA5LEdFTKXIAg2ypo6hZikKYIO9/77NRxecSa4ehHPjf43CI6iIJf/uISGetbAuIotRR1Cg5gKuy/y1I/Vg9py7A6P1McKjOUiu5+fzzz9Xe4bvvvlvlYIjoOWZgInUEVOruSaAgBzAylToSIkmoldx89tlnau1MEAQmN0R11eO9BB/crHPJTo6iEB3m/wYAOPeRL+RGEnSMK3KApc1rvl2iWkat3774+PjqjoOIdImhvORVpxQiF/8blzIyBaRIbogIQBWeLUVERERUm6n1r0V4eDjmzZsHU1NThIeHV1p3+fLlWgmMiIiIqCrUSm7i4uJQUFCgfF8RgTPziYiISGJqJTexsbHlviciIiKqbZ5pxtvdu3cBAE5OTloJhoh0Q25hrmRtmxiYsBeZqI7TOLkpLCzE3Llz8fnnn+PRo0cAADMzM0ycOBEREREwNDTUepBE9Hzx3uYtWdvudu7Y5L+JCQ5RHaZxcjNx4kTs3LkTS5YsgaenJwDgxIkTmDNnDh48eIA1a9ZoPUgiqv1MDEzgbueOuJSK5+XVhLiUOOQW5kJe5y5FJ6JSGic3kZGR2Lp1K/r06aMsa9u2LZycnBAUFKRxcrN69Wp8+umnSEpKgpubG7744gt07ty5wvrp6emYOXMmdu7cibS0NDg7O2PFihXo27evpodCRFokCAI2+W+SbEgqtzBX0h4jIqo9NE5ujI2N4eLiUqa8SZMmMDIy0mhfUVFRCA8Px9q1a+Hh4YEVK1bAz88P165dg52dXZn6CoUCvXr1gp2dHX788Uc4Ojrin3/+gZWVlaaHQUTVQBAE9pgQkeQ0vonfhAkTMG/ePOTn5yvL8vPzsWDBAkyYMEGjfS1fvhxjx45FWFgY2rRpg7Vr10Iul2Pjxo3l1t+4cSPS0tKwe/dudOvWDS4uLvDy8oKbm5umh0FEREQ6Sq2em8GDB6ss//bbb2jUqJEyqbh48SIUCgV8fHzUblihUODcuXOYPn26skxPTw++vr44ceJEudvs2bMHnp6eGD9+PH766SfY2tpixIgRmDp1KvT19cvdJj8/XyURy8zMVDtGIiIiev6oldxYWlqqLL/22msqy1W5FDw1NRVFRUWwt7dXKbe3t8fff/9d7ja3b9/G77//juDgYPz666+4efMm3nnnHRQUFCAiIqLcbRYtWoS5c+dqHB8RERE9n9RKbr755pvqjkMtxcXFsLOzw7p166Cvr48OHTrgv//+w6efflphcjN9+nSVR0ZkZmbyvjxEREQ6TLLH1trY2EBfXx/Jyckq5cnJyXBwcCh3mwYNGsDQ0FBlCKp169ZISkqCQqEod0KzsbExjI2NtRs8ERER1VpVeir4jz/+iGHDhqFLly5o3769yktdRkZG6NChA2JiYpRlxcXFiImJUd4/50ndunXDzZs3UVxcrCy7fv06GjRooPGVWkRERKSbNE5uPv/8c4SFhcHe3h5xcXHo3Lkz6tevj9u3b6vc+0Yd4eHhWL9+PTZt2oSrV6/i7bffRnZ2NsLCwgAAo0aNUplw/PbbbyMtLQ2TJk3C9evXsXfvXixcuBDjx4/X9DCIiIhIR2k8LPXll19i3bp1CAoKwrfffosPP/wQTZs2xezZs5GWlqbRvgIDA3H//n3Mnj0bSUlJaNeuHfbv36+cZJyQkAA9vf/Pv5ycnHDgwAG89957aNu2LRwdHTFp0iRMnTpV08MgIiIiHaVxcpOQkICuXbsCAExMTJCVlQUAGDlyJLp06YJVq1ZptL8JEyZUeH+cQ4cOlSnz9PTEyZMnNQuaiIiI6gyNh6UcHByUPTSNGzdWJhrx8fEQRVG70RERERFpSOPkpmfPntizZw8AICwsDO+99x569eqFwMBADBo0SOsBEhEREWlC42GpdevWKa9WGj9+POrXr4/jx4/j1VdfxVtvvaX1AImIiIg0oXFyo6enpzLJd/jw4Rg+fLhWgyIiIiKqqirdxO/hw4f4+uuvcfXqVQBAmzZtEBYWhnr16mk1OCIiIiJNaTzn5siRI2jSpAk+//xzPHz4EA8fPsTnn3+OJk2a4MiRI9URIxEREZHaNO65GT9+PIYNG4Y1a9YoH4NQVFSEd955B+PHj8eff/6p9SCJiIiI1KVxz83Nmzfx/vvvqzzfSV9fH+Hh4bh586ZWgyMiIiLSlMbJTfv27ZVzbR539epVuLm5aSUoIiIioqpSa1jq0qVLyvfvvvsuJk2ahJs3b6JLly4AgJMnT2L16tX45JNPqidKIiIiIjWpldy0a9cOgiCo3IH4ww8/LFNvxIgRCAwM1F50RERERBpSK7mJj4+v7jiIiIiItEKt5MbZ2bm64yAi0prcwlwJ2iwCBAUgGtZ420Skqko38bt16xZWrFihchO/SZMmoVmzZloNjoioKry3eUvSrnkroDDHGdn5vpK0D0Uh5P97m6MoBFAoTRx1nImhPgRBkDqMOk3j5ObAgQN49dVX0a5dO3Tr1g0AcOzYMbz44ov4+eef0atXL60HSUT0NCYGJnC3c0dcSpykcRjI/0GnhfsA0ajG2zZBHq7KSt53mP8bciGr8RgI6Ohsje3jPJngSEjj5GbatGl47733ylwZNW3aNEydOpXJDRFJQhAEbPLfJMmQFADkFOTgle2vSNI21S5n/3mI3IIiyI2qNDhCWqDxmb969Sq2bdtWpnz06NFYsWKFNmIiIqoSQRAgN5Q/vWI1OzfLFyYGJjXfsCIbWPq/GD7yBYxMaz6GOixHUYSO83+TOgxCFZIbW1tbXLhwAS1atFApv3DhAuzs7LQWGBHR88rEUB9yQyn+a///NuVGBgB7DqiO0viTP3bsWLz55pu4ffs2unbtCqBkzs3ixYsRHh6u9QCJiIiINKFxcjNr1iyYm5tj2bJlmD59OgCgYcOGmDNnDt59912tB0hERESkCY2Sm8LCQkRGRmLEiBF47733kJWVBQAwNzevluCIiKiKFDnStW0oB3ilEElIo+TGwMAA48aNU97fhkkNEVEttbS5dG07dQFG72eCQ5LR+KngnTt3RlyctPeRICKichjKSxILqd09CRRI2HNEdZ7Gc27eeecdvP/++/j333/RoUMHmJqqXmrYtm1brQVHREQaEISSHhOpEgtFjrQ9RkT/o3FyM3z4cABQmTxc+sRwQRBQVFSkveiIiEgzgsD721Cdp3FywyeEExERUW2mUXKTmZmJ69evQ6FQoHPnzrC1ta2uuIiIiIiqRO3k5sKFC+jbty+Sk5MhiiLMzc2xbds2+Pn5VWd8RERERBpR+2qpqVOnokmTJvjjjz9w7tw5+Pj4YMKECdUZGxEREZHG1O65OXfuHA4ePIj27dsDADZu3Ih69eohMzMTFhYW1RYgERERkSbU7rlJS0tDo0aNlMtWVlYwNTXFgwcPqiUwIiIioqrQaELxlStXkJSUpFwWRRFXr15VPoYB4H1uiIiISFoaJTc+Pj4QRVGlrH///rzPDREREdUaaic3vL8NERERPQ/UTm6cnZ2rMw4iIiIirdD4wZlEREREtRmTGyIiItIpTG6IiIhIp2j84EwiIqpcbmGupO2bGJhAEARJYyCSEpMbIiIt897mLWn77nbu2OS/iQkO1VlqJTfu7u5q/5KcP3/+mQIiInoemRiYwN3OHXEpcVKHgriUOOQW5kJuKJc6FCJJqJXcDBw4sJrDICJ6vgmCgE3+myQdksotzJW814ioNlAruYmIiKjuOIiInnuCILC3hKgW4NVSREREpFM0nlBcVFSEzz77DNu2bUNCQgIUCoXK+rS0NK0FR0RERKQpjXtu5s6di+XLlyMwMBAZGRkIDw/H4MGDoaenhzlz5lRDiERERETq0zi52bx5M9avX4/3338fBgYGCAoKwoYNGzB79mycPHmyOmIkIiIiUpvGyU1SUhJcXV0BAGZmZsjIyAAA9O/fH3v37tVudEREREQa0ji5adSoERITEwEAzZo1w8GDBwEAZ86cgbGxsXajIyIiItKQxsnNoEGDEBMTAwCYOHEiZs2ahRYtWmDUqFEYPXq01gMkIiIi0oTGV0t98sknyveBgYFwdnbG8ePH0aJFCwQEBGg1OCIiIiJNaZzc5OXlQSaTKZe7dOmCLl26aDUoIiIioqrSeFjKzs4OISEhiI6ORnFxcXXERERERFRlGic3mzZtQk5ODgYMGABHR0dMnjwZZ8+erY7YiIiIiDRWpQnF27dvR3JyMhYuXIgrV66gS5cuaNmyJT7++OPqiJGIiIhIbVV+tpS5uTnCwsJw8OBBXLp0Caamppg7d642YyMiIiLSWJWTm7y8PGzbtg0DBw5E+/btkZaWhg8++KBK+1q9ejVcXFwgk8ng4eGB06dPq7Xd1q1bIQgCBg4cWKV2iYiISPdonNwcOHAAISEhsLe3x9tvvw17e3scPHgQ//zzj8pl4uqKiopCeHg4IiIicP78ebi5ucHPzw8pKSmVbnfnzh1MmTIF3bt317hNIiIi0l1VmnOTm5uL7777DklJSfjqq6/Qo0ePKgewfPlyjB07FmFhYWjTpg3Wrl0LuVyOjRs3VrhNUVERgoODMXfuXDRt2rTKbRMREZHu0fg+N8nJyTA3N9dK4wqFAufOncP06dOVZXp6evD19cWJEycq3O7jjz+GnZ0dxowZg6NHj2olFiIiItINaiU3mZmZsLCwAACIoojMzMwK65bWU0dqaiqKiopgb2+vUm5vb4+///673G3++OMPfP3117hw4YJabeTn5yM/P1+5XFnsRERE9PxTK7mxtrZGYmIi7OzsYGVlBUEQytQRRRGCIKCoqEjrQZbKysrCyJEjsX79etjY2Ki1zaJFi3gVFxERUR2iVnLz+++/o169esr35SU3VWFjYwN9fX0kJyerlCcnJ8PBwaFM/Vu3buHOnTsqz7AqvUuygYEBrl27hmbNmqlsM336dISHhyuXMzMz4eTkpJX4iYiIqPZRK7nx8vJSvvf29tZa40ZGRujQoQNiYmKUl3MXFxcjJiYGEyZMKFO/VatW+PPPP1XKPvroI2RlZWHlypXlJi3GxsYwNjbWWsxERERUu2k8obhFixYIDg5GcHAwWrRo8cwBhIeHIyQkBB07dkTnzp2xYsUKZGdnIywsDAAwatQoODo6YtGiRZDJZHjppZdUtreysgKAMuVERHVZbmFuzTdamAsIAkxEEYIip+bbL2UoB7Q0wkDPJ42Tm3feeQeRkZGYN28e2rdvj9dffx2BgYHlDiOpIzAwEPfv38fs2bORlJSEdu3aYf/+/cpJxgkJCdDTq/K9BomI6iTvbd7SNOziBPe8PGxa2hySpRdOXYDR+5ng1GGCKIpiVTa8fv06Nm/ejC1btiA+Ph6vvPIKXn/9dYwaNUrbMWpVZmYmLC0tkZGRodGVXUREtZ0oigjZH4K4lDipQ8GpO3chr9rXi3bMuAcYmdZokzmKQrSZfQAAcOVjP8iNNO4/oEpo8v1d5eTmcSdPnsTbb7+NS5cuVevVUtrA5IaIdJkoitIMSaFkKKy0x+jU0FjIDUxqNgBFDrC0ecl7Jjc6R5Pv72c686dPn0ZkZCSioqKQmZmJoUOHPsvuiIjoGQmCALmhXOowSua91IY4qE7SOLl5cjiqZ8+eWLx4MQYPHgwzM7PqiJGIiIhIbRonN61atUKnTp0wfvx4DB8+vMzdhYmIiIikpFFyU1RUhK+++gpDhgyBtbV1dcVEREREVGUaXWOtr6+PiRMnIj09vZrCISIiIno2Gt9A5qWXXsLt27erIxYiIiKiZ6ZxcjN//nxMmTIFv/zyCxITE5GZmanyIiIiIpKSxhOK+/btCwB49dVXVR6gWRNPBSciIiJ6Go2Tm9jY2OqIg4iIiEgrNE5uHn9COBEREVFto3Fyc+TIkUrX9+jRo8rBEBERET0rjZMbb2/vMmWPz73hnBsiIiKSksZXSz18+FDllZKSgv3796NTp044ePBgdcRIREREpDaNe24sLS3LlPXq1QtGRkYIDw/HuXPntBIYERERUVVo3HNTEXt7e1y7dk1buyMiIiKqEo17bi5duqSyLIoiEhMT8cknn6Bdu3baiouIiIioSjRObtq1awdBECCKokp5ly5dsHHjRq0FRkRERFQVGic38fHxKst6enqwtbWFTCbTWlBEREREVaVxcuPs7FwdcRARERFphdoTik+cOIFffvlFpey7775DkyZNYGdnhzfffBP5+flaD5CIiIhIE2onNx9//DEuX76sXP7zzz8xZswY+Pr6Ytq0afj555+xaNGiagmSiIiISF1qJzcXLlyAj4+Pcnnr1q3w8PDA+vXrER4ejs8//xzbtm2rliCJiIiI1KX2nJuHDx/C3t5euXz48GH06dNHudypUyfcvXtXu9EREdFzKbcwt+YbLcwFBAEmogjh6bVJh6md3Njb2yM+Ph5OTk5QKBQ4f/485s6dq1yflZUFQ0PDagmSiIieL97bvKVp2MUJ7nl52MQEp05Te1iqb9++mDZtGo4ePYrp06dDLpeje/fuyvWXLl1Cs2bNqiVIIiKq/UwMTOBu5y51GIiTyZBblCd1GCQhtXtu5s2bh8GDB8PLywtmZmbYtGkTjIyMlOs3btyI3r17V0uQRERU+wmCgE3+m6QZkgKQm/sA3rv6StI21S5qJzc2NjY4cuQIMjIyYGZmBn19fZX127dvh5mZmdYDJCKi54cgCJAbyqVpvCBHmnap1tHKU8EBoF69es8cDBEREdGz0tpTwYmIiIhqAyY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU7R+D43REREVLkcRZHUIUjKxFAfgiDd072Y3BAREWlZx/m/SR2CpDo6W2P7OE/JEhwOSxEREWmBiaE+OjpbSx1GrXD2n4fILZCu94o9N0RERFogCAK2j/OU9EtdajmKolrRa8XkhoiIdI8iBzDIrvFmBaDkwaESzjchJjdERKSLVrYFRFGatp26AKP3M8GREOfcEBGRbjAwkTqCEndPAgU5UkdRp7HnhoiIdMPjPSUf3Kz5ZEeRAyxtXrNtUrmY3BARke4xlJe8qE7isBQRERHpFCY3REREpFOY3BAREZFO4ZwbIiLSObmFuTXfaGEuIAgwEUXwInBpMbkhIiKd473NW5qGXZzgnpeHTUxwJMVhKSIi0gkmBiZwt3OXOgzEyWTILcqTOow6jT03RESkEwRBwCb/TdIMSQHIzX0A7119JWmbVDG5ISIinSEIQsmznaTAuxLXGhyWIiIiIp3C5IaIiIh0CpMbIiIi0im1IrlZvXo1XFxcIJPJ4OHhgdOnT1dYd/369ejevTusra1hbW0NX1/fSusTERFR3SJ5chMVFYXw8HBERETg/PnzcHNzg5+fH1JSUsqtf+jQIQQFBSE2NhYnTpyAk5MTevfujf/++6+GIyciIqLaSPLkZvny5Rg7dizCwsLQpk0brF27FnK5HBs3biy3/ubNm/HOO++gXbt2aNWqFTZs2IDi4mLExMTUcORERERUG0ma3CgUCpw7dw6+vr7KMj09Pfj6+uLEiRNq7SMnJwcFBQWoV69edYVJREREzxFJ73OTmpqKoqIi2Nvbq5Tb29vj77//VmsfU6dORcOGDVUSpMfl5+cjPz9fuZyZmVn1gImIiKjWk3xY6ll88skn2Lp1K3bt2gWZTFZunUWLFsHS0lL5cnJyquEoiYiIqCZJmtzY2NhAX18fycnJKuXJyclwcHCodNulS5fik08+wcGDB9G2bdsK602fPh0ZGRnK1927d7USOxEREdVOkiY3RkZG6NChg8pk4NLJwZ6enhVut2TJEsybNw/79+9Hx44dK23D2NgYFhYWKi8iIiLSXZI/Wyo8PBwhISHo2LEjOnfujBUrViA7OxthYWEAgFGjRsHR0RGLFi0CACxevBizZ89GZGQkXFxckJSUBAAwMzODmZmZZMdBREREtYPkyU1gYCDu37+P2bNnIykpCe3atcP+/fuVk4wTEhKgp/f/HUxr1qyBQqHAkCFDVPYTERGBOXPm1GToREREVAtJntwAwIQJEzBhwoRy1x06dEhl+c6dO9UfEBERET23nuurpYiIiIiexOSGiIiIdAqTGyIiItIpTG6IiIhIpzC5ISIiIp3C5IaIiIh0CpMbIiIi0im14j43REREuiS3MBcoyJGsfRMDEwiCIFn7UmNyQ0REpGXeu/pK2r67nTs2+W+qswkOh6WIiIi0wERfBve8PKnDAADEpcSV9B7VUey5ISIi0gJBELApMQW5ggB8cBMwlNd4DLmFufDe5l3j7dY2TG6IiIi0RAAgF0XAwESS5IZKcFiKiIiIdAqTGyIiItIpTG6IiIhIpzC5ISIiIp3CCcVERETappDoBn51+PLvxzG5ISIi0ralzaVpVxAAF6eS96IoTQy1AIeliIiItMFQDjh1kTqK/1eHe3HYc0NERKQNggCM3i/pM6WQ8wD4qZ907dcSTG6IiIi0RRAAI1Pp2q/DvTWP47AUERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckNEREQ6hQ/OJCIi0kG5hbk1/oTy3MIiQFAAomGNtvskJjdEREQ6yHtXX0naNW8FFOY4QxT9JGkf4LAUERGRzjDRl8E9L0/qMGAg/wd5RdLFwZ4bIiIiHSEIAjYlpiBXEIAPbgKG8hptPy33Efrs8qnRNsvD5IaIiEiHCADkoggYmNR4cpNbUFSj7VWEw1JERESkU5jcEBERkU5hckNEREQ6hckNERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckNEREQ6hckNERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqlViQ3q1evhouLC2QyGTw8PHD69OlK62/fvh2tWrWCTCaDq6srfv311xqKlIiIiGo7yZObqKgohIeHIyIiAufPn4ebmxv8/PyQkpJSbv3jx48jKCgIY8aMQVxcHAYOHIiBAwfir7/+quHIiYiIqDaSPLlZvnw5xo4di7CwMLRp0wZr166FXC7Hxo0by62/cuVK+Pv744MPPkDr1q0xb948tG/fHqtWrarhyImIiKg2kjS5USgUOHfuHHx9fZVlenp68PX1xYkTJ8rd5sSJEyr1AcDPz6/C+vn5+cjMzFR5ERERke6SNLlJTU1FUVER7O3tVcrt7e2RlJRU7jZJSUka1V+0aBEsLS2VLycnJ+0ET0RERLWSgdQBVLfp06cjPDxcuZyZmckEh4iIdJOhHJhx7//f1zBrmSkODT2ufC8VSZMbGxsb6OvrIzk5WaU8OTkZDg4O5W7j4OCgUX1jY2MYGxtrJ2AiIqLaTBAAI+mSCj09PdSXm0vWvjIOKRs3MjJChw4dEBMToywrLi5GTEwMPD09y93G09NTpT4AREdHV1ifiIiI6hbJh6XCw8MREhKCjh07onPnzlixYgWys7MRFhYGABg1ahQcHR2xaNEiAMCkSZPg5eWFZcuWoV+/fti6dSvOnj2LdevWSXkYREREVEtIntwEBgbi/v37mD17NpKSktCuXTvs379fOWk4ISEBenr/38HUtWtXREZG4qOPPsKMGTPQokUL7N69Gy+99JJUh0BERES1iCCKoih1EDUpMzMTlpaWyMjIgIWFhdThEBERkRo0+f6W/CZ+RERERNrE5IaIiIh0CpMbIiIi0ilMboiIiEinMLkhIiIincLkhoiIiHQKkxsiIiLSKUxuiIiISKcwuSEiIiKdIvnjF2pa6Q2ZMzMzJY6EiIiI1FX6va3OgxXqXHKTlZUFAHBycpI4EiIiItJUVlYWLC0tK61T554tVVxcjHv37sHc3ByCIGh135mZmXBycsLdu3fr5HOr6vrxAzwHPP66ffwAz0FdP36g+s6BKIrIyspCw4YNVR6oXZ4613Ojp6eHRo0aVWsbFhYWdfZDDfD4AZ4DHn/dPn6A56CuHz9QPefgaT02pTihmIiIiHQKkxsiIiLSKUxutMjY2BgREREwNjaWOhRJ1PXjB3gOePx1+/gBnoO6fvxA7TgHdW5CMREREek29twQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3GjJ6tWr4eLiAplMBg8PD5w+fVrqkKrNkSNHEBAQgIYNG0IQBOzevVtlvSiKmD17Nho0aAATExP4+vrixo0b0gRbDRYtWoROnTrB3NwcdnZ2GDhwIK5du6ZSJy8vD+PHj0f9+vVhZmaG1157DcnJyRJFrF1r1qxB27ZtlTfo8vT0xL59+5TrdfnYy/PJJ59AEARMnjxZWabr52DOnDkQBEHl1apVK+V6XT/+Uv/99x9ef/111K9fHyYmJnB1dcXZs2eV63X5b6GLi0uZz4AgCBg/fjwA6T8DTG60ICoqCuHh4YiIiMD58+fh5uYGPz8/pKSkSB1atcjOzoabmxtWr15d7volS5bg888/x9q1a3Hq1CmYmprCz88PeXl5NRxp9Th8+DDGjx+PkydPIjo6GgUFBejduzeys7OVdd577z38/PPP2L59Ow4fPox79+5h8ODBEkatPY0aNcInn3yCc+fO4ezZs+jZsycGDBiAy5cvA9DtY3/SmTNn8NVXX6Ft27Yq5XXhHLz44otITExUvv744w/lurpw/A8fPkS3bt1gaGiIffv24cqVK1i2bBmsra2VdXT5b+GZM2dUfv7R0dEAgKFDhwKoBZ8BkZ5Z586dxfHjxyuXi4qKxIYNG4qLFi2SMKqaAUDctWuXcrm4uFh0cHAQP/30U2VZenq6aGxsLG7ZskWCCKtfSkqKCEA8fPiwKIolx2toaChu375dWefq1asiAPHEiRNShVmtrK2txQ0bNtSpY8/KyhJbtGghRkdHi15eXuKkSZNEUawbP/+IiAjRzc2t3HV14fhFURSnTp0qvvzyyxWur2t/CydNmiQ2a9ZMLC4urhWfAfbcPCOFQoFz587B19dXWaanpwdfX1+cOHFCwsikER8fj6SkJJXzYWlpCQ8PD509HxkZGQCAevXqAQDOnTuHgoIClXPQqlUrNG7cWOfOQVFREbZu3Yrs7Gx4enrWqWMfP348+vXrp3KsQN35+d+4cQMNGzZE06ZNERwcjISEBAB15/j37NmDjh07YujQobCzs4O7uzvWr1+vXF+X/hYqFAr88MMPGD16NARBqBWfASY3zyg1NRVFRUWwt7dXKbe3t0dSUpJEUUmn9JjryvkoLi7G5MmT0a1bN7z00ksASs6BkZERrKysVOrq0jn4888/YWZmBmNjY4wbNw67du1CmzZt6sSxA8DWrVtx/vx5LFq0qMy6unAOPDw88O2332L//v1Ys2YN4uPj0b17d2RlZdWJ4weA27dvY82aNWjRogUOHDiAt99+G++++y42bdoEoG79Ldy9ezfS09MRGhoKoHb8DtS5p4ITadP48ePx119/qcw3qAteeOEFXLhwARkZGfjxxx8REhKCw4cPSx1Wjbh79y4mTZqE6OhoyGQyqcORRJ8+fZTv27ZtCw8PDzg7O2Pbtm0wMTGRMLKaU1xcjI4dO2LhwoUAAHd3d/z1119Yu3YtQkJCJI6uZn399dfo06cPGjZsKHUoSuy5eUY2NjbQ19cvMws8OTkZDg4OEkUlndJjrgvnY8KECfjll18QGxuLRo0aKcsdHBygUCiQnp6uUl+XzoGRkRGaN2+ODh06YNGiRXBzc8PKlSvrxLGfO3cOKSkpaN++PQwMDGBgYIDDhw/j888/h4GBAezt7XX+HDzJysoKLVu2xM2bN+vEZwAAGjRogDZt2qiUtW7dWjk8V1f+Fv7zzz/47bff8MYbbyjLasNngMnNMzIyMkKHDh0QExOjLCsuLkZMTAw8PT0ljEwaTZo0gYODg8r5yMzMxKlTp3TmfIiiiAkTJmDXrl34/fff0aRJE5X1HTp0gKGhoco5uHbtGhISEnTmHDypuLgY+fn5deLYfXx88Oeff+LChQvKV8eOHREcHKx8r+vn4EmPHj3CrVu30KBBgzrxGQCAbt26lbkFxPXr1+Hs7AygbvwtBIBvvvkGdnZ26Nevn7KsVnwGamTaso7bunWraGxsLH777bfilStXxDfffFO0srISk5KSpA6tWmRlZYlxcXFiXFycCEBcvny5GBcXJ/7zzz+iKIriJ598IlpZWYk//fSTeOnSJXHAgAFikyZNxNzcXIkj1463335btLS0FA8dOiQmJiYqXzk5Oco648aNExs3biz+/vvv4tmzZ0VPT0/R09NTwqi1Z9q0aeLhw4fF+Ph48dKlS+K0adNEQRDEgwcPiqKo28dekcevlhJF3T8H77//vnjo0CExPj5ePHbsmOjr6yva2NiIKSkpoijq/vGLoiiePn1aNDAwEBcsWCDeuHFD3Lx5syiXy8UffvhBWUfX/xYWFRWJjRs3FqdOnVpmndSfASY3WvLFF1+IjRs3Fo2MjMTOnTuLJ0+elDqkahMbGysCKPMKCQkRRbHkEshZs2aJ9vb2orGxsejj4yNeu3ZN2qC1qLxjByB+8803yjq5ubniO++8I1pbW4tyuVwcNGiQmJiYKF3QWjR69GjR2dlZNDIyEm1tbUUfHx9lYiOKun3sFXkyudH1cxAYGCg2aNBANDIyEh0dHcXAwEDx5s2byvW6fvylfv75Z/Gll14SjY2NxVatWonr1q1TWa/rfwsPHDggAij3mKT+DAiiKIo100dEREREVP0454aIiIh0CpMbIiIi0ilMboiIiEinMLkhIiIincLkhoiIiHQKkxsiIiLSKUxuiIiISKcwuSGi50poaCgGDhwodRhEVIvxqeBEVGsIglDp+oiICKxcuRK89ygRVYbJDRHVGomJicr3UVFRmD17tsrDCc3MzGBmZiZFaET0HOGwFBHVGg4ODsqXpaUlBEFQKTMzMyszLOXt7Y2JEydi8uTJsLa2hr29PdavX4/s7GyEhYXB3NwczZs3x759+1Ta+uuvv9CnTx+YmZnB3t4eI0eORGpqag0fMRFVByY3RPTc27RpE2xsbHD69GlMnDgRb7/9NoYOHYquXbvi/Pnz6N27N0aOHImcnBwAQHp6Onr27Al3d3ecPXsW+/fvR3JyMoYNGybxkRCRNjC5IaLnnpubGz766CO0aNEC06dPh0wmg42NDcaOHYsWLVpg9uzZePDgAS5dugQAWLVqFdzd3bFw4UK0atUK7u7u2LhxI2JjY3H9+nWJj4aInhXn3BDRc69t27bK9/r6+qhfvz5cXV2VZfb29gCAlJQUAMDFixcRGxtb7vydW7duoWXLltUcMRFVJyY3RPTcMzQ0VFkWBEGlrPQqrOLiYgDAo0ePEBAQgMWLF5fZV4MGDaoxUiKqCUxuiKjOad++PXbs2AEXFxcYGPDPIJGu4ZwbIqpzxo8fj7S0NAQFBeHMmTO4desWDhw4gLCwMBQVFUkdHhE9IyY3RFTnNGzYEMeOHUNRURF69+4NV1dXTJ48GVZWVtDT459FouedIPJWn0RERKRD+C8KERER6RQmN0RERKRTmNwQERGRTmFyQ0RERDqFyQ0RERHpFCY3REREpFOY3BAREZFOYXJDREREOoXJDREREekUJjdERESkU5jcEBERkU5hckNEREQ65f8AopLGRFalu1QAAAAASUVORK5CYII=", - "text/plain": [ - "

" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "for site, result in data.items():\n", - " timeline = result[\"timeline\"]\n", - " km_estimate = result[\"km_estimate\"]\n", - " plt.step(timeline, km_estimate, where=\"post\", label=f\"{site}: Survival Curve\")\n", - "\n", - "plt.xlabel(\"Time\")\n", - "plt.ylabel(\"Survival Probability\")\n", - "plt.title(\"Kaplan-Meier Survival Curve\")\n", - "plt.legend()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b43334bb-c77d-40f4-99b4-c2f9adb37d7e", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c760a0ba-5450-4ae8-9c2d-774c9bf84515", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nvflare_example", - "language": "python", - "name": "nvflare_example" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.16" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/hello-world/hello-km/demo/km.json b/examples/hello-world/hello-km/demo/km.json deleted file mode 100644 index 39dd3694f3..0000000000 --- a/examples/hello-world/hello-km/demo/km.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "site-2": { - "timeline": [ - 0.0, - 10.0, - 25.0, - 30.0, - 40.0, - 50.0, - 60.0, - 70.0 - ], - "km_estimate": [ - 1.0, - 0.8571428571428572, - 0.7142857142857143, - 0.7142857142857143, - 0.5357142857142858, - 0.5357142857142858, - 0.26785714285714285, - 0.0 - ], - "event_count": [ - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1 - ], - "survival_rate": [ - 0.0, - 0.1428571428571428, - 0.2857142857142857, - 0.2857142857142857, - 0.4642857142857142, - 0.4642857142857142, - 0.7321428571428572, - 1.0 - ] - }, - "site-1": { - "timeline": [ - 0.0, - 5.0, - 10.0, - 15.0, - 25.0, - 30.0, - 35.0, - 40.0, - 45.0, - 50.0, - 55.0, - 60.0, - 65.0 - ], - "km_estimate": [ - 1.0, - 0.9166666666666667, - 0.9166666666666667, - 0.8250000000000001, - 0.7333333333333331, - 0.6416666666666665, - 0.6416666666666665, - 0.6416666666666665, - 0.5133333333333332, - 0.385, - 0.25666666666666665, - 0.12833333333333333, - 0.0 - ], - "event_count": [ - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1 - ], - "survival_rate": [ - 0.0, - 0.08333333333333326, - 0.08333333333333326, - 0.17499999999999993, - 0.26666666666666694, - 0.3583333333333335, - 0.3583333333333335, - 0.3583333333333335, - 0.4866666666666668, - 0.615, - 0.7433333333333334, - 0.8716666666666667, - 1.0 - ] - }, - "global": { - "timeline": [ - 0.0, - 5.0, - 10.0, - 15.0, - 25.0, - 30.0, - 35.0, - 40.0, - 45.0, - 50.0, - 55.0, - 60.0, - 65.0, - 70.0 - ], - "km_estimate": [ - 1.0, - 0.9230769230769231, - 0.8461538461538463, - 0.7692307692307694, - 0.6923076923076924, - 0.6153846153846153, - 0.5384615384615384, - 0.4615384615384615, - 0.3846153846153846, - 0.30769230769230765, - 0.23076923076923078, - 0.15384615384615385, - 0.07692307692307693, - 0.0 - ], - "event_count": [ - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1 - ], - "survival_rate": [ - 0.0, - 0.07692307692307687, - 0.15384615384615374, - 0.23076923076923062, - 0.3076923076923076, - 0.3846153846153847, - 0.46153846153846156, - 0.5384615384615385, - 0.6153846153846154, - 0.6923076923076923, - 0.7692307692307692, - 0.8461538461538461, - 0.9230769230769231, - 1.0 - ] - } -} \ No newline at end of file diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf deleted file mode 100644 index 9de6ad8d7c..0000000000 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_client.conf +++ /dev/null @@ -1,116 +0,0 @@ -{ - # version of the configuration - format_version = 2 - - # This is the application script which will be invoked. Client can replace this script with user's own training script. - app_script = "km_train.py" - - # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. - app_config = "" - - # Client Computing Executors. - executors = [ - { - # tasks the executors are defined to handle - tasks = ["train"] - - # This particular executor - executor { - - # This is an executor for Client API. The underline data exchange is using Pipe. - path = "nvflare.app_opt.pt.client_api_launcher_executor.ClientAPILauncherExecutor" - - args { - # launcher_id is used to locate the Launcher object in "components" - launcher_id = "launcher" - - # pipe_id is used to locate the Pipe object in "components" - pipe_id = "pipe" - - # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. - # Please refer to the class docstring for all available arguments - heartbeat_timeout = 60 - - # format of the exchange parameters - params_exchange_format = "raw" - - # if the transfer_type is FULL, then it will be sent directly - # if the transfer_type is DIFF, then we will calculate the - # difference VS received parameters and send the difference - params_transfer_type = "FULL" - - # if train_with_evaluation is true, the executor will expect - # the custom code need to send back both the trained parameters and the evaluation metric - # otherwise only trained parameters are expected - train_with_evaluation = false - } - } - } - ], - - # this defined an array of task data filters. If provided, it will control the data from server controller to client executor - task_data_filters = [] - - # this defined an array of task result filters. If provided, it will control the result from client executor to server controller - task_result_filters = [] - - components = [ - { - # component id is "launcher" - id = "launcher" - - # the class path of this component - path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" - - args { - # the launcher will invoke the script - script = "python3 custom/{app_script} {app_config} " - # if launch_once is true, the SubprocessLauncher will launch once for the whole job - # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server - launch_once = true - } - } - { - id = "pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - } - { - id = "metrics_pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - }, - { - id = "metric_relay" - path = "nvflare.app_common.widgets.metric_relay.MetricRelay" - args { - pipe_id = "metrics_pipe" - event_type = "fed.analytix_log_stats" - # how fast should it read from the peer - read_interval = 0.1 - } - }, - { - # we use this component so the client api `flare.init()` can get required information - id = "config_preparer" - path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" - args { - component_ids = ["metric_relay"] - } - } - ] -} diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf b/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf deleted file mode 100644 index 8161e93e79..0000000000 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/config/config_fed_server.conf +++ /dev/null @@ -1,25 +0,0 @@ -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "km" - path = "nvflare.app_common.workflows.wf_controller.WFController" - args { - task_name = "train" - wf_class_path = "kaplan_meier.KM", - wf_args { - min_clients = 2 - output_path = "/tmp/nvflare/km/km.json" - } - wf_fn_name = "run", - } - } - ] - - components = [] - -} diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py deleted file mode 100644 index ca8d592d47..0000000000 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/kaplan_meier.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging -import os.path -from typing import Dict - -from km_analysis import kaplan_meier_analysis - -from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common import wf_comm as flare -from nvflare.app_common.wf_comm.wf_comm_api_spec import ( - CURRENT_ROUND, - DATA, - MIN_RESPONSES, - NUM_ROUNDS, - START_ROUND, - WFCommAPISpec, -) - -# Controller Workflow - - -class KM: - def __init__(self, min_clients: int, output_path: str): - super(KM, self).__init__() - self.logger = logging.getLogger(self.__class__.__name__) - self.output_path = output_path - self.min_clients = min_clients - self.num_rounds = 1 - self.flare_comm: WFCommAPISpec = flare.get_wf_comm_api() - - def run(self): - results = self.start_km_analysis() - global_res = self.aggr_km_result(results) - self.save(global_res, self.output_path) - - def start_km_analysis(self): - self.logger.info("send kaplan-meier analysis command to all sites \n") - - msg_payload = { - MIN_RESPONSES: self.min_clients, - CURRENT_ROUND: 1, - NUM_ROUNDS: self.num_rounds, - START_ROUND: 1, - DATA: {}, - } - results = self.flare_comm.broadcast_and_wait(msg_payload) - return results - - def aggr_km_result(self, sag_result: Dict[str, Dict[str, FLModel]]): - - self.logger.info("aggregate kaplan-meier analysis results \n") - - if not sag_result: - raise RuntimeError("input is None or empty") - - task_name, task_result = next(iter(sag_result.items())) - - if not task_result: - raise RuntimeError("task_result None or empty ") - - global_result: dict = {} - all_result = {} - for site, fl_model in task_result.items(): - result = fl_model.params - all_result[site] = result - timelines = result.get("timeline") - event_counts = result.get("event_count") - combined_arrays = list(zip(timelines, event_counts)) - g_timelines = global_result.get("timeline", []) - g_event_counts = global_result.get("event_count", {}) - for t, count in combined_arrays: - if t not in g_timelines: - g_timelines.append(t) - g_event_counts[t] = count - else: - prev_count = g_event_counts.get(t) - g_event_counts[t] = prev_count + count - global_result["event_count"] = g_event_counts - global_result["timeline"] = g_timelines - - g_duration = global_result.get("timeline", []) - g_event_counts = list(global_result.get("event_count").values()) - - g_km_result = kaplan_meier_analysis(g_duration, g_event_counts) - - all_result["global"] = g_km_result - return all_result - - def save(self, result: dict, file_path: str): - self.logger.info(f"save the result to {file_path} \n") - - dir_name = os.path.dirname(file_path) - os.makedirs(dir_name, exist_ok=True) - with open(file_path, "w") as json_file: - json.dump(result, json_file, indent=4) diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py deleted file mode 100644 index 542d316c47..0000000000 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_analysis.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from lifelines import KaplanMeierFitter - - -def kaplan_meier_analysis(duration, event): - # Create a Kaplan-Meier estimator - kmf = KaplanMeierFitter() - - # Fit the model - kmf.fit(durations=duration, event_observed=event) - - # Get the survival function at all observed time points - survival_function_at_all_times = kmf.survival_function_ - - # Get the timeline (time points) - timeline = survival_function_at_all_times.index.values - - # Get the KM estimate - km_estimate = survival_function_at_all_times["KM_estimate"].values - - # Get the event count at each time point - event_count = kmf.event_table.iloc[:, 0].values # Assuming the first column is the observed events - - # Get the survival rate at each time point (using the 1st column of the survival function) - survival_rate = 1 - survival_function_at_all_times.iloc[:, 0].values - - # Return the results - return { - "timeline": timeline.tolist(), - "km_estimate": km_estimate.tolist(), - "event_count": event_count.tolist(), - "survival_rate": survival_rate.tolist(), - } diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py b/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py deleted file mode 100644 index 2aff5c5cc5..0000000000 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/app/custom/km_train.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pandas as pd -from km_analysis import kaplan_meier_analysis - -# (1) import nvflare client API -import nvflare.client as flare -from nvflare.app_common.abstract.fl_model import FLModel, ParamsType - -# Client training code - - -def load_data(): - data = { - "site-1": { - "duration": [5, 10, 15, 25, 30, 35, 40, 45, 50, 55, 60, 65], - "event": [1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 2, 4], - }, - "site-2": {"duration": [10, 25, 30, 40, 50, 60, 70], "event": [1, 1, 0, 1, 0, 3, 4]}, - } - - return data - - -def display_results(results): - for time_point, km_estimate, event_count, survival_rate in zip( - results["timeline"], results["km_estimate"], results["event_count"], results["survival_rate"] - ): - print( - f"Time: {time_point}, KM Estimate: {km_estimate:.4f}, Event Count: {event_count}, Survival Rate: {survival_rate:.4f}" - ) - - -def main(): - flare.init() - - site_name = flare.get_site_name() - - df = pd.DataFrame(data=load_data())[site_name] - - while flare.is_running(): - - print(f"Kaplan-meier analysis for {site_name}") - - if flare.is_train(): - # Perform Kaplan-Meier analysis and get the results - results = kaplan_meier_analysis(duration=df["duration"], event=df["event"]) - - # Display the results - display_results(results) - print(f"send result for site = {flare.get_site_name()}") - model = FLModel(params=results, params_type=ParamsType.FULL) - flare.send(model) - - print(f"finish send for {site_name}, complete") - - -if __name__ == "__main__": - main() diff --git a/examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf b/examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf deleted file mode 100644 index 5c81903a41..0000000000 --- a/examples/hello-world/hello-km/jobs/kaplan-meier/meta.conf +++ /dev/null @@ -1,7 +0,0 @@ -{ - name = "fl_km" - deploy_map { - app = ["@ALL"] - } - min_clients = 2 -} diff --git a/examples/hello-world/hello-km/km_survival_curve.png b/examples/hello-world/hello-km/km_survival_curve.png deleted file mode 100644 index b06564f129043e66cecdb5eabfc3a9231ad8072a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 61673 zcmeFZbySq^_cl5-k`f{ff>H{obPg)rjWkM2w^BnGbO?x2gLHQ{qICDr2skjLzz_or z=NbKczrW9U-*>IE)>-TPcm9D{Gf&*lzW2TFy|3%qPq?PK0x1za5eNh#RZ^7I27z!O zAP{y7AwKYpc?

5GV+wB>P17wb{-hURv&tq|@Jvr;CdsFjLm&pWfbWcx$0DDl@vm zcZ24st76=bP_n(f$96%W&PCHuf|(ngi}-ZeOXoDk6B-$f^#OSoCu!Gk9q3-$X?&3m zazVOXJRgD9A$s@pt0osU=HF=040y_%@Lyka=yaI>{S9-78i?qO_>ND{C_U<=V}}vyvzSNbQCu;sQsV20RK`z)Caf!=f+e)*y#TbLemag^Z(PEwiak* z%jd3keaJU34-jfS`x&_vN-nL~^`5ktg$9u`yVpeyR^@L!)<(@Qqd#5Stc{*c%$P&c zmmt?yf>yn8$KQzgbG|YOeFSZ$56I>l|C?tbj*P*0RKE-dZaiTCk zBV~J*=VkkoQ2gB1(2rl~{SsxtuK*A;U@KDRX+og`gYw^2ek_R209asc8fGs6sac*9$gp&UX z)4Q)!gDl97Q3f8tF~$0mp*zDx=&jt?ChJ6?XZk0&_ON1V+jjKP)guWcckVN$gZSLrvGW zC7mpIjHEdWyRY|NG7~1E2&!c726e_lF4t;)r8wAd?g6GMGj+NwoLVK%yc}r1R6)TZ zX!vm`%O&gl@m}*jGjU6#=nufqUMX*WeK``lKGwKSywg$-GNDd7Y6YG>r?hUa$Xp&B zR3=Rmj=I{1KtGwd_br?{KAun&$c5QLrFg4)+3Xj+X4EeBnqBaB^eI=wP8U2p*1dk* zGT0G`Z_ns}j*zcyHo`MhZkxD1ah~T7{2j@tl=k91-T3i@2Gl0isjg=9De*?7t-De) z7`2&=$_3U*LGB+1L*>qeg28f-7^#!l8o-P5`#4e`-9(Xxa3V)XD4emP6%s<0FEIsqME^>b(^Nv@+jg zB=$bD7j&0VTdZ|HH+AjAn*hvZ)~?$ANx1EF$!}{T*<5`@`ob|L2uXm%Mf(RzUSA$H zx5ABo*Zf8Lse6VlY%2QcJpVWI0tu0@pPb;$^+HSd4 z>eA=y_T6;0s(`iCV$evA%lHe#g$+oY&x+9tU$B)t26DA+p1GLbqAjAV-}OCh^#5> zj}}?_JY%f);|&U!%ZXH%HZhKuFKj#Vp@HK0zOXx*i52l4cq=8m)yDZ3=ks2m0tdx; zs$DEFvet>I6Dka1KRs>ol{#NfXf!L^_li%FAN>uiLO4p)?5_L; zP4nC$xCRI3}n%`UW zSsto;Bn-=Pem35|w?+$Z(x6j7Y@+sb5C!EW$sVS4KKt_Sq`Nk@J;|tMKI1}2W8H=L zu*X-IGy_O?L=z5V&)zYmrXohCNpDBWGIwFSWTce#tIWsn&dL zvp?F-obL8U%F=}Xt_V%Q`>p!{zhL9BW*$+{j9B?`sH0H^1(an9-b;HQ%B07@uxz%y z?HCU1T!FVRyAdiEabPe${xj` z0YwXr9I6b=13onYgu`^E0r5#!TvLXQUBEaH_G$&~Mx)@b85y}NBD>Qpl(zXe1d zOK;6$1R+|~Uu4b7{dJ=!Z5MryZ}Sg~K9Yc3B5;G4JLO9EA$_zdfhtP^N>MAHHno~5eP8R2?|?k8 zdzv<#%sCuw?-6)9R{fkCcu=j{R=$?gi(r5U=gT9=E*tZEmrmwgpq#={yGJ8naD6rG zD62HX0c6r)IuAT0{4$ERxuckGWaZ8W#*Y`@L~%`W$fQSF;B=#qjC)!y92`qu*btP5 zH%a^oV~??;_nL$X_ZT_VPL2Ocb;$rr49v-TovNM>{x-B3%G>taQ}(SIc`|~@WFGz# zi;E8G=bf9adlm*I2tIibZt}IA{md)ONPE=G>)i6Yn@(9&FHrdwo(Y@YQuAT|{hqm@ zj~)XQ>6S5KhwsT5ZO9li*EZ2Rw%~ynA%)imE1h(E%1q)R;4Q21-bp#Pw4X+xBug8n zkG@mEG@>y7pV#Q+>(q38hmVeI*io~1iSG1F69uN}TEXDm8D}r7r)l0~?%}veVKeRn zyvY|KR9oA$^qnl6pK(U|49sI%11(fKiuo6CT(@mybnkw=qdehQuK~dknL$d=uHxb- z5A(9(=K+xl1(pTXepS62X5#)cn53C!g*o%~ht@B%ow6a1xG9}f_1Qgkm6&9Gq6h+b zz7f4-xKSW3zf4acry==UHiHpsrTabzUv199#Ia%Z{hOgPut(n6C+t|V}LS)gV zaeZdd-WR$)mQ0$j<z=kFm1ULjMZkD`j|AHBFruiSx+YGhd= z2`zk9Mub%78hRj?J&^TuDJ(k!YyW2c)Q~;-HQC8oTiW^8(gVIvgoQfSTdz;A(dL2W zbG8}8CkGFL+V%tX{VcWT8E#NxYbM}uSjxUcMqdPpTP|1YWk@*j3-cvNLQoUX?v!32 zkHgXVn^zPOL9(#ZYCaZ@^;L9@tNjcxi?GXstBc>#LbP&j(&I1{h z1y1xpCEuV;Uw#lg;LLda^1bHnBpI^f95##S_*F}b>#9Xm|crub2@WD_c#{PDigUFGjK zNJ0;nE@*Tt(GICJV>qhK$X^xuo;<<8UO4HSJ`Ki)o}0D_D+8Hc4L?t6LvJZDE z%$1aA&T)=*Ov+qJ7z|LiMClb!#mudw!17Z0@A6`E?1&BR3x*$fuUEJQiOTCKbT3b{ z1mh30#Zfz9Gcb-6O?;34+3WE(w_UGYVpIr=+?FKKT{E}N?SVu{dzIx&twLKE2%F2b zyMw4_ur{LTAb&6IvMHP6C98GR(B%?E%cR~ra6GNkIC-YbUA<){XGIXYe0 z&(^FWzNtO>K`mAqn@uD?TS!gD%gK0v{VK-Q-T+8WZ!p@GW2LOcNbpIlOl=q8H1Igg z23ZFe^IR4Rblj%fiHRcJo~>JOAMls5=%lOHACP(yGO0;Xir{$MN(!0VG>V??r6X9; zKhAel)lq{cagd%(8s?A|Bzjp6t!P{g$VL=oR9XaB$GbT@g2)4`{B_%TI?qv!ZDf=| ziKPA|Nk=5g@nz*7cE~R2{4DzA`8gD2?VGcgLcmB7Zni^nHgCPH+t52JQKYIiiHi4{ z&;4)yzKujKECBh6=cxT@>UCiZJ3(xdwguE8)|)ES80u=9`1&rRzMRz{?9``s+IPbt5w z465|}w74a+Lx@)PjGHj=KGEUd-3Y@L*Sx2u@;!%Msqh=L07_`IXo#ydZpE~=bdPi- zKr9)VWvUlc@{J|zh`DFd_wJa)%2I zz(P3E31G)^O`rs6#lJN5W<;KM%g?dz!d~;8ARYatWxkG31wDrdQ;)={<;`~G z#E%Rr1$iMbA@TzE7#PR5SORKfrk~`opm5j*?q<`iK^uHzRum zTkaq-qS|;xqbR9X**e-#Pg@3yrKfIZkN^eb3Hy;5;b}^Go@Jh*UXw~;>wBS0Y0p2V=cix6yx0kK`-@Uch;cnp_2)cbkT5_cAto?4Dfx=B0T zCCpGucW67Sep>UDLo0Oye9+_Rz^?~GE9AP^wDC_VRu5a>BDZ-M*TOkcxK8Vlwz@0i zaY{2a(K~7`VPzI1T zesG`;*a0`14$3Jlu|4umn}c(9>_~6fl^dwvB>$=3crL?Fu*lRY2^5JEPes$CJ4!?i zw0CnV+}c2R1h)o*X&@fTVhT2GV!B>uYhmJbTJ+SeKcmEly+j++7GqXo*GPoLU}?h0 zm06f;XRD|WqnNsIeujs+?mO_9=!(9J#OswH9MPCN4Xz{#P#u>@zmo@eX%&`@>g;C| z%rid=in8sZ!rny-$awf2a7L_Um=V01+os>W^)Sc0lQvZt2y%LDl^`y)n_&%8<$4j~ z9^(TX9C&Ei;*M!7xasV}dSMTae!R-J?$y$*@8UVjB|7RmW zOF@L6VcAW?!b!p;-NHogBli(&XNtD&B%HAxgseH2N=$37RbcCJ?#yDpro7(bp+%b* zTjYcK;*lGN_YUEKS+qkd<)F^YC!vSi;|!&8IMLemlf^lKA>U$FMtoZkF)4DuO7|Ff z!$0;zJLhGDY|`Y2{DFbDBeiKl%&TNwEg9hl+ubRhET9Vg(d~};*4g^jm!>9$*SIJ2 zolvh2(cz>QMx>PA1RXhF4GlGiq|S z42LY3UwD%cZ8C!_GG{YS-cB|Jmm;yHJ8RnY(WP`X2)&_XIz(6+FdLr;~Sc3=e`MDO7>?@54`p_}{&uQ?!tFj86k=)-jw@5~8tGb)7^%S(v9oX&FN!J^?>$#kC$3f%_r z#Tfb>7))Qf6)=>Z<2+`R_<;=42ewLqJ^m~$?>!vfG+y;A-@1>CW)l z*Y75c$F4D;OZJbE#5=5b+Np}hIpYYOlX!0f@Js{OSZBsJC}9%%U`J6xpZ_Y?C3mCB z{;HT9h>4cQOh|CS@gN<4*|wM-zpGi2G(IPXhU%}N7nV~E+|3EEA~$Q`K>Q$|rM=%l zN}B|)C_Ws-)-m^b8B#zt_&tV0QdI^(w`OPH&=GD+yc|6;p0}aw@8Yu^smSd!(+s!#t-I&;8F}E!TezWm z`OEu^3d0XZ?eQVr7u2-{kbKq^H}dV9a^FPF5eAug6qkhzwsA*Gggb-)_XLfV7-$^cyM;sbH}8 zZ`d8RnYYxbVR_ZXBP`rQ9M6i7K4hYTVn02!mlgM&1ce#57v%Q=8X-9UZ>BOuf)2ys z5E_h<=uy5)s{8!eI_>soj+k3Xj@7N#YnLF_{rQgXjcLDa4miKv^Kj~!)zS(Wc+ZgLw!7+|(hBcY8rpF>+b#iCzHPF=UO;NLt66a~ zHc(!jP8&?$uS`ODM8Gr`YZf)GzNu^b6-zu74sFNwnB%Ep6B>16GjuG19@hu@NX1hn zfTnzrfE}RazMw8t+i~GfG~&VTbaHR#on-+7(|OJ-EeH;UdQ)WwM_HG;0;8L(8PyEK z0YORh+GFYHXNA>l-k@sfJ5SPs!7`RVpQ1SY=!w~8*)&jc=DNAeW%g;Xw>Llz^(cq> zzFTlCrI}6MGxm|Jk++Anw*!rc?NtZZxB~LYC&SfpjQSsQQ88zDY%|JDIi~zPk^wXL zpiI_vfX(;DtS>f1ZjP&;bW?T-Nl;P!NH<$4_VH@wi0T7c+lLXm3&!E4_m1S38oMb3^Ay7;iM zP$kcbsJ0r=o|0pI&HYkfx%ASS0VgkJWUH7I1UAc z0$CGRB`I%-Ee?9p_`m=)=_UKd@nE%S&YA?ZD2kuY)`v%3#O=%IaueT`F@HIlp<~nc z9%+NfibgY%DejT9+!Tngz(OEE&%kzN5SyJBz^g~TeOOo09x`$nnCgENqgY^ zZG`O4wEJ>1fLrJ6aY*^%pi7K-%{BSuW^5BGUo5(^!*0sUj(e0DTq~bewZJzcE1IV_A7Z*?<%5?{=$*ID$i87#N24%aF!=kn`=YWCX;;d9b z^bm`0QgR)lzjg?{quCQ~j*~`@AoA#CzKaXijj6AxFhuEb*JV!8*8(xc-NmAdZ>8@P zO$QljlgClpe}L#Ni`z6xE4vRz7*n~LwBXm$?Xavld;(RYjz z53Lr}gi^302D@=&Vt7Hg_$%Ydk5%Z_M&HD;^G;P2_*)0*@{e`%;NEh3SZlaAHL?Q% z>ZlZcxI4{MYy1#{5>c@*mYM5#fwT6YOw0+E*{%)bq-ML|DE+=ywQo$@xXU=@mFaSd zIYTKoj=n{o*MR)PH0OsTk8^hPCZy%SGI#P@B~k<9wiR{mBV^czut9J(p6?>~rEu&1Y4OR!v9;!1sRAu{vU;4R&M+gf9?SIlyt!kIc zU-7=;F}kB_JHiHA z!LmX&9j4P9Kt{)zbv=cw9=bc<`8{~#?v+hiq9@E|Twcayv7l*_3TY(*{jh^*JoFa~ zIR*gxgMUER{@XSlWrJojOuyKMX+O-oCt!25sMtXeewS?<{ES1iFEQ-(1Y|qBU1q-c zqKXKT<~GA`zGPa~wR-5=YF74DofWS_IiW;6>c<#C*oH3=fnk8?`;AuZa<*aU>FoDb z1~J4=-cU>>Oj*ikE`Fh_ac6eZ4>vVDw9q-t2rg8_AR!v#q&&Ze5WN3Dadv<+o`a6f8|h_dGbkw|5kaGdKK)AnaHQvv4F|^FURF*U4wwRF$02v zcDsAcl}Sen&jM<6=4W%p0$>xrcg$*)48HGWd~t{RE++}4%}w>zAzU1jP!cm@-C5h> zGk)Fvu(%&WX>i&e2T1jhO_8=iPovpZ{cuOykus#%xYC0zb>qB1qv$QL=NA5(E!ZSWt49@YGZw=2l|5225vg-PGJbpIUCj@^`>*N4zKKBIzu)5h;Z#?^vQ9A}^ zRswim1K*Wkfn$czN0K(GmY+vwQuo`TTzqu=qc`I_m*5<3P;*o;J0*UkG=0$0Wa*>2S{u`-}8;+ ziAeso&j10Ii}8i6`XBH~fLnszsqoMD7WnsAl^+|U0ctjX#>|gbLc6>j?dyAeb$aCJ z`*(1Modhg7cb>gpKLjW$xZtP_FuZ8Y1u$H`9`5Dk^2c9&!IbAXA!wIye0hw^LU_#s z&#iq=7kwrG-AD>Rez(^$ToM1@&O?_V*Cd16W2I4C*GvFiab>#Vv^n}!%=igC*L5n( zlH$KCUK?>a^4QpUnh_L;K8LHN=L6IgLIc|+3o?Cacx}5_6=uQy?|&1h!^0ic>rrJbvgs6EX;uC=`TwxE~!#BvWylQr?X8P+X}*A zAo9fc@0+S8GC9sJ%M&*OIdX)W|FJosPqI2=5VX@2Uypk>Zsxa}gx+pP7Y`i3n{U~rh{gqnvHCCK}p zanTo*h~pzC`m=tewbRN#7;?pgXpajHYP`!@`;N|K;Hx-WAq)+346s*u z_sbgC)FgImOy7g6EilCupg3uKyxy4cp)aU$r)t1m*Dc*^S_wn3wb}Yiny>JWP?BCh z4nY6N=QV9RbKl(bj#BKBk5cVYhWIeWX0~y9H3NjDJ+& z`y#ybdJPgr7(?7x&Xs2UNG6t?E%nCh2-niaUnz~uJ&YczXyw=p#qY7pR!;*%Nre^L zuXbIIq*M#;10&SIr}nxT?D#9{R2E%c<>!NL7v3)a{k80yc`(Dk$61E;U?-S^BVPzN ztF~iVyIg9|4ueCj;kthk)!aGLoE zN*yAW$IaijfSdeBwKv8L00~^nl|Ab0Xz7}j42V=2JS*jumymj96&|LbRiDrPCpzCF zs|^`n^eEL*t5GVw)Y*@WFx3B6elsbaxhS!fC~ZWmcJeLxAs~KUz=% zy@Ao!NnXhRkqrE)Zu7oHZMoADgU%~LbNQ>wEujkYj4V- zi0VRl%!@a`xSYX0v@&!5a(%`=!4jE%)KB~7XE>)jgK6{Lwbw25 zM3!D#)`f~YG`GtciNLH97&p0Gw`q4B6Uq9S{dG^K$t=IL){F>E4(;bUg*==q3fQq- zm?_^ndUiZ7kX>rMLt}g%<<^Q4h4;*qPtgH>3QSkHq#fXzFY!A7jUJk1=_uFvB-CG-L6cJiJiZMePeSGD)U>>8SV&3AZlAOeE#Np(=J)#f?#@nIIB@idaP>@}zH@`@6;HU!i8&ht(l^XzA6rRPJvNXDwryj!>AC zq`Qt|GkCZEez_#Rq#V7qdoZ)B_HB@d2Hf~(7dU}_E5+EUR4WQ zEdRsa<;nJ)p>DKrUXWSvSOJM(fy6H-4SYjB9Yb4L?Uz#d zK>&-V3DLG)ahWVA?@_kj-?z3RAD)Y7b2lR)HAhJ=ozx3d=u}iv0O+p+O83QK*q`%};ahZ}H4)8o1kY4Tzlvw(&Y`6tG`q`C1S>2Yrn>QE~-WU`41_%Y-;+FGH@~uc7<{#*anxF~__>%UZRlYaiC7U~Tg}8(4ly86 z`A%0=`>wTi8wgP&GR@=4_j3V3mq^#w%@=xjC9nm`B1&W{ko6x5)%^JUf*-;F-}!!m zc#c>y{AX1U>hdYjNRx^KT0)bb>x<(>Jtk0n$&=xlZk{L2J}$=wSX*DE9@)V2-k^TAHgYybX{HgzSu_qT&Mr9=U=3DCU0qI)mb$&o=|PZ>ui=Qu26k1DfUrpcMu^fHbbVt2e3C@bkL^& zGO6QH#%Nw_wYeThO`ih6i(BcQ~ zcZ^(`JyBZ$8EaQ$SMOV3>DT6C`X^O&$O2$HmIa&lBo^1y%32~Us?xH+9JMK3(`GE4 zZSHUNi7-1f9)5Qq&;9UW;Y(fno9#yjK4(%rx%p_ckuSP5cH+ndGFLuO?Go^~iIgRm zcf?Qh5A5&Yg$og$Z@R3U`Lh%^f*dl`&B<84tkxD|+x>vu4p=*I=PBm)cGOQ>62FW; zm7OiA#&N?4j;cy;op?!eKEJE-R_=R_%vV2G8&iYt1g@LVm@J3r^+4@&(M6Dk)V6&z zC=zU)JUZ-;3`Wu2XA?o3Vg$!&50GukI1d1y!W6a~`*E2Ikbp{A~``mS7N^2CFTlVQQhd)zC*S??f+gTO0&*Bg; zNo5%l=3oG2*s(6pSL-@%__p~zqlU@Ll^H}YdFKSR3*%i1#kbgy$4#mV25xs-L0C4npW>0pUOBQA1S0rMKSR$GpqF>c11uq$a;CxHwVNe`ieWMmZ5cMO3;@d4 zQ4`Wrx4nf`9szY|sYq~h(0-Qc|NdY|jrIAdon-jg@6y~~^j&+_X}Ue-H*m+a9JG)G z;1;uy6JU=5N^*{mDJ!AVAKqf=Cw+Oe&yY#ubD=QHbpB8i+J-Lrpg&!pB|Lu4x(nIQ z1lGayT7C?g;PCe*LF+ZRAdOVLhlP(6(-m@M%+$GDL6JR<)Myjaf8__^S6DpuuV;ra zDqr`MZo%W`Did5-i|$q3QPUQTV;|IC39V3~;34hSD90f^`Y6oVd)-f_~TFFcjNfebt0Lb7aG zhv}>3u^Y67ooc@2%@thBZi}L(@k3IxY!2Xb7Ajg?@!BxcxgHVf)tXO3P2bpDb+RJo zO86(u-`=di=zr*p7Ta?iyz$Trsiw2q{lz!J5b6a?0Q|M{`~%N{El!sFf$jK{=Iu2^ z>R=TGtd8v?MwOQUVkHM{-b{ zGkh;SW?un18}GsANrow9KxPg9i&*^522^H}0hd22D#zz9{N8L^7QQj}xe!Pu6+OKq zU9lle))g6+v`sqdjkhh|>tNxW5^{Wr{XlH^@khju+H(pvKaRpumf|-4)+$-Eh{<#u zCtFqN=qS#3{y;SsiDo=}P%_u&f-KUb5{+tDG~wqOaA14^zX)%GC*JH8y4v!Im~k3aFLepgFDLrgT>&$ipKG1K!XfumnpUemS{f=o>&v zd)}2J0U#8Oe~=}+%L4W@cF=PTm;L=*iLH$2yHW|JqCJn_KVYGFQ3Vc)mX;L;$S3gI zu6aWA*^QM!F*Xm5DwPzIR?)f#k5-BGme)p)9w|f5`}VcS$G9GT&&R?xbt`JVKiuGp zEo2|evGC-fgffLyewBU+&jH6hO~FEn%gO|A!H*Zgcct%~xw>dQE==iAvg&_?(IV{1 zL>-wBPzT?lFpuYMv-L{Wp0PQXX$CF=MABKREXaW*yaUK7iD8waq| zz?|W@Ual>+&S_O!O%A@_asx$eg7Xb^hE6E@6^%XaOs2YQV)~@Yca8N}QzFL;drrf` z4ixsoqnT~XmM9Noc|NiZU{r-kMb9RlF!Km-ds9+AO^)4p5y!uSbtpt?Ln=ufSpQ4i zs-nqmCNnN|_XYAMtjRDXB%C+h)`BP1(Q$0Qg!8pw_o@7`G@|6D0#TkSzgtK*& zUXW_YKL%j>%m{=S9m)7u-cFcJufy)X&E|c~RZ3uaP^aQ(-Q0Lr?aQ?lYW8h7IJ~MsXAS-U0VxI@p2zhOJ*=pnI*N)4v`3yICx2d^)z3Qmyb_&g zPHgAT`@SFz{=9j>XOFIYdV+3X&eoFVVP0$#)wxGWT~nQ-xRw-BfTL(kVvS^yE}5;9 z<5^IFI6c232k)#XcPfR~y{AmT7KcRuIqW5Mj|>RV6y|X+4G22dM6%M#O6L^YL#;IF z2c;7O;y&M^IzPH&?)@7P{=XP{WkjAi?WKfsgiS9QkA#u(`TAfL-MmmJ*rIFXKpbvN zt-4z78YWgWy>tH42OgQ3co8nj9VimMs0G~=87^;9+7NiecgXouyDT?|k2}wzf>1Q< zrqWnRe!lQW@6eZVE2xfABU=l-Ht`U1j;d_*-2HDi{1-+Z?R-_AD{TXP1%$WtYx}_i zm`*19k4hYUyI2y)axyz|bPx`U7S9@{SLKPGlw3JtAlPJHo6TvgeGW9ZLpDIh2Fdy$ z72J+3(wzwF-F%r*c@v_y%Fm` zmmv;}uAJy|ST~k{la&to<65u)Ll_5*VR;XJkZj5efy!9wrxZ%wb$k=+ZIz}hBTS|f zwa3{gnB5QO-i=eUTWR{s8iD^RcK&8zBFlHQv)c;g6~eXdxd8>J-qkyhAJ}pC25hhZdSN3F80(MR*LNV0fSJG3q&NrEmOLT z*H3+t+(hVO;OJ=xF9Pv8U>$a-3VSq_(l38CmGKuK`0%f7iEG8h(H?AT?5XY`smNO|^MSox0qe;agkyjdUAk z-ZQ9o!))XLfy46ejOS0jUiAO}@UPAE|BXp9BD>B3S|b<8rtY)?NItUu*_Eolpyd^Y z+;nL>x4GJNE5bBAV~D8AL)3Qp2gocy+F;;G@>>AB{fEC9Jp=Zv5wGQd(_8>7l6I>6 zL9+$)(U!I|A#8omRzFNUAz(_J1}FtjL}C8)GIPE!z*dX{4TqlqT%g!<@KKKyh{OvJ zS^Pr@Ax-`e34(zb06l!&1k>`I70K5>Y(1u*}1nW4f1y}ls2xMd1}?e;*=sw#s^ ziyMB28*f`Tlg~o@6UQk%12tm41`AXtd0-p|P&Rv_w(>C?VQHXpy_LV+_K?VO8o&s) z`}6Ob*nNFBjzJwU7<3JQL~U%+H2jP^trNXk#A9?lIEO>wzIqo9{y?`(embTryY615 zIphji-Fh2n{&YOL_vtxXY0F{%mQVYIsrc{jR31c!4R!_^j?!g+i#%z7%NzOA=lFCD zV0WZ$6C@5L7I{dvt!gZX+W{kX6C?6D=p@Jo`WahiR)MPD(Aa0uYXay^{{%FZR;TUx z&O6oT0vP6f4B;Sl*wfhtpbg^y1Ebk48~bBAvsvPyHf!FJ`PwtncVHxr0NVhoAJD`; z0G30wbpHYx)BWnM0h%p!`nhGaC(!j5olu*pk&3JrOaQ#mm*9j_Ow0d58NdjzWBPEn0G?^MoxyzWRcI2XL6XV6_fBJmp4chC z?+(_r0S%8mEqe_B&7v{i#pDsi4opDLtJszH_4$^|Oj-FB!1q)# zHPeb@S|1IGXu#7v)K7R#js2*{+5mV}Xg(x8_YCOK|6mW4J#S1xnA`02${ey*BJP*< zI`Lr|`42@!PSh-b{EXXvH1mmAkzE?5ryrIPD|x7>E?27DU+8HYu=5=GkPr5`v9TfdE(?sP%08cmvSg zYNhabZ6HwQK&1U#4gZczX9lCQRz0SJKn>H!4Xg|efQ!`N>v&g<5LFlX9LLrONFg?K zhvaAs?_Q&UH!%n+kOb=);jIRGb}~KE#8!s46jyHG4gr*+7^0`Jyi413j)+5e z&MsRJxL7_wRa~NT+13?8S^GdPx% zAzo6hbThc9VqS6(7&zP8nr!r{ux813djsB`gC{YC)dpxT{Ap`XZiW(%v@DgA+%vdy=fF^VZ+$%=&eZFHBFEVh&C9|6}gSD&D|Q8|$F?a8m( z2e)L{%(1B54oy|=?gGS;$^d`6KfdD1w4N6pI>!b($ov{LLq{l1nMp~J0=!b<$VdQ* zr?M-y3*?>{sY*N9f$*$FC?hvEr3LxhU~vZqSgKRx>~m_8#Nb;P_ESNchtTw(ZLD-|7#nfNP?1&eG! zY}qIhU0-$CfKFq--QxTIdjt}H+aBMgb6l0v$Rlp8+_};^lif8$}FZ81q&dz`9`a zjz5_(tVhgeZ1Jr+xj6>{)qST{GF~yFaQ$q{pZY7Hojs^O9#?^HvT*7v(=)h*z=~gG z%O5E0I-Rjm;`XL-Dzim@v@{}OT~izZ(0ofa=`=g;%&E1KW^8KYB2zOqq>NX&>j{B# zdOS+yNljlElQi(Yg(Z@v%=c9M5%}LHvF;+N(P?X57G^$s!I9e${f+YBGT3(9b*$u{ zKg}?7vS5*Kj~-vdD=>@U?{&rsnm0T6wiDGNu&sg!xTu*X_}LP&EVcEp0u9$@_0AXo z_BrFN>;5pKxt_*~#vCUhex+WL=QxJ2JiXycmHwb`oNZxTr?gYBEQ;#n6SE|suv%BT z8DPZJ)g3Qf6@HpK>#hHP%7sMgX5ZovIR*mAzm09YIvqH=a19WGgu){!!nEijKXIgvWgO&7Q%|% zq_L}L+k=Cz-{6Dd*;4~S834jj>j9#=Y;cik)lH;u#J*dRFBRkVe-ZbVVNrJ7`|!|W zfQX8Kgdiv(DBT0nEr@g}(j_f9Ah&clfFL<^Do9HXDka@r(nEI+!+($7&mGV6`}iKm z`&qf>y7pds#ktOPZu<%+)x@Rippdn7xINveTUioIcFT`~9ETxe;rJD)YbjZH@vKKz zIb#kOD)e_7W<|ll>%o6+Ux7`ZZMf}H$$IO(=ksTaCQ#?K{o`dcBZ7K{;)pt~neB;| zlu?$a&#K;Z+noNnR?BCX*z;|=b9lx?s+z8YImk8v>ijDq8N7fYgu1Gq?Cl=*eI_%? zdd?WQzq+#!^c)q--q$AWi4ap`c)E{*N3PnB+|% z*w?aoX~oPe4ns$0oqVG>CMO-PPS37%OTbQ7D!mOoLG)$}hX+87B6RI-0l{$}YnC62 zkT`jotaUtt&0?6qQzQxgr}HSxJu1e8Z2zov>Ss6J&AoF9k$&%spOZ4D z%enI8wkL9U+CJvuXxvr~$B;uXL}!I!g;hF8j@Te`(=Ps2gBtK( zQk@YW=pBJ#PKRY)XMMw@z|~;QmXh8O*IdT&_nWt!ld4PL2I}QS1zTG6rS=hmi>>S1 zH7=eucRKs;5(H=CIvD9O%&rP}Kco9@niP39+F$<<5Tl-!m*L9!PPFsvq-PFO*O5cQ zDu|X$?K)05qlBC>U0!Rod!tK;>>^pO{mwAw_Lz+UoKU)IA1QJ-d2^m~ap)%e@un7k5G8Qp^^wb&CPLR*3@DR4+H;by5SGF zY{{l}9bGrL@jm9o>O{#;75KFD*Fn1b{+$4$;5`(*uwNsOv(E*85?=Uc2mANx8%1{1 zlwGNFZQ1MciOxo@Lv^V79xp85F)Mb{JaixX*@yz2YQcLQ3H2f`V)yo!qjd9bLyZ)c zijw;jD`oH>UZ?WK0{N+7CY)Cs0MgQKjh5nFQ@S7C-4&sZLED=bm~FM-P_WGJoqV(J zDwYeSA^S=JxqdwJ5s&t8R;nhhQ$4fSVM1_=#*Z#9@+Feq1g@ zH?a)WskG}seiJDTbe4%h4muRXJYSQ_qeFsHD{g7cbUCqm5WeyK7!2_)5M41(3a%ad zs!>O6d(=mH#Fkhq=XWr8^G%>MFW%UDz`H0~^0 z3AGik?y>g9{^Ty0th!Ht4u5c7D%jaKFEs<9G2V_Yy|g@%-?}-(h4Wag?!+ZiCgh_0 zF!_yZXLB7+>NJdf7P3le+;FbLYd;(~hu_~ErbM-4u{0-vjqng_MXeYvZG3U5SSfy6 zyMFip<%oi4+3EcpgE~J7YMiUKcKC)2S-y|UF5VVWEerc3tuUH*BE3HIp-wtjNI{+e z*NdY0<-KQAML}^0 z@n9*aU#vB*9^jjmg+KUOiXm`rT3oY8!qs?(5@=zg?>QoKIpDovsxRN^Qrnt%qIGA$ zv+xJszQM}^E)xB5t#`SBo;=hAC*spJt|-rL|4r3_rNTSmG2YcX$K-$IXq?0lF?UOa zIGcpN8B=v3+u&FhM%lQu9FEunF_Wo^*6A67K8hTUlzO{YUP$(gRRl?L;T2puk^RKY z)0Hgc>wutk{!7piJLvJB+BEcJr8Gv&ojdE$h|0jwCY}qOywn$11Wh@pb*}*|gXS~Y zOx5Wl>ywi@ie{nu>Gy63Mbsx`w)4*u04VlA0IM6YlmGF+_G>tMG*8t@^fE8gVN6f9 zN=f@^-PQ#OL3-&Ub27gh**|Pb)QQm-pNgZ)aUQ+-e?EnRO3dT3*HM$XWn|5voj($p zk}P(-k|=`qDN>uZ`4Y`BjU7m#n)%hN|CIliBYunpIdh-=&3ylp5C5y-T<}Yvu^hDT zu#`Z6_+R>kv}yvN^Ie?Juv;=%eB06hIq{OtEi z1KKtSuA0w4p9U9m@&x?tFG!j7|Em|@Tmig(6(ZW##?=7z>IYL1MzETGEVlZKg@qMr z1Vl~#!Rd`>0O&IqrF2CA5G@09fS&cPo*K9T{VBX)IEg3nGyw%@j|-3Cj>rIfOfVA;PHhYL=}YZnx-TJF8_En zdO1W_=dZ;4?SE{b$c;Dc`}Ee&@BP1kZ$LgQ1BiJR0<)K62~;{&xVr>Q5&%x$ho$Ob zwJUGm;5|5r_Vo{9(>?*}nJe~@PhBW50avJFCB_8-K}km!V8$hN;a>sr#Q&+JjTH-+ zt72hR!UjK3Ujd>OHcb6xo5M8;?|!QCY=XH9Il!0q)U~hM?~|dnjmfaU3jA@5l>#Ea z+4%)}KBqoU29x0r8(rtROWVUS4r-ML1J91wdYoEl++NnaSur|W(HgC92&j`lOk)+F zvH*JgD5w0*?o@2`XHPeqsi%J(^uLXI@kJmY$G%b$O4bm9W<6k46+3?EDliRD-A7A2 z{>T^RYFSvetP<;dMxd9)q4Agd?-CH&{J{0;7tT#p8#V^zMk&Xp>L32IS3#3@qR(ubc0-7 zST}=b+fqn?3pfm&ZhWvdvs+J3hHc z#k6*%J;FHR|L_?mH{MkPK5jo&mu?XO!|<$rEL}ukk!=nndV!OB{@HLKDCw_$7o4<5 zp7nv>kL#z$B5PQ*(j4aZUGHruEBb7aA1R-c?5c|$73c>-t0_gBQ{ZU2iscB5B^e%m zw!9GZL@QzgAd<1q*MRw{fJvARZQnt?t(Iv)^w#%&EM+cEq1s>#yuF1nC$sue2pPx7pnv~!@a3W-=5KNW5Ro8qQE z72)-m3V>tU2)}zuVeF>5G;TrgMIoGhxE7$b+&qS$(@_Q6qZ}VN4+N+7v`Y2sdj!4K zsqbYAFekO(gphBFImiz$yyBX_!iomvJ>ce5j(({HnwUGg)*ZO&2mAfgSRA$xOe=L) zA6Q^vqxe<5L9F;K4hAcH=Dni3lHrY=p%cQA(={A`_HL_*HCB^0kj7P87igfCcKX@T z#NeC&xHD1q%?eT3tbbc*F88)+$I4#*iV zjsX`s_hI#wdY%2`%@4qNeBx5NpZYcfs0&i(Kr!2qCa%l#YrBGL+9&}RFhJz(By|Bl zufSvQDM#X0#I)cEeKh$LTxt}82YG7Lo96LH0Y?{fMtGBlRb3mFNa z!MSm|U3CCP6}{NM82e-Aa40qRByYSvBk76-fU^y`FXuN;&?w2g!Z$+yskoVnGBq5> z77KxM*9Pn2<}n-)@!m`^ZWHqmRyHo)Sb z)c{1?2vggN-)3p`v^SgdoX+Nh2BKax1*O$lj@Sv*VMJl;=wZdF6eu1 zE$KpJFJ|5+UZS;;CwKnj)Gf+iG&tMnI6i9(f;43FJlTp-)_QHJo5}_>EOtzUzKK& zem8*kA=m$KvT~*M5D1XiVgW!btDEqF?}K#Bkhz?Avji`1dfiM~tg4+D zsLS;;a(Da_hMd6@92s~i!A)&?e*qY&wiV-eK}Nkg1)Sd~4@U{Kjk?O3Zt9^W6|A&o zBF{3t5@NK0)`F8%VZya>IH;edfrK*G6TM+eHx1k9W~+%jBIEIw(aKyw@5MBY<|eec z?7+P1n24qykLPEEHFcc2>Hhm-^N!t&LCMTgvVs5c{}D5)*Vo(P3Y9{Q%3dqJ_oovP z@P0HKdb!BpnS(1%N^aCL)hPu3tx1<9w)M;H1*9f!tBE&NC4)v)sY!R&Mq1E}+=Cdd zdses)kFO?aEXk4XR}V->d~&KK#m8wRg&logyYGMh+^DvDDL3a_kfKq*(3vC&3vO=V z27s<(&DsaI%oqGUm(9v4c>IK1T#O(Z25uwdM{EzRxygGgN~Q|0ld<@=L{v_?j(>2< zVv3E??a?*{;M}~6xHY>k6)?}{U>9z=hYM3iE!$*%dc zbr5|BA=PHHbn*Mctj9GsYw%7zSXus$72GTU0BId;PC?A%=9H-SBXlUw&MIu; zbvifhU3!aNX^Kt~F4>xDj*zEpP2CBi@CYCrkr^5C`c*CtAvMJ|slc4LR9VOAP{#W* zRW&F^>c$83b$Kc)wvrYC&c8evtt(tBoZR5%slP7!Fm-1lf?V>>PyfUd2Tl zd@3m~zKK`gU4M4FgLzLKS3AEXo`!0HZ$pYW*Ao3+y%fy5tT!%CJ-sfuJ$0IVtgY5Hk0Wps^`=OY{ahwMSZebU4!sYYf z(p!a-C#t#+drEtQ#d`sWqyJ+Gm$+9;pChugpzh^aZKp=Hh_9Qgc3x)XtY`->oTw{;0)t7`!f@TV#d-qQQhET4u*H7ySgS#)CJ<*NQ(X zUgYZGKBoUP=rGgC%-?L}XRwMNPB3R^RnGfiVO3QDCo<%^srclDmA@B(_YLwX9h9w( zf3Fmb@=eSP%eMTq{*=ljNpxTs~f+`jvh->$R%i1KicDpwut@T+un#XpZ(Y$yKF`saX%AQceEZm-nw|PilG#HQY6bnb?Qr{-u#uHNkdhkhB#Eqll z30y5$f7y*Ee9}fb4DZr)B`4S;coFIw7XIcVQMujz%-t zTQ2pO>%FGK*f#QI8|s48_;7J{IeryU^ZgfLxsn+_cK@zkKwBenTt#UZx_{`rw7Z#E zWB9t#D6R$pW1HOVnC=^2vf&}b7}lP&I(L^Ah5I1Shhg-wvPNeQ&(84Rb0-#;-g;zr zVY$g&j7#HoQMX+z->6^6y3cB?Tqb?dRJO*uIyRYuraSHCr#Xr(^0x9!@emuGxx48a zL(cE77=2`!yY2KPnbC@2=YIanvk&2X0kONW3TT@7?C z;~Wt~r&Thtj-2lx91UT+6^wSyfxK1$nZz6v|T%#RhNRRb&y2wlIOI0<>wTv@`J3keyY|< z)a`pOSkvv%JTcqE&D}7+yy6CVkf1&DDGVruR%IH_@Xi!@|5-4CCEv(6RNB4$${^v; zH&G@o@zfTaB7rXIOaA8@R{X$;Y`Fj%+~7d329@=qWobtxt;f;@LPhfFluj4 z0s)$=sp3L&2AfEDxEkY2*)VeAPGGFS8)ST4tNLQoZY{E(b2@}_l!4aMS zWUWV%w6n*?CNEJ#v%>6Y4^g&l3hqIeQ8)jS5|v(Up88QeWZZ>v3nw>5n3(Ebm9@Xu zx(0G(grfyVnY;;(c+F%@jw(g{Q|2){bXCrC_* zv1Td!j#|2{H?5)JdWxSi)3Yz7Dt>w9x;L)#?ZX`xoODz5`-5t%$@k-0L*@i(HmvIm z%!kpV&(-ddJ<-Iej1RjLA!PAPb%_O{f|EIOG0cK(Xkb7|UrRy_+uYl4SD`Sr9Y z)-jqRyeWFQ({i=>XysZ>Wn|XD4Cxcd<(UmOvFqN%A6BdvA(cUKJcib%QhjAaa&cg8 zrr_R|7W0jUD2D~wwH%e^8=lmh&A20RJD2jy{39=ZY)Bd~%ll~a&7Pp3VNk&GPv-K9 z@ajcBJDqmj_a_CR=9k4wYqs-g`Wy$1rtx%_5=JcWUH*neou_6s7+%z3nKOL45hM06dKYTh6!YBgD=a)-VcW)f)61~IA z73h*~YEJcavNoXgo!g^z?vbFZ&VR_jx~SN9e2v}@Z_OH4ppUP0rYAIPn}6@lF)@QL z)w1KqPH8lW!Y8-rlU$O)@Ml0`Q#vH;z5zV^MKur)k?^7=6I;U3a~V{Dw4`Yfk-Xk7 z;BNKK!dNn1P`AIzvM_Wwg5y`(C1l;PU5KOk$@b$IlH;tkaEvrvIx|y_k7`LgQ@@`A zKD?kiqpA0i^7?yROZtzjsMvHPU)nH~HT`n+%JWazkFrDwb|4}nZkJOxl@cD{aL+K- zX?R%2-!gWixAa;eJ&58`Y7VY{!T&H{^kPp`2@5V!W9ECSz`3m1kj3ik%@6FW_|(J$ zp%rgbf%?-P)ek2eg-S}Lx~j5W}m>F{npX;ioGO*!wL@)noXSqP_t7MyOJuWfh^J)3rM2`$hzy74t2#!%nbAS890d=}k#Bs8;?BX=3_4=Mi_@1``U8s*&%CIu3Wx=Cs+`GQelWgC8&!m5K z{aF9R^XvrWgxYuY#uOH1STmA$zZ&iy-Z%F&M2z=B6{oeJ9K(lstGfen`yMMxtL~lq zre{tQMU{po-n3`6`iCUdvAQdB28W55zLp&O{v`*|%GJ|TO_6ih&+3Yk0&BcmXJUJ` z%^ZzJ9JAez8|mzBz~;{F(;Ksx6iOJC)w%loJu`FU6daQ>IW6vVt!=fRblV}i)`nL_ z&BM@?KS*-e$J`84*6QCJNj6-CRb~&1!;S{7XgkL$6@_Db^GdG^6{MA3qk;*UIIIk? zbh{C~BMn08HzM8!K_^%-CmYLbiCuP4>r0jG2bUzox@_OCoAb3ERFyAGwP+p3_<*TP^<|AnhA)%s zWWEgy7)LVLO$N-Xf+IGrx}fWc&y+&CS?!|XXT>V+@~mN=GP;qh^y4H-@Q;_za00hl zIUDJA@V4Eo3y=C2lB+$l?Pk*xjYWXfnNh>)=#;yA`?G8-^`Z#rKmX?oKJcy$Cv(|s zK6H0GsM(o~p>Sg8v71|v?^X;ZK4%LhQHj0W!h1F^0yV(DupNjNM^7eTL<U|MTqoXO}FSe%$Euwsy0uEh*;F?rq@9!0dRa z+w0h}@U12%T}Xn{z)&b+G{uIO6#5SIDh#p~a?~!3)(JZD59K%ulMjbG$uG^@^y!R; z^>5#s9~nH4j#wT4wJ7iCf}jvrY4idEhFuE%HQAatH(AC*Cgd)YQS3%y%{A=8y~7&A zi4f$1>wk=6jf0z;@ig%3EmX1Ko1@aJ;G0s<%`CSp(aC~N%}J6nXq?Y@;rwTjS`?gy zJli=-1lM4#ZjE$n5l5Fu!0P&AV7mMddbfmKR$>G#5|ou7BYw?^m=13Y+C4ZVgo|@} z18PVHyKrlk)KwNuKr~BwEDeKHdmWbqLz8g^AOq(Kg=DS94{v=H+S_C~t7$KHl+b!K zv9WZ0mJ~-4{dL2R21d>#%BxaRC@fvVMS1-2_UQTN2jei6l#O~N+E8r;^c~~9K&Cpa z#cyvfIj)U75_H>0(ywxCk}~%467m0Pj6d0fM5JeAEX@CiX2uj6)>*yTat(swU*0CF zsWC;b)3KrCEMU)qfxDjW zfjo=l=r)DWx%Qne%QD2(KSNiMxvOh**cV*w8Lb}~(UzB&KW=)*h`IU@EHB5dhbua( zNj~zyd0RO0D-6CTb||m82@QI<WDbZ(f=r4Y~Smfh`-Jz3biW4EG_u2JY57^8I_+;HapLfnBG< zRo#a&VwYugOENbTCDAyL8OFc2pSxs^)1ai}JP`d;PVo)~0@=cw?GA93MYP9pwu5O3 z)0POD64UmZqg75hw}qVLUc7kGQ3!RDM96mI^)B@u4lj*X*d`ta^prk!Zr3)drwoFQ z&}>uO2FC@0_CG4U6JFxJyC}cA*tyU|VH5*Yq?8QZ7j9Tid@25@Q`(F_IoVShYSxv+ z@<6xj;pRDLYGKL4j7^~waVp6|KTh^nmE}B2{oe`yW#SOXIX%V(Ymg<*DD^y@((Hj; zHOaAZ>wBV}2Qje^^c4122D5z5PPp7QCX7Vpx6#MDB^JF5yJ9Y@L+`tig%y!LXPH_> zhC2YwR*dH|&9ojaWMM=c82< zEc%BhS;}13$4nn@j+!HQaq`+lyORV>Vm((rzCiV5Wj%g-k@-TAVcp8{hTGdC|4A~~ zSHB}XNCPwonn=lSmlX3rPZJy@h8EI#hr4i!6?iAY%}Is zb|$B0>j!x?p*}!=ZqXfsy%#TF8#IC1>|v*R|NVpA^Zlkt)jvXk9~v#~j7Z+#!GKW+ zMRHj6>QeHoYMjZ6*`q7ed_0l%Lm%Uh342DxybXRtF?DjJSCfTp-1fLvUF+1HBFZdB z1b8~!^_UauV=xGlrCt1@kH>VzeI~3m%mvR_Xw^qBXcBmju57xP#>y%qP%hUWc#yKUN>umO9PQ#p-_qqWi z3(Jq&GSd~d|MNk3ib(Af={`@4OeC#sgwv~>N4Lnxo^KrYQ+^Yk+|3d8(#+TRkwlt% zxK!nOWiY7b^>(@vLTs{e8sLc5uIK2iEQA8xoY>wm55y&pb(=_-0rLiLn+W0yy9(n= zH~Ih?%s+tUidDUK_1BQl=Gf-ct`kc%YHp)UF=uKuM^V`2o{9kmrDA>yggqM)Ul^d{ zcQ$jmk;761R&f4|WWNo}tT~L5xy^ZbEt%uTz$kC>LjNQ$?`3LUtCz0pN=iyR_E3+# zT)-+jn?MGW-%Gfn>uf^dtx`wSQ?Bup(GiFvnwfnDwqY8o_|NS(xMI3@>7zQBCO`K zccAG_H9tlzKIly3=l5Gm7xuFf;Ikd$SrX)-by;rr;6u>fMOyYVmnNDUUWI)Yp5+t( zxeDZnXxm(pn3%Z4tdok|+#?df(K)b?C^VOIh_iA{CC;=xR(`1~xsxMxM(-YwJJrwRgT zp!xk`X|one9i@eM@E?L)$4t-yA!8`&hqfd`^R5Mxg8EL zrG}yD9tCT?occMcesub+kq!b8qyLqtJPuD;Z#myNhspogE?I8%!X^WBzmox9sJ>!w zT$5}th_hbOzrGKG5yu91x<)$OfFryaQtX33JivbX|DXTF-!z@S=!jY&mRD82>X1Ed zt>3+e-h)6|`Tz5xDci#y61-K_<_Q8MoHv8@EsGpH0_E0-yH&{`zFz-`)=N1qsa2_wq995&=gCJ9N0m<7+1vwWPXp z`WKaHIA1=8%gIpTT!3Ix?B7c?-1pRUyV!Je`!FgP$ro)H{UvyzNigqC>pbZjn39Ll z%SP)e9D!1m17!WMRH3ja^clFlIOLacK+r#RJrdZ(>P7BhbU#_J#&Ocd?tS5A2&V9d zH6~vh-8XPOnv?|_!v3G#dDZ!o3}&tSBdd`OO1Gbib7Sp0oV~=J5ZLzTzg}K>yRsr& zzxpN{s_SxhAmKgH*`EzIVBd`ef8}gATOi5*L*6XVYaczjCEtcQZTwkF zi)MTvopfrp*f_mU>Cy;~x=#`~x7fmkD^zIIy@Ni$_5J+#Pm=%(O2=Wr>~tX07UI8T z+WE6x0y;z^!bqeQ_9Ga{W>WCu1n=I<^*f$k=rpb&+A4NvQW%{3(T|kr7m%xbzmxn1 zI+3}}w`#l#$6}Wj-cGTVwB)DKaUL47lGS<24Z&vZ4K~%)GY=V9UPebkHJU@cH5C`g z#^-C4)uP@vU8eWl0+rY=`+S!GXKqi|Y*joDMw$l>zk@j^vYiKMM~TJ~oV{j!k>B0^WOP)fG_{gE}lXzb5(2R=UOW*adHG>$Lu6!f#CzEA-!8vaJ3L>Jq; ze7sY)_h&jVL~P4}jL*0xT~9j_)MS(Lf@jOxTkMzM?gml)$y~O>mSEriIFf{r!64|b z+?cUKYc^sO@hEg+d_GB}>$p^-n)|64`V~0&kUN;G;#lLL$IM&;e+9~VY=!6xD&8#+ zr65&*Z=LGqaj0S_cTsmo+hq(Kc z5!O_S29nhKOr0oD73GP}(_^AXEwM29LnfE3)2$ZMtUX+F`;HMhM z=52SkQ55KfazBtLDS>k0)BgT!I{n($YA1)AoLP!-u~@Ov9dL!~U@aPzgJ=TfqCH=1-d7_;R^yKA(yD-90NYY(w(gRkLE;BQ;6?1lK zI$C1NW!7=a3CHE;XPIy+b&!`?_NKq}0*b^X#$Ry-y^dbB8cY~HM&oS7Ee}|XdfAn- z3)Rb+(I#6KJX)W*3t)oB7gabca6fyyqM35^nJ_!VzkdSSbIzI^6LZ(|U=7alG+Qo? z!yxlTQ;2%L28%lwP|uW)X5Il}x^3ywp)ZFE4IFpo<>lq%E*ziYfdGNYgblA;>v1-( z{v@NWP{uAI6na#8;m-&6(kUdVl$865j7tC;0WO!GVtdKl!NGyqF;#~MD%80-maE2yEqQcEsPe&n^}sBmgP@oGv=~2~H!<+%t_LEnYaev_1kJl{Q}S56 zBSx}5%l~l2*lM`&f%nNC9DLyqFd3Z*s!8>dm+~OrO^Bzt{0vy>RhTAtE$mC+87xv# zQ?soO7sb_kpDJK$J8?f;vnAEK;UXiXkJ)fHsA?6htn#ZyJkZ}&p=U46&cOJ+AbChy z2sk&h-B!Pi-s{!IgkW$`(7Oa)>tEF*sK)wA%~eZxmqKo^b*Q9L`3MOwbBKu<^4U$? z2h~(Gpk8eRRKC3ij$074;Li{H3R2QcP{o?Ln}aIabZ7oYiQA?jct|owdUNW=QJ%1G zpfOZTB%cxq0$xFItpMf(nCzC)y_N`;NjwEo$w<(UjPe| zf>Pk6*=8)Kk=^4BO$rsG915K{_mu$7M~@z*kP_?sYkB$aq}!^5a5 z>|hq3tVoiOOWW?9JE8x2i&x7%cUZJPr#i0=b*`Pb7aTdQUjTwlJ5>-r-~97` z*JdV`FFUGEqMq_wy94Efj(8s`KB0n~BDe5T^X`OGoVW*iW+X10hP7V=NgwKiB@Z3+%xl=PU70FM&~u&!cnZ!ed_tQ2>~aFvt!G0>A{2m4zHO6lP>bY79$ zmC1QipsK&PP0eTXu0KaL^!v{(ueNv6AlPr4^Zxd@hJ9D`_r3&RQIr%>&rF?CGcIuY z0caNmYKFG=Y+8;10qGmK`G>?)oujZctlR#}{C318RRRI#{>rV+LP+1FEf|= z1F%{9-{$>os?O8&>l*^IQi{nVCJqjjn;iQ0qnVYf3yJ~UQi`L5eVdt^AfYM0doLxf zV{JH;JDF?-o)XJY>Pj1z!*Da9*WrW%Aak)wvSZ4T5D~ot3(`JxYTEAlyXqh43 z(kblJTSmFRs&9oig+U_2{4bfd&l2~g>v3{7N1kc+o!h7BL!Ea`Y#&8@x*=GpcbC_z z3n=G8eGC=Z>5MlyX}iOe>T6|huFly zhj3nn8Qv0d&c(v)?zA`9G-Jr`sk|1mZDjx+@RV*$De_3KUpy%)Hy3a|P=|CKxb7mC^Hi_&)~dCW?e9acEHy&hh1|e_RjJwrPf%qwlj=T8<7w z=dxP}*IVVDCy-!_9-0xu^ ze^AbL>;4WP=nj;#0*Et|CyoLjHq9&^Sy$F4-_5=k!q|T&UMIeSN=Jw9Vm1IWe+SR% zwDu~erH%_Zod$MqfoD&gD_XCvxE4*tC?9cG{)b%g;43rPph?_!zrB9s-U}Ll#bztT zGa$Tnf_ZF4o>&d!K9>PkTXd}M*?NfAHa`&nff@WR8*fgGR|$$7RZpX&9zBFr1cxpZWDc=5v;xzIlZcl{d+_Ud4JxHQaE5bkDl$67;_C6lr z<#*~?3i4@>-*%{xT%zFMq0ne~LvRTT!Y>kmUTx+% zd9>2;5+Ep907+St3&En#Ap2zE;_6GX#A2)vG`kS5Zbiy8tD-9V`zpATEiXviSVSxK zTZtP~sD;U3#~>)r+yg-w{1$MU*Wp8-^x7k#nENa&EGeXqWQ8sCEuWycdH~?*F4o9K z<%31<0^5f_>O575<4ESt*DA6zqi|avW1^&_bkmQ@u)$9?l}RM6^+qilfd|@{%MN)Pa4>z2jg3W zZ6AV{P`2QqcNw(;uDyfnS-5t1SvWO+!l-ne>b53gcoMup1BM4eL2D$3)(QS|1)Ozq zINlF1#evhseYjg?B;tMCm!-(%x~5GoP8CeSq5sO0a1-EPsw;y9_kfB?eb}u>-`E+$ z!^3&oxn1Ic$WCs%I7noz>pxUxNC*MwyHs8oB)uKa_fL)iN=2*Yu+Vk`+}p>;xwKD> zO+`-oCl!LBp}>5w-~DE)JBQn(FwJkU(%vW!2q{fsn2MMm~!v(C%y(f3sOxE-l% zs>9SXSc~>9Wt92=-TLZy`H2!!#jbH@L3o#x56}u)ocabPR+<~f*FSqJ3C2JksaeamlGjPA0&4bIKsaYe$}Y zHK%xK1K8oBzC^h4S*bIB-+1qnK~MBvP)N3NVuAqns+-RHIwhH2Kw*a2cylpP0?Ide zU1rmLApY@i&sKkY7r+)lzGeKh(da7Jt0Us=B?1G-yi%_eNk#>Kd47XDfJArQ**qz! zSDDU^?7fsUY>E&n06Yk8A1W9a8Lk7Uezi?0NT3(H?XVE{*}8>@)jP|z_Yd@j@DsWq zD@xe_x$eDwvTZl=`Kyht(>)K^Pmtqus!@1kr&(KQudrh;$pEbW-p zU;(+PZAwd-Wxvw>g|}TNs3^57+~G*zd8c_obDx77YK#(Q@{N%7NuakAP+8NzkWnod z)}_6&mNj+mHq(GJTH{vKktEnZ@icdPEJqb#q@SPPE=ln~_oW#P$h~0SdxE?Grx`?v z&n&Z?7r;9FvJS&r;h$?s>PR>2Hq&D9<9(jr&bPnYpVltj5=O}g z5;Z=YCBg5@^3$@B{yp*XXSWO_`D+B-cQVT?da-tw2*7U$sBTq3SEq+FgabK{EXX_7 z97)Fs+B9E&KBDsUz#uqIZ3goPl^vEy2VacH^H5Q&kF}~`uF3ZJ^sU>!NiF5T8fe`6e^-3lTK?>b*HgJU#G8J zVGn1k%h?2qC~@7;^NHIB8=!p<>E3hHe?@O^Ie5Hx7l%zFmmbU#RE(mVlDlR;X%~MJ ziQHg6QQUSDzP%5D9Do)w4!{WzS7AP>Eey+R@m&=twM$eQLMGWi=f#{^7C9fJp_xFC ziJEN+P0We`>13CCU2_^Oq-f~?6_(s}d8O`kWvSI5tL11(7Iv`3>}JndsL%e&GbbtO z!Te{PqMjbH#sJiY7{mUyM!Y(04`xHqDC=O?1rJy9LdmdJKM$GW%<0XcPFZ&MsJ^?a z(!x1$MKrJo^nI8)_D6_B5vu-RBQ266mZeMs|T%zX3q|KlhJrIe;0Dh=gb9%o2Tt zil@~kY6s|4Nir}nbo3HaSqYKnmO4~_H#3xMla>U%9v$^UeZk!&tJfsdd`i`>>sD`+ zp(M#~2eNWc-X@88R}bVpl@-GrnK<3A3!Q@I5)JZO^fYvAq1aQcwbajKdgfT154%93 z`N2K}>&6=}K}dRl;BJx6b`uFHX19yD5HEs-?&MOeyQfO(J#5jp5bZIPB7+EsnaXj? zIo;X_;Mg4S9cbux+QWcSr``K-(%g)?;8_7PHpWsTzwo{4)q#q(sFFGiE9+07DD;p= zlhrQVbi4kfV)tR->VuAt>2~ZHhb$}258NlyV*YD6Zi{+ogxq|(MJe9P`(8*vr^OGE2HmIqtsT}!ELevb`Ip1=N+AE`#(uiCp-yE++5MO>9p78T$8$mH-7x42QuOdm>nl9%@mKGWTTKihA>I zS|^E0`)Q8O1YSlA_HStZ{u{rpHPkjqb*2*reZ1&Z3yyliFp7+r%Bxy0+pc2=Ob5k# z;_ds?zQO``XMZ~`(mkIl00_i0lJALC0eFj>IybRceB->o^lYMg>Jm2F`PXZ6rpQc> z6q{t(O;)Sdd%5RngiI%+g8tidD4dCNQpT64 zfD4bqw!c9s(b=aui8cj!=dawl*rbIdUdSb%$98N$#0(G{7C#^U&qH|D>Vf`c(l#~; z4Arc5wq$vd>ia9*C0+*Nbanyt`gI;IyA@dA1e7)lAje8kr7roeBlCE=-g^NFF3y?< zign?XCp*~t?yA68*8+gV9wApY_xDrm`{VXoiIT}K|CdF$KL@!P_DVXDNK}80Qx_dD zuZqWig%3kH6&1dAzFi?6G*8esEarT$Lg4X#;*{*YsFZxv^ywV6?mg*`rK_%_u{0 zaq*(}@0ExHST&nU;k!z+dL~Zqb9EK`G5hN=OU-pmz1_L6JG`)?#kS z@M~9XZ*Nv|19yPso zUeRPBW|`xh^jU!fruMDT$O;i?demy4*_62sdOKgYZYwjN=~+civJjkpYxCkyckUnejC;=*=ZXf&7@>9;ljb^$T52!z!oWk_0q7R9=)AOC{pM92C=?C< z^4Ng7SN+`adcD1WEfotfL^IeGY37rN33%`YZdP*+Z&Y+@@{Zp^dxOk3V{yi)g!{_x za_|<#?d`kg5(49ta#(%>Ye>Vy!o#8!uCE+*wYHc72qmu!A#_=CA?%@`WyhIK&>;N) zQOrq3#(M!(6*j%Rv>+wwd3C3EaNB*dh4zmJ{AdjfudDMdXM&;m%^aM(A2kkNeYkC| zQHs`EjWgm5@Yb6@AYTwPt~=>p({Y=926R-r_tX8Cfm|V7JmXJXcXm4ebXvA8zXA zQsy#kOy;@^n2#CsmlX>I?7+a-#N>}P+57HN%Y)X!eOBVHgX&UGCEQNkD1`Xl9jqf6 zlQcWzMitO|LS`9;H8$aA=)m7CsL3Cgn(E9<)u8-+K`!wU=BCgAN;Mvl$_td3)4-#K z1C-J>rj{hC2JXdYKmZ2ZCdaK+VE+m8K3YRY(}7?R&3hX$5owSGNV@a=@-`wSS+8${ zxLg&vHG_|LNJ&wo&kzfQ97RaD+BN^|Kec-MuYoexm-%|7nxeMfj6_{K&(yW=UD>_N zKqq)_O;pP6;5Juq2%R(;9|%4^v5;^zmYajwXCot&K_W_Mmw?box5JQdIxTx|e=1Hf zGxGuQ5Q|_2A-~$B7UCDVZsx~tR?cro+^snn6cn`N)%)hp{rKmh3?OEU7PcSOgMqSV zVfmjzD%A+0Ceu&BRFnBwU>lpDJ@OgIbQk|GKOP(!8tpXE1l*mP>94uI3&gIy?U0)^ zf=aU`)F|@Yl^-7;9=Pk8zWr+NVC==%g{Ss*E6(LsJFq@Cpm{E(P&qD`w$d^$FAu?Z z5mE}Ec?fJZrH7>|{^w7w-VTIHbafu z9biNFO2-O78)|Na13 zxmpY6$Dtc;4DIWYa#W7PXbnRCwdax`*!$kWIzs>%Nj2jV?tDsV_xTibe0i}6+_Af^ z%x91*PPdn0sO*_F4`s3YK_MYeAv;IQtnT~?!2aV;_8|EOU;cHf5IOjN9Y{noK=dC^ zj8rVZ_5If_?a65V^Qi6VQuZu9@chb7tY*94XP>mpCVZgpEwtT`r~7Ns5s5g_n?wt| zYk!Zjv8IVC(F{rYkOJDg%+$=x!zZyiY~%My`12dBLWHZW9Osz}c5n9=*z+-py1W_M za2d#59wIFKa2c~|*ATf+f%&tvm$bE`^9?F>VTqpRA~ChHy817#{ymB=KNbQo1i}Su zndeUc=2{^!u9jC?l!M>y!u~8g82#tB)2Klr$@AzqwdmQWan}pY)g<6P{2{`OMD)*< zuHkRyffnuPM-Y?syOr!;T$?UvE)$w<@ecU_4Cn6~dmK@_0;8Z3a++sZc@18-} z+a;nZ_{%*HhGL!;jGCIVl(@f`Zrfj^d^7*{S;Snxn-2>ks41_*p3_}qTsmy{$IsMi z9I$AP<=MNDH(o?b@2qw&LkZz$=<|IBqR(B@acCnHyN7kFA@mf{zdn5C>{&VBta(8@ z8HLyf_#Q+Li^wriG)%f$Y$x`lZdcrc;0>|-s zK>)6y)%Qbb$lLZb5dKFge}wQ=J%Qd!7wJT8*;y3Bg;g%sB5BBjg~zB3-`}n!GBQ## zN0oLHBNtYKn&-Ns6CqQe_Dr&;|tIuUzmCx zu%a;s@gLADUu1nVf2Ez-rp9HAmSre9Ww~x5JPZ)6fqp_dL@8Z z#dpx7d@4Nj*XM`K5O!)hw{8DSDd5lAmR!{It|Dq@8Aj|VEaz>_bWeST1ziPJ5RO<94=uyfBrKB z_pJ}-K^zM!-<-s9MwlQB(!LD7u%~Nr+=D`it0QLMwL>F4kET^BYeY;?@z?>u-I$UNvG5)TcS-a z&THmS5Bou6hySRY5P!e=JP>G@Mi@!U3;fknc-W+g6!qM0y*h`8i(uS{Mkw|aQOMu& z8dll1t=SBfz1J=<&~J+7U;%e0WU)0}=~H$zw2gRX69f8T*a7!{c0i_A!%rc?gNEPe zfyCDgILDe9mkv3j z4W)M3z;j-nFB7?2%Sg_mo|US2ZIEQq`em}5lSp>`Vvmtvxz z$J)n=!2&~+gv0j-rEHyd5K?TNkFeQ}5v}fjYob`{G987hoj>+p*H@Ayr$GinZY5$e z21`*dATgdr^BEe zXI+iq?8;H2-vkNR{8oGS{VxfZhq8-IKSyc4F_MFRKkmrRaVz|_bFw8)6Ty9hSH^+* z)mK-z%jd98}J!WmR3Ox3@AGlRY{m-rE&L<${1d&y(;h0Ut~482JK`Lb&jE- za;x1P8`O4grZO1Zf`C-Cbww3ik~F}rj#+L@Q(Np3b~Lk;C?HJ@@B48DMMk|-$flNP8k+_V zz%Z6qf9BV4aZY<~vy9Exd=6+Nm(rK7kQsq|JNoZ$Kh#^#`?|zA z>%BpgVI`X&7-QgVHNa3jQT*q3!lc((POaX8NohSpp>=h2(u~;e>S-o$&o$F8?knuk zc3)eS4wj4Xw1g{tiqBxsf9U5F^)}Rk@F&$yRr-~6=lX;X*Qfc^PBo-6CvD?iN+r$M zKIc6PP!v%k{ofX^CX`l4ReRdo8$BuxssRZhpNRea8jnEB7eCIh zdcu{Icj&Vg#t^_fA!oPp4EtUXG#r=k{FRUEMqg<^iO|4^gX_xI5nB1w-CNqNFXc*` zOXw>*+WKm$G6+#3#Toka&CT`YYA`2Ad%QcUHtN^&FLtxl{>SbPb~jFF zbsbK50g`%6!mjx^BQCs!2HmwyvnGDLl1Wl%nvU)fI9mQ@NTG;`mIMB}2Nmg~IdbFu zC08(CMidnlN4`9ch*;lDMeR7d_5sgi);iZ8FJRI#IyZ0Mn;<2Vt{>-O%`hAspbM0M zYCL~6!_V)sF5__^KuFN3hNfY+V2FxPAC%89VkK7Bwo0c(97%U3jccjFLPJgDA3y29 zlLB0D2p$a8A8&swBt>{_gC~UwiOgmMXb8+mQHb5~QM&Lu1!em1dcAr2?kr>)u~1r)K-8_n zXD|Pmp3WFum2TJgY`uG0ZUOx@J>a=r*)q(dPRKC4O^VWg-46*3LoUGP;u3@_k zC#gLP-u=KS0(-7Iq_OLHHX=3&nhvT)X_}lD*kV&o);7@G8f0tZV~WZ5Teyb=J-$Z}!_1B$&m4=xN!j_jhc|EV=E@Js)o?gS~MxS?})4w{vxZ2_7yv zM{iMLvC(j61a35#fxpjJ#U_wL~&w?d^K7s5?IB5yQ)u{rfz$+Pk>` z$1C0C-2;3!ZcVPKI@kH7C|p$tPF9v?3A-t}B|$Jh6{D8h8&n&)y5dM)Ez8h*`5DWV z`nU@OBhR|D2p2_-URrl_L`vHnk73nd%SH^ZzAlr;4~kM~kC=@}jQ7vB80{_41H0SK z9Gg+XI$v1D2yJ$PcP)>XmFL%B^qp ziu;6Kx!Wrwzv}AhlK!ngLPV4xFhc@Ems>3WUnCswEpar-(<^-!R0SPM6Q{Zvg+Nwz z&#k%V8VT5QB1As=J3~K^@|g<|3h7_pOn`lok=G#C?UnDT75F8M#BzUqu^|&gmJr#- zL80nrAe+p9n05NmNa6gTtag;&o*cr=f4(zLkkYada$(csNB#`O3e$R6lFzy3xd9uBYc z0cU*cEu1`mh!y-IASOI}=_oVn`SS@wG9q?JpbRNq zOp26y6Os&UV#+o{wjYOt2+5s7INE zF{+l1IM}ilnUqoXL0og=~`*0+7cV}T40=studq42)pC7n2$jZt( zD7ohQhu$&N%voit)ru3ejqM1G&CuocJvA4V~8+PHA%EpZxA@# z&yd&u_enQ=Fip&Nn2g~GQX!3Oh_~+^pZm`OW))(J>CtCJ2vcdZ-!=R@4xV^mk zpR;%|&Mg&%F@QI8?G^`>41=uKZ4e6V)n+3J`HLGb%?+6U>uEs;7y}Axbz%}4huWzT zES8qe8-ny0L9-?$_##tic?)k#^;b^$udCMA{d^O^HZCj-o^2^e9fv47y4Mh+A3#ED zY<|%7!=BXq?@O0P4jMsIu(_b_u1l(istHqsr850|vJc`b5XeEYcIglQ{?GAxSg!_H z{$&U~_*X+wjA(&1_9H|1^9=uaou>{tk!oy7lFC-Tfh5U%N>hV0!vO)+K-Ue?H`rl= zuH!itYd=S@h6oT1$-P*GzWYi~b~>FjqEbO(V9@!oLR@*QTxTyo{riGy+S=x~B#Nk9 zP>nwWd3g-b*(+$$6xde~z=sQ4_b4HG4Vi_ndR;VT zi;m5z!bGnYq&}Iw4)yjy8!J(~6JFati?hL7OVF209qLX&t*wF1g zr1GTc#lz@kZu&jJJrUGV)r?Tb(y3T0t%$T{Pb9{Q((q(W1P63Eb#a{(fT%{~*Lewd zYm;AW<`HRDdxonbt3i5-#1J4Vv$49M3zNb}EL!5vB0L7jSUW4)s3gkfR-m(V^sk=- zDqtjaekmg%)J?%c!Sf(km9(8YQ=owvg0iT+*aWRL61`&~deyu>tik&b*Kc%jTC*MM zmsF6(32@OaY$=4&+zofX50b*xNV2k1J+7&SdV0x7Mh&?v3xfAR*@LIF$rOU?Jt&(j zxnQ`v@m^a;`-(xz){CF@Tv~>)zFNl;iMT~)8T;JOmEsI2`z`9Z-!Bv$6?M*YeLe>2 z2a?QEuY{bAK~fj|i}kg8{YBFOeuJ(oG?j0FY>9y?o+bdI22**8klS`~x~u*927905 z-N1Z_%KDoo6G3KzvmK}MO47k2f~T*@qE?J^0)pae@_6p`UWhoIm%#d1m=h1_10n}D z*{mM&bm#d)B!`hbcZdTyI>VAtZe&zyJflWn_V7DND9Q)k_MriLr4jn$_D|kK;%k>K z`5skRpGY4C=^_!4Tdtx^a)`_Y(%9bfd$9_+@DC`V3Q8Whk}@kGX}2DK4aug6%7LN0 z`lpV82wJ+xvqNDXQng70HQnSHP|Wuci6K7%yz38&pFITrfAfbfi4u}CRt})%nSSf- zi5?6+GREHvuHXc&IcmPe^v8;WX~e9KPO4!JBtv^V0g;h=#%atU&w{8b74qgEKXXlI z+PXu%n1nFmEHB9nqMpC}>b3j30-A{EoBs1U!!MBoKi*<^g!VPQv6CJ$d4#a@N2|Gy zn}gnb50WODYnxxnRUzR!pLMmI)^p-Z``6)9ETMSDh+Ugf=@FY4uR_r-?tQHbM67?W zSb0NQS?pxAM|u0_2WE;{<8$8d47iX=A)g1EV8SEiBSzB(Q&2&$%$jPJM>KDn!huz6r$P^T(4Al z;c-OxE!xPZNrjke&i8#@962$@2RjL^jNnoaC2Xij#?fQS25<>$V@e^^j#&qHl~hiY zu42xa1OqF2mat5|WN{X;>EDK0mu{(kNEtXsX7QqNt>rs&SvT%W+U5Ex6}--WyhCv) zc4eUMZruGY&u!yy_?C13`WA#-02O^8(kThzTevcQ1^<34z)x@084}L^z3y6gv%XPG z-%=cvw<=J#K)n9*;Wi<{aKc)+t#tYIC~kr@7#|2T3lXWZc}KckS~TI^@5Na`BU2Qktd%FWzI<{oQ|sNzVi+xFvf@d_m8QM zFL-r%SQWjCtPr2LLixar@%#mRftU0ePmpx6{<~znYOLYE-y|1R(vVfJRQzfvt8VV3 z*rAw|`sSypaaV;xno|q!)FniY{nw6WoDpHU5u0-R0#ds%f2VxF+GXJ6S;nvL5fWoKa4udc-Z{EIiS-3gqz#^FkgW9@?f+JoCrNy@YohxuL2Nkjw*MLsAi znE%|U2M~MD!XmR}jcbf^^6;oZWX^i_Y&D6uj$y9-_W7-_DF<;@5s|e3oDy{N7&+p; zEnU19H2w9(mm~>DHHB7=Y9t`8$`CN`LAD4cDta#wDO~g)O0oQLZ?I&qq5~XAB%%w% zJ=ZV)aP$MDBqLu|2FZ})WW+Ppj)q4+`gx<9MhYU5P)4bn2wV(glh|lZCP;(2Dm8uVn$`)m7$dLkntC=(S-0 zf(_}B9csi;{k0Bhpx6M3fKG`+Mv&}I4JoZ)UCVbwqJ>_YIgf;E(DA+&Z(y-*higd? z>Z`!=d($V4|YS=v3_WzM3d;v=_LypxT^l9_!L=h zT|h_eW6?;HpLg~nvO!Qnah-;imIG9~!f!?HFB(zm6=$xmPLD@0qwLI&iY%_uUrbYE zxfbVJ1+78_f&E{hvH#D94uKi45-w9u5G9TE*LOu$J>cgj&pxyo6L5 zUKtlqJVd%91Z*1tBdHE%x&CdOieEA2cl{_Vr#S`yG5>8Zi&A~MmuBgO^> zw^-%2A?UZV()%SvC5^(jt0y%= z!Y%rfVGAa0@cxp)R0#Usx?~IO;or6hHgh|0SFUy~4K4WeF0v^{Y3~cbHVNTl+D}pw zju-=R5{wLsArt@pV3m$oVc2O1)IZaO$Sho#Q}f}FtW6^@np&2obx)?BLGLWJZkdFt z(h_>#NX4%Qm+ea-*_$CVP^`vaA)DEI`eXkre#57;m+uvIDp=`W7!Cd&J33kRtKY^1 zLhT4-g!G}ZPj_{3PzkQNIBxN-vl(Q`^gDJ~+g9f4=ri$*{$0SvvWBtJ5m=R@n%Wm{ zc?4jWb;e)>ByJd_J}IK65-<#A915vaMnn8Qeo`5L=6a_rQa#M+D#v}zlioN~m+p&1 zeQ|~=jqs-$vJz0qk(Lv+?A-^wbw*z@!MAcY|M}K)wal4Ty?V}SETvUZ*ibJ-y`Xs;!w47`xu3ttuL}kI&h{nc9tb=>B+^*5bJ5 zFs$DOV9njz2WW4w3ZLXoYT8?NoxhfymT<%*v`wG}Dw1)hhSi~`wfN7X?k?UGz>QQe z=KfEPOM5K;5Qn;=ZsBWgBdp&>0Z#}5tVKZ5W$FadhDF4XTZZcuk4p7^u?l$ZgXL|t7FWp@t$ z%hzSr+{J@VYfOD%)fa@+%r>5~D!<^aHJqR&G+6|ZVQd>~c$4yHJz>1(81M-ZkQnAh zRt+BitTYNVeOJ-7Dp^n01G5J_l%+=+rVP9fOWlVCS+mD&IlFKFzLbK0PW9AcZ`Xf3 zt^oeVlf9eu?)S}Nt~EULf6gDg99>*@ZL!bZwYf)vnK0RSd8RgvRaw()A`r`CHzJ*i zwz%(gW5vTu2$%HLy=5myl>hzfH^v)W|GWhk3V+XKW#L(`5GG>Quu9aX*i3r0oY@Lc;zV88>dGXQra)nSFJBlFk06#?|UP{JUd)sH-}z$ z?YQz%EceNPYcw({+~nRF3XI=O{Cyv0swd<9#NF zixN4~;r{%+)HRcB;nQhYT6w!+0pitNv(=IrvCi?)+Gh>j7MCL|&_#*kq5T;9;=^TM z9(|N;Ukgf<5DCCu5X9%1l3=1(m8ovh6UGX~j0qokjDL;91t*4l4NepbP;3?JwysrF zdns<-l1iavp&r(3P|>n@3)bvv{oXyeyGu=YCW>z@AjLTI>2Xa~t=(>UMXK(MYrjX; zknG{CNmSu97!!cqPfO4i{B2O0k7BOVBOm(ogUx!^=O#qjw>hL+6KHRRl{fvzcL!jD z!L&t;7Wn>tNb`-C;RS4D$!~ojmaR3sOM>B~-8(ps z;NYa9!b*%YAJ2YxCG)`s`BoG2pV;*m)$}-j?21k8)6s&wfBwETX*-RgC<*$|duHz* z{CLx?r9j3!^}UPU7yF4_a!jTtEV|(r;<(by-H1C3|9ah?{i?gdW24nIiYzdSCq3=5*F$cf~8J=&Ub{ zD{Q182?c|cw@6WK|61=JU;ZmjZTaxMY5J5xxf($)>sQ6_RE>+dQ3A**&btQc{jda~ z-1-}Qde$3UaLLb`Am6WNv)39f(frwMRw~$0=>SwrC}I(m9|_{ABL2Vp zvFBSgBb)C>?G;*a0`9|e@gp_6V)lfJ{&X=mUY{TD`@gS07%lMsYvDCBW{zWUvYu|L zJ6oCY0<*L65*FqmvexG9p;kf`mq+f7n|#q)ctwg?rzc#k`*ZtQsa{kq7Vr9P*MFqu zi;&|pr1xtex9e<3U-C2luG0HS&fV|?R&uxJ_V}Rl7@i^`r}JX1p(E{|JCSv%6)?1_ z1PPwkd8C*W;^MGhyVOK?#yi~{a&O~iUXA%$ybhr)G^Zy7!#&_0oaDkIVWC#+I zT~ki2x^E5c5w(!tc!jfMzIXT)y|==8gq4tuv0Z{++~0dwO~aU^Pf4k;79vMoaLSuF zpH61sn*dsNWqoRMuCLQsyZy9x@{Vo0_;?=E+wC3Co<|>7(q`$kY;qcs1rPK6Wk0ff zcMLZ|*mxg%U<^5{By6UidrW0L&)bx?Hdl*C>~GgyUUL13UCJy#g&g1g0#SqUUp6(* zNMh99=iUYjQ5!w_YD>};hr2_A6{o#hQl;2r3pP2o!b)L9|Krc$au`Ymr0Q{qu;lgy8zWFM&}2_L;z6W|K31w7~MO ztNs6f4DlrYt4s3UxX`>c7^LKavRhM`_d(#+UA?eb4hdqfEZfQoHrv4KCun%7Akhp0 zp5n|-G7R(M&rwfFw0lxdNcKW}3M&SF8_H> zE%T>zIlr-2#lu%e-v}VH6H9;!8h;{B7KF@lM8>}%MU#5y>Fgzl`#(Yc13!Fy&*}Vm zpI0ZS?}4z}vCk~-(|q|tsxMte#EFuXHmL@=!;Gh>4ZR#IKt^GuU2xW`-f{$rySmp` z3V*>Yv{h%qMguBmB+d1!&Jw8O71h+#2*rG{d-m{<5S}V*w18?0`i{cnxcSF`kSYX{ zF^<7JzaW_EBM87K6-cJ-U{c-brSg#-Ug!X|Br!(d2DpF>U;^OT zOAp2#SqX!#Hhc*Nya}lty2pt)sXr&vkZbkVfPDjnB15DC4t}jfc*hxFV~&=kK4rF) zQ$XkhgJ`X;#BqEXTS>;^2gAD5k41(qhcQZtA}MJRA>us->b*mexonO#N7)d4O3d!h zkiFr4H^rNiNg$bakCqXG2w?|Ti(KU$P-h*2!lxOzeVq**c_SuLea_q{BohkNg?zZV2dJp$f_ z8_7G1q}a4XR>Vw@>E3i}0}zXk{mwb{RFB?+%Dp+rV3s}}AkUgkElq{Njz~7}43GsY z=Y#;H6@VjXMin5GH-Y(n1yaCER^E0Way5aOfBbt+_OF+^-P@4E4X8;_EtypQ@GSun zMoHmUM+f0RcNKL6Aj3r7!W-o-Hmv^$M2S9;7!MWdVrNNe685K%-_Icx%H+M;>DmDE z4`T>nwsgP!^h!?vwzQCCn1*7hLwQ%ipc4CQ!F-2}=7YnE^=_C*AvP#-L7GV>O-78d z_CmyC+V_|SiwRTM(vpjSF{rX&Lgy*IFDnw5ycVm)kx`7V(;#Wz*`GFH{3GJJObzqM zQ)WUcbUqLV&2e{YwbU+MO(N>+c5A!<%g**YziW-nV-eEJN_%HAG2+Q(BGaOX%$S2% zlAjZaeT8Eoi3Qz}zJ`Y`#f=ukoHd1_BFW0chwDpAFNkw|C&#{!l!apxGol-ZrfMp--7!?v{)a7(SjrxX1uO6n8ymUf48I8-|X_ z-DM~;x1l*muF&lxP>qn{Hef2IzTfy48Fp?zc6+|uN}k$>EUDoHal)l0wd(#$`_C3c z8~tR;eDFJCi_#^lHRP&0!f3O3A2k9aQ%^z7?p5j~B$c&Uar<^Ra1~9guyfyi1l~Vd z)OH%A6m-?iIqoNq1j4vhgT=NSZYZ(Pkn%d&We1b|P8~aX4;d1Q4wKi~OUbfgYZrTu z2}p_o?<&5FUonBnoWV#o|LP@+>C%{cjlzBjK?bO`F6b`n33gh` zh;`~GD9)_n+CWk!Tgmq3PO+U;E3?gkm0l{*fIS0eEkS~LMZ@Avt?zPOccd5etCoOK zZOLSM0eqqh$!kxzLt~G6l`Q5Svs_eX?U365&!eId{G^yw84hrpq@1wHg$wlbKS$KN0Cdt}PFhKNl z9VT0=(@g*TVr!U?zx%UT9pMLPzTf#jsx%#1g(Z-Zt9rD+=c5cDxZ@=8U^9ejRGiVgup&gEKM!^buYjx=)eiyXVkJUu`SV7e=w#heMU0Mas+Srd& z^f6EIprHQo)F3?k?T}o8MjxioaXnTEpn_y|>f0T6~p zjPke&7KUN3a?w|5kHw4L(=Gda3ucG*G%tb4xv@H;b*}*d{l29)6lB#118Hgho0=ZGG$A=TWK%C8sf*O^A7lbxK4oO{go~w4SleX ztKU!Q_iGAp+tXWRn#?)_$s>Hq)#L%F^Miw{%Yr{~AVXCuGh)!;H|l3ErJCiz3PH9n`skzv7*x-`}>hwOlmB$2KrAY%D*+b*$)_c>zSGJ!1+MHaH}mCwYMtZ#koA-M3*EKSjt&H~&BW{yIJ~|M z+?GWyAk{6sNXd)0Z5KBAnBfNEAM86YmT2=@xwnS+mdEu;tzU&JoUZ#wjil1y8#M+3 zht6xjl=Y(KOdJ+Jrl@ZX;vil8>Qf7bP~2Z77)U0MKN0izM? zR|R_yM*`9qv*)xiP6LdDV_Sm9$M7^LT)f&__wISgUXUK8edz!Gzi))|xb$hRgU?bg zpt+AYY|(CfLR`vL)5Dsj)bu%qT`*DV14>wkVXf%xhqG^toUZq3hbuG=?OBuz+g(RJ zJsABLDy-<{P1*(^=1iM+G%%}?m}pP){-x7DTvfI5*>$uG0x-qt*-hE1G}LxiU;VhZ zcQS5n7$@zth=Uu(t=(q2&_9+4;i9LjVf)&ctFYa+KEq&bv~o1hU^j$SYx~ojF&ast zJfMoKEt`$~LaL{=uv?4$YoT(i22C^Uov20M*donlK>8dmw(!3hYyOuweyKc8lLnjJau+3)qAfj=E zV##`_>$<(5Vwx8XGzkMTw%Ydnmz!?^c5i@ei3yrt_*B5bJ1)}G^CmYH%?Y)3^zchY zwlG2Y?Q~93j&oBQU+Ir1K;6YoOQz`@7=;p!2!t6zlCxEBsh>q!43PRvdxsvB=Suwq zjbvO_7e-J++lWO1w}Me>KnB{co*aRxHIj!aNy5PWU&!+GhBzY_p>KR(SNC$YUIWj2 zoP{N#*0OYJ6-xLXEsZdc`J#|~NL#_IWY@}N2#(uQ92S56!#Cp)msT)1#YhxChVV#J z3D;}y>xi4RA_fxaR7@={zAH8NMqbZjA_m#V3`b9xnt%k}H{1zu)9DZj15Iu6_2SKL z2SdtpyybK6HlTAU1B5Fc*6Y_qU16LqC|6_mEAh^+Ap>E*96|<}b4HZJ7A4)MW@LP? zE9@&`b`&SIOWR1+gl@`WOTB;Y%d4WW=h#x5#E6%=z8CoNoY`s*vHM~GmIikwAzjR} z6m9@XX$%V-f@EDHU6%1n>B}DSnOfEiHFPl>NE*2R@Y!u-Lg6vDQ?{$eaLK&ldLPCkM_H}SFP^7I0M&113 zut91V1bOw~B9b^6U+*LJ1O7xzA4T_3ICjPJ@q7OHrF{p8vz${s=dinkX?!;kJ$PB` zIpeu+Z#lTTXjt%!kXtyytj@aE<@1TDXa37B(qMAo#NFh6%qRN_#j;}OikKlQB-dc{){&6c zm$4fx(T@FDG9Lc6Q-MC4XnfONx}QH(Rv*TOcLHA0Q=c)2frfNi9+&&-yw$y1SC>g# zg;O3#bNS0k$`}Bqvs$q@ctV#wX@hA%nKUystBN&$Vt6-NvE@>_kOL(B5ir|FKyB%V z?ovy~QKi6HNbDk0S!O6Cv!5qT$3L@MtPo3qYjr)6|3&rGNm7$ze4J;PlzfdqzOBVa zeYcL|LTf`K2QHTBGAG*iy9Q>f8(qsx3ZigyHj@XZ_pvLNn0g-#8*#owf+DNL(^>_# zM0`w{_mibK8FLo+m#eE?G<@qdRd`|~C7!QZDLGb_C}>|6K5@iL50$!4@)8Yc?mmU= z%lx13v0L69EZ5#*z^87@-05!EsQ^MaYg+rD?Q0EDs+Yh!=S6-RS@J&UgnkHf$(eH^ z<>-!B)3r&NNriDJL|E!kEVAPY7aMjkwWMNoP{Wptp7c09MKemsjcn1j%(*B;nnEm; zX4Pp~WS&Be%)93-b|$AQh?a@ojaGwt)=&tAc?es**c{p;;JN*NfE5fy8aZ3B`m4+3 zj);=sg2>nuwj^O1nZoQlkp^jMe3Qo{p}po{uOC?ispz*eUk)97lFz`Be2tnvN=5N# zM%{E8>A2oWB8lSCby+Q200NhVgPT_G%xJ$pkdZvyrYe$~nofqJ(~Ri0Gd{H_hTT-d zg9XsDOniMB@Zd-HkK|cSNtjmd>mE+U5H=6V=4#?HGJSJ%TZrwJt|{|Zo<-rWvunQP zPfM8vFhbO%@!GYni{_X8>*=T{NDqmSuM1OcBRR+_3IDl9XE`P8*33(-#`KKXN0<2T zJ?^O4%g=(6H9 zBkiQ^RARn$q0Dd5XOr)|rohFuJvWQf7j0c^c1>QF8V^zHJu#PC8sy(f%&0OyeIQe#C}zV$_PE;9*2_k$ z5l(5-#nEC1E5L$_LwuTaNi5T|uTi}H$l89!hX|B7!}_Bv&x!tRws1r5vl6K>b?e3G>!qiFh!BvD%TIK~r*ub#L&N!otMbsytf zRUtjQ(h1FyaW9;)OVDx}k;Y?}gB6Z6jK;B)6UW!3x>FM>t0h%mAMbs5rqQ|D^?SnE z_!Bg@Zr^<*%qV~w6mU|atxwd6?KnaF<^CYZMdB6I%LCfj`x6gd?9b+Q%wTMKY)Du> zHt#K2qr;*kbyW>m=1aoQr*!W62i_Ik)B_Cr&RqJyk=N%O^x{Tiv@-6{P;}OxJiQLoqq&=J8d~w?J7<6xek8amsiHUC>Jf?2wuAE zA>VhYaS~s+&Chu4pw!m}O)+k`s#ZS@37ffE{*nn6f#>%JEYy$E8Y%2k;q}>>HOzg} zAYt)~couuKCt5|1MRN5y`M2ztykcHXl3al(p)31}$?9XlfE3ATmocUb(r+f?ogQ2GZl|>7Og*x=Rf}(8%7189 z=WkWKT4%GwE`Q`y=vPPV$xH8y^bI#mtR*sOZLh|ZA21784nEP|NSRdCajpB=^$(O$ zr46(?#s`Bs>UUb$xow(`utdw)-fc%;P)M2?xE_=Ci7Y5IT#Ia?z5n_SJLdMwCKp}F zRQ3?|a@wKH%$U43uQulzk@S?y7F?sgiw#8`wu)mUDO33qtawMYk`LC>O>~k`?A>{R zg%q->l=~${BhJ&V-EBXn;FzZof9^#Ng!E?>2vcvLy#avMKq;qU;vAlWm*$aiAeSN` zI5%Ta@l#FC>15{`!w-E|csVlu*TFTv;o_?DD zSwkaprr)N_Li(k#?di3NM5|8mz-Xyx7U7#m2mDDD_-`V(!vPKXc%!H={FmMV6>>h7 z_x_6LIhX~F@+HoES<&@yxN~Cp4mV{jvGu!L!qihT|s)Q&UDPjPvV*WT>3g$adZ_$SRlSke`q9*!1HoG{6O^h+*>mmul>X;w`x*_s zUHKQh%!XueDQ$WpK*StdI@9^hO!9yY!IJ35tuy$|tf%T%AJJ7-@N)8{2GlUitZFl< zqB7Xx+h^{qC`Ry=hg=sgn=G)3-%`-vmBIR3pnVd|qwbV;us41M%Q&v{+5W*>vWHhg zI+ySY{xb~{tVB*#rM*_-Z?))Iyj}R;`h+qKfOg%S!$qW8KAb=zx zN5H^t`Drnw&zrQbLBdUxA&ChZ?o9q3T~LyAKn-4 zl(L;>BY*hR#K{F;Gq+22ip=-e&BNZP`r#g0Wyu~7laFsU|DF$1udNdA&3B#iJ*rx=rJ)vGIR&c(ifrVT3UHntzNPR<{`{r!;HF2!)TCzU8{f`6I5s@} zs5S1 z%~~#5s_7J`{rs(YSDy*Ib86q<<4!vT9V^?nzk=S29bC-s!VxTvoz8q8+Ne27Uq&Ce z9UH)>CAmWG)??DoMta*|y4LatDPN9>fLe6h$tJGX`%r-s`&28^z~) zag`R)o(XW=Hl-Ajzqv*&D&mMo!z~^z2VN(ic|3;~x)QzoxRWE5mag*kwjD#$nf(mw zJX*FK4W&%cSNthN#hz)!P}~-s*yp+*Vx=2`19!z=MtG=CRZQrDBXPdV<}rSa>H6@s zh?Dh4zm&3g>QOeG?w51FfH;Gntx3SJ^7VLZ<|>15+;aefRZ^7R%72z=azM+625z%*jkvy+V9mknRba#*+5DH?)t&23NSRSy8^$xuc+XI84A- zygGc%)c+FQ-B6;F`=_U~AGQ=-p|kFLTzZ=4j=gd%$L{yOGnU<-6;D9NY+Z9U{+O`N zi(jReq)!OP-@7Oumsh452%ZmnsQh$&smZAr^&9hMbnoy@&efAwLVY?oJ~_+j)U8_3 zRpwtG@B5C|(zxdCpnqSKhH(`y#J+QVb5&y*>dID1%du1bEQyWftL^3YDm&L zna=%qOL6k&1xa!a>~nhdi1?do>2Fht?jN*PP_^2{98_Jg zFkxy#nVn22=%K`?;3}uVsfP}qq^nsyvwWuOL}qT>cez8g4pM2o#gocmirE(zcq&V@ zrB|hw$zDNQmh5Ax!H*31(ci2(y0+VzEv6r`&MiswKANr5*7G5YiI4Fae-j>1Pu2a+ zvc*Z(s-$Zvk}+Q>_}%;c|p+MbA?SFp?&{p2FXwf!mcx=L-_lj>CtbW?)0 z(sQF=A(_%>d0MpNbnAo=r$<%bbq9rHUI-4gFG$SXVo3IEqNzEO*-x8#py^!B*{&&L zXBtsuW7gyVzV=|VYvTe}(j;RQxNSaoeA4_yV#=4-mLi+_%I4YdnRp9aS~yz`OTj@W z4uzF##V5t52F)$*HnH?eyG$*uw;cT3P^oTl#C2b9*g}SLF#n}D?lYKiikyU@y3KL# zQhaIE1Xs^1ybL3*kJzTCIvL|D*6Y_{dDwP^hVVk~!^1-|VX9miouB&Y_8$+s6WD!R zQ9g#4tD88X=H^JX67lTkcZ-pgA(5TzIV6ty@fR-3O>3v++&#sz>xE0*0$9Kz$uwFiwcD^+f1nx3Wqm! zbET*cn@O9ItYhpJJU^%>*fh`;&Czu%T^{-)KILBFibiFm zCt=eEB)QCqiM)3LA}gIP*sM4h&>y2rwwc-A^fq6%E%g!JCiluXzI*Zq{(AiiEDQt`kYz9~yrAP*dB0A;LFri$=)Inek z3MvJrgABRV6(!PfN!C(VL=hx0FfIu;=XuvOi*<`sJL{ zx9|I&rhT5@^BhQ33lJD4*V!9~@BXI$(g~$x&{SURadRsX#CUUGQ^j@bhG|RQ7!Yi9 zJ_UBb3?m)P^~pokes5XP0&#R7c+uF2TtqSG2=limqv?wKFxa6FP`o{!Ez4J);LJ~p zLiyz=8oV~L8+|FKc5RLnh{pzuC(^&uK_hJ>pMrtYaNpny=~Q_UW?Hm$OGKrtcS(^1I0xJq#Q^BT!7>Bab=Mn~DnZSSZ>eAW`8 zapuaJhW)kS7NWOkf~*YLmeDp7@bU3+FW4`5bPTI^WOT<}CV(lU zPF)$7^{3s429NFV@qoUV9WN* zHQEyKobW@d2Dlik4_F5EH~^Dhks*JGwjn)CU3)%rnf+|v&Tg9sJa(ZXh=P7?vpwzB z-d03f^8R^3g`hG|(NFtB4U~Og^{lr$O@Q$~P-Lf;S@WJ{fJ?SuK< z5DSS@O9gSE^r2yw)KZA*}Z{oV0^WVlcqWT5A(dR*35 zvDX;!kN`XK>uF(S$FbR>YLW`NiIi{r^Eq1`t|cWg%9%L0+!I9QPHVaE#zo>qrX6NE zBk7Qi0@{G|js@AG4sV}B>H$uknGK^dH3oOcrI08q;X-fe{22g2N<={>v%9o4-aS`uFfY@j(v52a4>Mvz z5Xvoi&f9y5h0Yz>yQ%)?_V+3IKIhaG^KxPn|D~UTAb&8`3_iyjcFL8gBxrrdR0+>0 zpi1`C+VUL{WORCBPeJv%@IN!h3UZ`akXqmE;@1VUlN;RTOfnIg_9#;~&+`Wpw!X(h zR6rr|Ow+UwKb$Lfx--Is6)dC69V`Bh>5n`dCJ^4k-O{k9&pf|?ap6OyZ|1`7Xin}@ ztK}!6oVM|&g>}LD7jrFY`S4DYTHtWAOIsWpZ3kEc<}(%3!UhwKt5|8bcGVXe~Yd!rXFr$Tr)^rN%1PM2F5R2LUX9Lbjce2zf; zl8{&p5eF>t4gQi-n)1N49Ny6HCM1u7>{d7|@s{;U7cBZqLiaZ)2dm=74TFk+%i|t2*P+pZAsnfg+7OCHGXGfIG$s#7jhrRWyghA zGw2ASTyG!NGn=?7En?rHkdPJEoz7e|5T39^Rb}yM*pW;lw?mU!7)ka2UpMB#;ALag WX4SmZ@jYUGCSk!5U)~H#xbQC+{ei&% diff --git a/examples/hello-world/hello-km/requirements.txt b/examples/hello-world/hello-km/requirements.txt deleted file mode 100644 index b02e27620d..0000000000 --- a/examples/hello-world/hello-km/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -lifelines \ No newline at end of file diff --git a/nvflare/app_common/abstract/fl_model.py b/nvflare/app_common/abstract/fl_model.py index a82ae57c2c..502e7a5361 100644 --- a/nvflare/app_common/abstract/fl_model.py +++ b/nvflare/app_common/abstract/fl_model.py @@ -45,6 +45,7 @@ def __init__( params: Any = None, optimizer_params: Any = None, metrics: Optional[Dict] = None, + start_round: int = 0, current_round: Optional[int] = None, total_rounds: Optional[int] = None, meta: Optional[Dict] = None, @@ -79,6 +80,7 @@ def __init__( self.params = params self.optimizer_params = optimizer_params self.metrics = metrics + self.start_round = start_round self.current_round = current_round self.total_rounds = total_rounds diff --git a/nvflare/app_common/wf_comm/__init__.py b/nvflare/app_common/wf_comm/__init__.py index ae5c6b2d3c..4e0bb0ef5f 100644 --- a/nvflare/app_common/wf_comm/__init__.py +++ b/nvflare/app_common/wf_comm/__init__.py @@ -19,4 +19,4 @@ def get_wf_comm_api() -> WFCommAPISpec: - return data_bus.receive_messages("wf_comm_api") \ No newline at end of file + return data_bus.receive_messages("wf_comm_api") diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_comm.py index bb314818ff..dc4c8a9914 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -11,10 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import time from abc import ABC -from queue import Queue -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple from nvflare.apis.client import Client from nvflare.apis.controller_spec import ClientTask, ControllerSpec, OperatorMethod, SendOrder, Task, TaskOperatorKey @@ -29,47 +27,45 @@ from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.wf_comm.wf_comm_api import WFCommAPI from nvflare.app_common.wf_comm.wf_comm_api_spec import ( - CMD, DATA, MIN_RESPONSES, - PAYLOAD, RESULT, SITE_NAMES, STATUS, TARGET_SITES, + TASK_NAME, ) from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec -from nvflare.app_common.wf_comm.wf_queue import WFQueue from nvflare.app_common.workflows.error_handle_utils import ABORT_WHEN_IN_ERROR from nvflare.fuel.message.data_bus import DataBus +from nvflare.fuel.message.event_manager import EventManager from nvflare.security.logging import secure_format_traceback class BaseWFCommunicator(FLComponent, WFCommunicatorSpec, ControllerSpec, ABC): def __init__( self, - task_name: str, task_timeout: int = 0, result_pull_interval: float = 0.2, ): super().__init__() - self.strategy_fn_name = "run" self.clients = None self.task_timeout = task_timeout - self.task_name = task_name - self.result_pull_interval = result_pull_interval - self.wf_queue: WFQueue = WFQueue(result_queue=Queue()) - self.message_bus = DataBus() - self.message_bus.send_message("wf_queue", self.wf_queue) + self.result_pull_interval = result_pull_interval self.engine = None self.fl_ctx = None + self.data_bus: Optional[DataBus] = None + self.event_manager: Optional[EventManager] = None def start_controller(self, fl_ctx: FLContext): self.fl_ctx = fl_ctx - self.fl_ctx.set_prop("task_name", self.task_name) self.log_info(fl_ctx, "Initializing controller workflow.") + + self.data_bus = DataBus() + self.event_manager = EventManager(self.data_bus) + self.engine = self.fl_ctx.get_engine() self.clients = self.engine.get_clients() self.publish_comm_api() @@ -77,36 +73,22 @@ def start_controller(self, fl_ctx: FLContext): def publish_comm_api(self): comm_api = WFCommAPI() - comm_api.set_result_pull_interval(self.result_pull_interval) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) - self.message_bus.send_message("wf_comm_api", comm_api) + self.data_bus.send_message("wf_comm_api", comm_api) def start_workflow(self, abort_signal, fl_ctx): try: fl_ctx.set_prop("abort_signal", abort_signal) func = getattr(self.get_strategy(), self.strategy_fn_name) func() - self.stop_msg_queue("job completed", fl_ctx) except Exception as e: error_msg = secure_format_traceback() self.log_error(fl_ctx, error_msg) - self.wf_queue.ask_abort(error_msg) self.system_panic(error_msg, fl_ctx=fl_ctx) - finally: - wait_time = self.result_pull_interval + 0.05 - self.stop_msg_queue("job finished", fl_ctx, wait_time) - - def stop_msg_queue(self, stop_message, fl_ctx, wait_time: float = 0): - self.wf_queue.stop(stop_message) - self.log_info(fl_ctx, stop_message) - - if wait_time > 0: - self.log_info(fl_ctx, f"wait for {wait_time} sec") - time.sleep(wait_time) def stop_controller(self, fl_ctx: FLContext): - self.stop_msg_queue("job completed", fl_ctx) + pass def process_result_of_unknown_task( self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext @@ -117,6 +99,9 @@ def broadcast_to_peers_and_wait(self, pay_load): abort_signal = self.fl_ctx.get_prop("abort_signal") current_round = self.prepare_round_info(self.fl_ctx, pay_load) task, min_responses, targets = self.get_payload_task(pay_load) + + self.fl_ctx.set_prop("task_name", task.name) + self.broadcast_and_wait( task=task, targets=targets, @@ -194,6 +179,7 @@ def get_payload_task(self, pay_load) -> Tuple[Task, int, List[str]]: start_round = pay_load.get(AppConstants.START_ROUND, 0) num_rounds = pay_load.get(AppConstants.NUM_ROUNDS, 1) targets = pay_load.get(TARGET_SITES, self.get_site_names()) + task_name = pay_load.get(TASK_NAME) data = pay_load.get(DATA, {}) data_shareable = self.get_shareable(data) @@ -203,13 +189,13 @@ def get_payload_task(self, pay_load) -> Tuple[Task, int, List[str]]: data_shareable.add_cookie(AppConstants.CONTRIBUTION_ROUND, current_round) operator = { - TaskOperatorKey.OP_ID: self.task_name, + TaskOperatorKey.OP_ID: task_name, TaskOperatorKey.METHOD: OperatorMethod.BROADCAST, TaskOperatorKey.TIMEOUT: self.task_timeout, } task = Task( - name=self.task_name, + name=task_name, data=data_shareable, operator=operator, props={}, @@ -232,7 +218,9 @@ def get_shareable(self, data): def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): - self.log_info(fl_ctx, f"{client_task.client.name} task:'{client_task.task.name}' result callback received.\n") + self.log_info( + fl_ctx, f"\n{client_task.client.name} task:'{client_task.task.name}' result callback received.\n\n" + ) client_name = client_task.client.name task_name = client_task.task.name @@ -244,8 +232,8 @@ def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): self.log_info(fl_ctx, f"Received result entries from client:{client_name} for task {task_name}") fl_model = FLModelUtils.from_shareable(result) results[RESULT] = {client_name: fl_model} - payload = {CMD: RESULT, PAYLOAD: {task_name: results}} - self.wf_queue.put_result(payload) + payload = {task_name: results} + self.event_manager.fire_event("TASK_RESULT", payload) else: self.handle_client_errors(rc, client_task, fl_ctx) @@ -258,7 +246,6 @@ def get_site_names(self): def handle_client_errors(self, rc: str, client_task: ClientTask, fl_ctx: FLContext): abort = ABORT_WHEN_IN_ERROR[rc] if abort: - self.wf_queue.ask_abort(f"error code {rc} occurred") self.log_error(fl_ctx, f"error code = {rc}") self.system_panic( f"Failed in client-site for {client_task.client.name} during task {client_task.task.name}.", diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index d666397252..555e1a623c 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -14,167 +14,169 @@ import logging -import time -from queue import Empty -from typing import Dict, Optional, Tuple +import threading +from typing import Callable, Dict, List, Optional, Union from nvflare.apis.controller_spec import SendOrder from nvflare.apis.fl_constant import ReturnCode from nvflare.app_common.abstract.fl_model import FLModel from nvflare.app_common.wf_comm.wf_comm_api_spec import ( - CMD, - CMD_ABORT, - CMD_STOP, - PAYLOAD, + CURRENT_ROUND, + DATA, + MIN_RESPONSES, + NUM_ROUNDS, + RESP_MAX_WAIT_TIME, RESULT, - SEND_ORDER, SITE_NAMES, + START_ROUND, STATUS, + TARGET_SITES, + TASK_NAME, WFCommAPISpec, ) -from nvflare.app_common.wf_comm.wf_queue import WFQueue from nvflare.fuel.message.data_bus import DataBus +from nvflare.fuel.message.event_manager import EventManager class WFCommAPI(WFCommAPISpec): def __init__(self): - self.result_pull_interval = 2 self.meta = {SITE_NAMES: []} self.logger = logging.getLogger(self.__class__.__name__) - message_bus = DataBus() - self.ctrl = message_bus.receive_messages("communicator") - self.wf_queue: Optional[WFQueue] = message_bus.receive_messages("wf_queue") - self._check_inputs() + self.task_results = {} + self.task_result_lock = threading.Lock() + + data_bus = DataBus() + data_bus.subscribe(topics=["TASK_RESULT"], callback=self.result_callback) - def set_result_pull_interval(self, pull_interval: float): - self.result_pull_interval = pull_interval + self.event_manager = EventManager(data_bus) + self.ctrl = data_bus.receive_messages("communicator") + self._check_inputs() def get_site_names(self): return self.meta.get(SITE_NAMES) - def broadcast_and_wait(self, msg_payload: Dict): + def broadcast_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, + ) -> Union[int, Dict[str, Dict[str, FLModel]]]: + + meta = {} if meta is None else meta + msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) + self.register_callback(callback) + self.ctrl.broadcast_to_peers_and_wait(msg_payload) - return self._get_results() - def broadcast(self, msg_payload): + if callback is None: + return self._get_results(task_name) + + def register_callback(self, callback): + if callback: + self.event_manager.data_bus.subscribe(["POST_PROCESS_RESULT"], callback) + + def send_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + send_order: SendOrder = SendOrder.SEQUENTIAL, + targets: Optional[List[str]] = None, + callback: Callable = None, + ): + meta = {} if meta is None else meta + msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) + + if callback is not None: + self.register_callback(callback) + + self.ctrl.send_to_peers_and_wait(msg_payload, send_order) + + if callback is not None: + return self._get_results(task_name) + + def relay_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + relay_order: str = "sequential", + targets: Optional[List[str]] = None, + callback: Callable = None, + ) -> Dict[str, Dict[str, FLModel]]: + + meta = {} if meta is None else meta + msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) + + self.register_callback(callback) + + self.ctrl.relay_to_peers_and_wait(msg_payload, SendOrder(relay_order)) + + if callback is None: + return self._get_results(task_name) + + return self._get_results(task_name) + + def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): + msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.broadcast_to_peers(pay_load=msg_payload) - def send(self, msg_payload: Dict): - send_order_name = msg_payload.get(SEND_ORDER) - send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) + def send( + self, + task_name: str, + data: any, + meta: dict = None, + send_order: str = "sequential", + targets: Optional[List[str]] = None, + ): + msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.send_to_peers(pay_load=msg_payload, send_order=send_order) - def send_and_wait(self, msg_payload: Dict): - send_order_name = msg_payload.get(SEND_ORDER) - send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) - self.ctrl.send_to_peers_and_wait(msg_payload, send_order=send_order) - return self._get_results() - - def relay_and_wait(self, msg_payload: Dict): - send_order_name = msg_payload.get(SEND_ORDER) - send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) - self.ctrl.relay_to_peers_and_wait(msg_payload, send_order) - return self._get_results() - - def relay(self, msg_payload: Dict): - send_order_name = msg_payload.get(SEND_ORDER) - send_order = SendOrder.SEQUENTIAL if not send_order_name else SendOrder(send_order_name) + def relay( + self, + task_name: str, + data: any, + meta: dict = None, + send_order: str = "sequential", + targets: Optional[List[str]] = None, + ): + msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.relay_to_peers(msg_payload, send_order) - def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float] = None) -> Dict[str, Dict[str, FLModel]]: - acc_size = 0 - start = None - while True: - if self.wf_queue.has_result(): - start = time.time() if start is None else start - items_size = self.wf_queue.result_size() - acc_size = items_size + acc_size - time_waited = time.time() - start - self.logger.info( - f"\n\n {items_size=}, {acc_size=}, {min_responses=}, {time_waited=}, {resp_max_wait_time=}" - ) - if resp_max_wait_time is not None: - if time_waited < resp_max_wait_time and acc_size >= min_responses: - return self._get_results() - else: - if time_waited < resp_max_wait_time: - self.logger.info(f" wait for more results, sleep {self.result_pull_interval} sec") - time.sleep(self.result_pull_interval) - else: - msg = f"not enough responses {acc_size} compare with min responses requirement {min_responses} within the max allowed time {resp_max_wait_time} seconds" - self.logger.info(msg) - raise RuntimeError(msg) - else: - if acc_size >= min_responses: - return self._get_results() - else: - self.logger.info(f" wait for more results, sleep {self.result_pull_interval} sec") - time.sleep(self.result_pull_interval) - else: - time.sleep(self.result_pull_interval) - - def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: - try: - item = self.wf_queue.get_result(resp_max_wait_time) - if item: - return self._process_one_result(item) - except Empty as e: - raise RuntimeError(f"failed to get result within the given timeout {resp_max_wait_time} sec.") - - def _process_one_result(self, item) -> Tuple[str, str, FLModel]: - cmd = item.get(CMD, None) - - if cmd is None: - msg = f"get None command, expecting {CMD} key'" - self.logger.error(msg) - raise RuntimeError(msg) - - elif cmd == CMD_STOP or cmd == CMD_ABORT: - msg = item.get(PAYLOAD) - self.logger.info(f"receive {cmd} command, {msg}") + def _process_one_result(self, site_result) -> Dict[str, FLModel]: + self._check_result(site_result) + rc = site_result.get(STATUS) + if rc == ReturnCode.OK: + result = site_result.get(RESULT, {}) + site_name, data = next(iter(result.items())) + task_result = {site_name: data} + else: + msg = f"task failed with '{rc}' status" raise RuntimeError(msg) - elif cmd == RESULT: - payload = item.get(PAYLOAD) - task_result = None - task, site_result = next(iter(payload.items())) - self._check_result(site_result) - rc = site_result.get(STATUS) - if rc == ReturnCode.OK: - result = site_result.get(RESULT, {}) - site_name, data = next(iter(result.items())) - task_result = (task, site_name, data) - else: - msg = f"task {task} failed with '{rc}' status" - self.wf_queue.ask_abort(msg) - raise RuntimeError(msg) - - return task_result - else: - raise RuntimeError(f"Unknown command {cmd}") + return task_result - def _get_results(self) -> Dict[str, Dict[str, FLModel]]: - items_size = self.wf_queue.result_size() + def _get_results(self, task_name) -> Dict[str, Dict[str, FLModel]]: batch_result: Dict = {} - for i in range(items_size): - item = self.wf_queue.get_result() - task, site_name, data = self._process_one_result(item) - task_result = batch_result.get(task, {}) - task_result.update({site_name: data}) - batch_result[task] = task_result - return batch_result + site_results = self.task_results.get(task_name) + + for i in range(len(site_results)): + item = site_results[i] + one_result = self._process_one_result(item) + task_result = batch_result.get(task_name, {}) + task_result.update(one_result) + batch_result[task_name] = task_result - def wait_for_responses(self, items_size, min_responses, resp_max_wait_time): - start = time.time() - while items_size < min_responses: - time_waited = time.time() - start - if time_waited < resp_max_wait_time: - time.sleep(1) - items_size = self.wf_queue.result_size() - else: - break - return items_size + with self.task_result_lock: + self.task_results[task_name] = [] + + return batch_result def _check_result(self, site_result): @@ -182,7 +184,7 @@ def _check_result(self, site_result): raise RuntimeError("expecting site_result to be dictionary, but get None") if not isinstance(site_result, dict): - raise RuntimeError(f"expecting site_result to be dictionary, but get '{type(site_result)}'") + raise RuntimeError(f"expecting site_result to be dictionary, but get '{type(site_result)}', {site_result=}") keys = [RESULT, STATUS] all_keys_present = all(key in site_result for key in keys) @@ -190,7 +192,41 @@ def _check_result(self, site_result): raise RuntimeError(f"expecting all keys {keys} present in site_result") def _check_inputs(self): - if self.wf_queue is None: - raise RuntimeError("missing WFQueue") if self.ctrl is None: raise RuntimeError("missing Controller") + + def result_callback(self, data, topic): + if topic == "TASK_RESULT": + task, site_result = next(iter(data.items())) + # fire event with process data + one_result = self._process_one_result(site_result) + self.event_manager.fire_event("POST_PROCESS_RESULT", {task: one_result}) + + site_task_results = self.task_results.get(task, []) + site_task_results.append(site_result) + self.task_results[task] = site_task_results + + def _prepare_input_payload(self, task_name, data, meta, min_responses, targets): + + if data and isinstance(data, FLModel): + start_round = data.start_round + current_round = data.current_round + num_rounds = data.total_rounds + else: + start_round = meta.get(START_ROUND, 0) + current_round = meta.get(CURRENT_ROUND, 0) + num_rounds = meta.get(NUM_ROUNDS, 1) + + resp_max_wait_time = meta.get(RESP_MAX_WAIT_TIME, 5) + + msg_payload = { + TASK_NAME: task_name, + MIN_RESPONSES: min_responses, + RESP_MAX_WAIT_TIME: resp_max_wait_time, + CURRENT_ROUND: current_round, + NUM_ROUNDS: num_rounds, + START_ROUND: start_round, + DATA: data, + TARGET_SITES: targets, + } + return msg_payload diff --git a/nvflare/app_common/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/wf_comm/wf_comm_api_spec.py index 83da8314b7..461dcbc938 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/wf_comm/wf_comm_api_spec.py @@ -13,16 +13,10 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import Dict, List, Optional, Tuple +from typing import Callable, List, Optional -from nvflare.app_common.abstract.fl_model import FLModel - -CMD = "COMMAND" -CMD_STOP = "STOP" -CMD_ABORT = "ABORT" -PAYLOAD = "PAYLOAD" -SEND_ORDER = "SEND_ORDER" SITE_NAMES = "SITE_NAMES" +TASK_NAME = "TASK_NAME" # note same as app_constant constant (todo: we only need one constant definition) MIN_RESPONSES = "min_responses" @@ -41,59 +35,69 @@ class WFCommAPISpec(ABC): @abstractmethod - def broadcast_and_wait(self, msg_payload: Dict): + def broadcast_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, + ): pass @abstractmethod - def send_and_wait(self, msg_payload: Dict): + def send_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + send_order: str = "sequential", + callback: Callable = None, + ): pass @abstractmethod - def relay_and_wait(self, msg_payload: Dict): + def relay_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + relay_order: str = "sequential", + targets: Optional[List[str]] = None, + callback: Callable = None, + ): pass @abstractmethod - def broadcast(self, msg_payload: Dict): + def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): pass @abstractmethod - def send(self, msg_payload: Dict): + def send( + self, + task_name: str, + data: any, + meta: dict = None, + send_order: str = "sequential", + targets: Optional[str] = None, + ): pass @abstractmethod - def relay(self, msg_payload: Dict): + def relay( + self, + task_name: str, + data: any, + meta: dict = None, + relay_order: str = "sequential", + targets: Optional[List[str]] = None, + ): pass @abstractmethod def get_site_names(self) -> List[str]: pass - - @abstractmethod - def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: - """ - wait for result - Args: - min_responses: if min_responses or more sites are received, then the result will return - resp_max_wait_time: the max wait time after the 1st site response is received. This is used to deal - with really late site result arrival, instead of waiting forever, we set a timeout. - if resp_max_wait_time is None, it will not timeout - - Returns: - all results with min_response - """ - pass - - @abstractmethod - def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: - """ - wait for result - Args: - resp_max_wait_time: the max wait time after the 1st site response is received. This is used to deal - with really late site result arrival, instead of waiting forever, we set a timeout. - if resp_max_wait_time is None, it will not timeout - - Returns: - Tuple of task_name, site_name, FLModel - """ - - pass diff --git a/nvflare/app_common/wf_comm/wf_communicator.py b/nvflare/app_common/wf_comm/wf_communicator.py index cdd1f3abe5..61bf60a677 100644 --- a/nvflare/app_common/wf_comm/wf_communicator.py +++ b/nvflare/app_common/wf_comm/wf_communicator.py @@ -19,12 +19,8 @@ class WFCommunicator(BaseWFCommunicator, Controller): - def __init__( - self, - task_timeout: int = 0, - result_pull_interval: float = 0.2, - ): - super().__init__(task_name="train") + def __init__(self): + super().__init__() def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) diff --git a/nvflare/app_common/wf_comm/wf_communicator_spec.py b/nvflare/app_common/wf_comm/wf_communicator_spec.py index 01a6b0607b..15dbc0eea2 100644 --- a/nvflare/app_common/wf_comm/wf_communicator_spec.py +++ b/nvflare/app_common/wf_comm/wf_communicator_spec.py @@ -1,18 +1,56 @@ -from abc import ABC -from typing import List, Dict, Optional +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod +from typing import Dict, List, Optional + +from nvflare.apis.controller_spec import SendOrder from nvflare.fuel.utils.class_utils import instantiate_class from nvflare.fuel.utils.component_builder import ComponentBuilder from nvflare.fuel.utils.fobs import fobs +from nvflare.fuel.utils.import_utils import optional_import class WFCommunicatorSpec(ABC): - def __init__(self): - self.strategy = None self.strategy_config: Optional[Dict] = None - def set_strategy_config(self, strategy_config): + @abstractmethod + def broadcast_to_peers_and_wait(self, pay_load: Dict): + pass + + @abstractmethod + def broadcast_to_peers(self, pay_load: Dict): + pass + + @abstractmethod + def send_to_peers(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): + pass + + @abstractmethod + def send_to_peers_and_wait(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): + pass + + @abstractmethod + def relay_to_peers_and_wait(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): + pass + + @abstractmethod + def relay_to_peers(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): + pass + + def set_strategy_config(self, strategy_config: Dict): if strategy_config is None: raise ValueError("strategy_config is None") @@ -22,17 +60,23 @@ def set_strategy_config(self, strategy_config): self.strategy_config = strategy_config def get_strategy(self): - # if self.strategy is None and isinstance(self.strategy_config, dict): - print(f"{self.strategy_config=}") + strategy = None if isinstance(self.strategy_config, dict): strategy = ComponentBuilder().build_component(self.strategy_config) if strategy is None: raise ValueError("strategy should provided, but get None") - self.strategy = strategy - return self.strategy + return strategy - def set_serializers(self, serializer_class_paths: List[str] = None): + def register_serializers(self, serializer_class_paths: List[str] = None): + self.register_default_serializers() if serializer_class_paths: for class_path in serializer_class_paths: fobs.register(instantiate_class(class_path, {})) + + def register_default_serializers(self): + torch, flag = optional_import("torch") + if flag: + from nvflare.app_opt.pt.decomposers import TensorDecomposer + + fobs.register(TensorDecomposer) diff --git a/nvflare/app_common/wf_comm/wf_queue.py b/nvflare/app_common/wf_comm/wf_queue.py deleted file mode 100644 index 6638d5373c..0000000000 --- a/nvflare/app_common/wf_comm/wf_queue.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from queue import Queue -from typing import Dict, Optional - -from nvflare.app_common.wf_comm.wf_comm_api_spec import CMD, CMD_ABORT, CMD_STOP, PAYLOAD - - -class WFQueue: - def __init__(self, result_queue: Queue): - self.result_queue = result_queue - - def put_result(self, msg): - self.result_queue.put(msg) - - def has_result(self) -> bool: - return not self.result_queue.empty() - - def result_size(self) -> int: - return self.result_queue.qsize() - - def get_result(self, timeout: Optional[float] = None) -> Dict: - item = self.result_queue.get(timeout=timeout) - self.result_queue.task_done() - return item - - def stop(self, msg: Optional[str] = None): - msg = msg if msg else {} - self.put_result({CMD: CMD_STOP, PAYLOAD: msg}) - - def ask_abort(self, msg: Optional[str] = None): - msg = msg if msg else {} - self.put_result({CMD: CMD_ABORT, PAYLOAD: msg}) diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index ab32d64884..9072ea3f27 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -17,8 +17,8 @@ from nvflare.apis.fl_component import FLComponent from nvflare.apis.fl_constant import SystemConfigs, SystemVarName from nvflare.apis.responder import Responder -from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec from nvflare.app_common.wf_comm.wf_communicator import WFCommunicator +from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec from nvflare.fuel.message.data_bus import DataBus from nvflare.fuel.utils.argument_utils import parse_vars from nvflare.fuel.utils.component_builder import ComponentBuilder @@ -26,6 +26,7 @@ from nvflare.fuel.utils.json_scanner import Node from nvflare.private.fed_json_config import FedJsonConfigurator from nvflare.private.json_configer import ConfigContext, ConfigError + from .server_runner import ServerRunnerConfig FL_PACKAGES = ["nvflare"] @@ -141,12 +142,15 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): component = self.authorize_and_build_component(element, config_ctx, node) if isinstance(component, dict): wf_config = component - communicator = wf_config.get("communicator", WFCommunicator()) + communicator = wf_config.get("communicator") + if communicator is None: + communicator = WFCommunicator() + if isinstance(communicator, WFCommunicatorSpec): strategy_config = wf_config.get("strategy") strategy_config["lazy_instantiate"] = False communicator.set_strategy_config(strategy_config) - communicator.set_serializers(strategy_config.get("serializers")) + communicator.register_serializers(strategy_config.get("serializers")) data_bus = DataBus() data_bus.send_message("communicator", communicator) responder = communicator @@ -156,7 +160,8 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): if not isinstance(responder, Responder): raise ConfigError( '"workflow" must be a Responder or Controller or has a Responder object, but got {}'.format( - type(responder)) + type(responder) + ) ) cid = element.get("id", None) diff --git a/runtest.sh b/runtest.sh index e7644cf736..b7df3b6be1 100755 --- a/runtest.sh +++ b/runtest.sh @@ -92,7 +92,7 @@ function check_license() { folders_to_check_license="nvflare examples tests integration research" echo "checking license header in folder: $folders_to_check_license" (grep -r --include "*.py" --exclude-dir "*protos*" -L \ - "\(# Copyright (c) \(2021\|2022\|2023\), NVIDIA CORPORATION. All rights reserved.\)\|\(This file is released into the public domain.\)" \ + "\(# Copyright (c) \(2021\|2022\|2023\|2024\), NVIDIA CORPORATION. All rights reserved.\)\|\(This file is released into the public domain.\)" \ ${folders_to_check_license} || true) > no_license.lst if [ -s no_license.lst ]; then # The file is not-empty. From 30552e49e05073d014d8406a500ca25933f7bec2 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Fri, 19 Jan 2024 21:36:26 -0800 Subject: [PATCH 28/41] everything works now --- nvflare/apis/utils/fl_context_utils.py | 2 +- nvflare/app_common/ccwf/server_ctl.py | 2 +- nvflare/app_common/hub/hub_controller.py | 2 +- nvflare/private/fed/server/server_runner.py | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nvflare/apis/utils/fl_context_utils.py b/nvflare/apis/utils/fl_context_utils.py index 6882432de9..819e69186b 100644 --- a/nvflare/apis/utils/fl_context_utils.py +++ b/nvflare/apis/utils/fl_context_utils.py @@ -60,7 +60,7 @@ def generate_log_message(fl_ctx: FLContext, msg: str): _task_name = "task_name" _task_id = "task_id" _rc = "peer_rc" - _wf = "strategy" + _wf = "wf" all_kvs = {_identity_: fl_ctx.get_identity_name()} my_run = fl_ctx.get_job_id() diff --git a/nvflare/app_common/ccwf/server_ctl.py b/nvflare/app_common/ccwf/server_ctl.py index 1167df8a8a..5ec2e868a1 100644 --- a/nvflare/app_common/ccwf/server_ctl.py +++ b/nvflare/app_common/ccwf/server_ctl.py @@ -57,7 +57,7 @@ def __init__( self, num_rounds: int, start_round: int = 0, - task_name_prefix: str = "strategy", + task_name_prefix: str = "wf", configure_task_timeout=Constant.CONFIG_TASK_TIMEOUT, end_workflow_timeout=Constant.END_WORKFLOW_TIMEOUT, start_task_timeout=Constant.START_TASK_TIMEOUT, diff --git a/nvflare/app_common/hub/hub_controller.py b/nvflare/app_common/hub/hub_controller.py index e9d0392072..d7699ed2b0 100644 --- a/nvflare/app_common/hub/hub_controller.py +++ b/nvflare/app_common/hub/hub_controller.py @@ -134,7 +134,7 @@ def process_result_of_unknown_task( class RelayOperator(OperatorSpec, FLComponent): - _PROP_LAST_RESULT = "last_model" + _PROP_LAST_RESULT = "last_result" _PROP_SHAREABLE_GEN = "shareable_generator" def __init__(self): diff --git a/nvflare/private/fed/server/server_runner.py b/nvflare/private/fed/server/server_runner.py index c9dd76a3a1..75bb9b3d7e 100644 --- a/nvflare/private/fed/server/server_runner.py +++ b/nvflare/private/fed/server/server_runner.py @@ -125,7 +125,6 @@ def _execute_run(self): self.log_info(fl_ctx, "starting workflow {} ({}) ...".format(wf.id, type(wf.responder))) fl_ctx.set_prop(FLContextKey.WORKFLOW, wf.id, sticky=True) - wf.responder.initialize_run(fl_ctx) self.log_info(fl_ctx, "Workflow {} ({}) started".format(wf.id, type(wf.responder))) From c2970731a19113e2765fcf511767e96a1a487c35 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 27 Jan 2024 18:37:22 -0800 Subject: [PATCH 29/41] merge with new data bus changes. The code is broken now. --- nvflare/app_common/wf_comm/__init__.py | 4 +- nvflare/app_common/wf_comm/base_wf_comm.py | 10 +- nvflare/app_common/wf_comm/wf_comm_api.py | 17 ++- .../fuel/{message => data_event}/__init__.py | 0 nvflare/fuel/data_event/data_bus.py | 103 ++++++++++++++++++ nvflare/fuel/data_event/event_manager.py | 45 ++++++++ nvflare/fuel/data_event/pub_sub.py | 34 ++++++ nvflare/fuel/message/data_bus.py | 52 --------- nvflare/fuel/message/event_manager.py | 22 ---- .../private/fed/server/server_json_config.py | 4 +- tests/unit_test/fuel/message/__init__.py | 13 --- .../fuel/message/message_bus_test.py | 84 -------------- 12 files changed, 205 insertions(+), 183 deletions(-) rename nvflare/fuel/{message => data_event}/__init__.py (100%) create mode 100644 nvflare/fuel/data_event/data_bus.py create mode 100644 nvflare/fuel/data_event/event_manager.py create mode 100644 nvflare/fuel/data_event/pub_sub.py delete mode 100644 nvflare/fuel/message/data_bus.py delete mode 100644 nvflare/fuel/message/event_manager.py delete mode 100644 tests/unit_test/fuel/message/__init__.py delete mode 100644 tests/unit_test/fuel/message/message_bus_test.py diff --git a/nvflare/app_common/wf_comm/__init__.py b/nvflare/app_common/wf_comm/__init__.py index 4e0bb0ef5f..3291c2225f 100644 --- a/nvflare/app_common/wf_comm/__init__.py +++ b/nvflare/app_common/wf_comm/__init__.py @@ -13,10 +13,10 @@ # limitations under the License. from nvflare.app_common.wf_comm.wf_comm_api_spec import WFCommAPISpec -from nvflare.fuel.message.data_bus import DataBus +from nvflare.fuel.data_event.data_bus import DataBus data_bus = DataBus() def get_wf_comm_api() -> WFCommAPISpec: - return data_bus.receive_messages("wf_comm_api") + return data_bus.receive_data("wf_comm_api") diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_comm.py index dc4c8a9914..7ec48b8f11 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -37,8 +37,8 @@ ) from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec from nvflare.app_common.workflows.error_handle_utils import ABORT_WHEN_IN_ERROR -from nvflare.fuel.message.data_bus import DataBus -from nvflare.fuel.message.event_manager import EventManager +from nvflare.fuel.data_event.data_bus import DataBus +from nvflare.fuel.data_event.event_manager import EventManager from nvflare.security.logging import secure_format_traceback @@ -74,7 +74,7 @@ def start_controller(self, fl_ctx: FLContext): def publish_comm_api(self): comm_api = WFCommAPI() comm_api.meta.update({SITE_NAMES: self.get_site_names()}) - self.data_bus.send_message("wf_comm_api", comm_api) + self.data_bus.send_data("wf_comm_api", comm_api) def start_workflow(self, abort_signal, fl_ctx): try: @@ -102,6 +102,8 @@ def broadcast_to_peers_and_wait(self, pay_load): self.fl_ctx.set_prop("task_name", task.name) + print(f"call broadcast_and_wait to {targets} \n") + self.broadcast_and_wait( task=task, targets=targets, @@ -110,6 +112,7 @@ def broadcast_to_peers_and_wait(self, pay_load): fl_ctx=self.fl_ctx, abort_signal=abort_signal, ) + print("\nafter broadcast_and_wait\n") self.fire_event(AppEventType.ROUND_DONE, self.fl_ctx) self.log_info(self.fl_ctx, f"Round {current_round} finished.") @@ -228,6 +231,7 @@ def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): rc = result.get_return_code() results: Dict[str, any] = {STATUS: rc} + print(f"{rc=}") if rc == ReturnCode.OK: self.log_info(fl_ctx, f"Received result entries from client:{client_name} for task {task_name}") fl_model = FLModelUtils.from_shareable(result) diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index 555e1a623c..68d7eded00 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -34,8 +34,8 @@ TASK_NAME, WFCommAPISpec, ) -from nvflare.fuel.message.data_bus import DataBus -from nvflare.fuel.message.event_manager import EventManager +from nvflare.fuel.data_event.data_bus import DataBus +from nvflare.fuel.data_event.event_manager import EventManager class WFCommAPI(WFCommAPISpec): @@ -50,7 +50,7 @@ def __init__(self): data_bus.subscribe(topics=["TASK_RESULT"], callback=self.result_callback) self.event_manager = EventManager(data_bus) - self.ctrl = data_bus.receive_messages("communicator") + self.ctrl = data_bus.receive_data("communicator") self._check_inputs() def get_site_names(self): @@ -69,8 +69,9 @@ def broadcast_and_wait( meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) self.register_callback(callback) - + print("calling broadcast_to_peers_and_wait") self.ctrl.broadcast_to_peers_and_wait(msg_payload) + print("after broadcast_to_peers_and_wait") if callback is None: return self._get_results(task_name) @@ -163,6 +164,7 @@ def _process_one_result(self, site_result) -> Dict[str, FLModel]: return task_result def _get_results(self, task_name) -> Dict[str, Dict[str, FLModel]]: + print("_get_results\n") batch_result: Dict = {} site_results = self.task_results.get(task_name) @@ -176,6 +178,7 @@ def _get_results(self, task_name) -> Dict[str, Dict[str, FLModel]]: with self.task_result_lock: self.task_results[task_name] = [] + print("return batch_result=", batch_result) return batch_result def _check_result(self, site_result): @@ -195,15 +198,19 @@ def _check_inputs(self): if self.ctrl is None: raise RuntimeError("missing Controller") - def result_callback(self, data, topic): + def result_callback(self, topic, data, data_bus): + print("get callback, topic = ", topic) if topic == "TASK_RESULT": task, site_result = next(iter(data.items())) + print(task, site_result) # fire event with process data one_result = self._process_one_result(site_result) + print("fire_event to POST_PROCESS_RESULT") self.event_manager.fire_event("POST_PROCESS_RESULT", {task: one_result}) site_task_results = self.task_results.get(task, []) site_task_results.append(site_result) + print("acc site_task_results \n") self.task_results[task] = site_task_results def _prepare_input_payload(self, task_name, data, meta, min_responses, targets): diff --git a/nvflare/fuel/message/__init__.py b/nvflare/fuel/data_event/__init__.py similarity index 100% rename from nvflare/fuel/message/__init__.py rename to nvflare/fuel/data_event/__init__.py diff --git a/nvflare/fuel/data_event/data_bus.py b/nvflare/fuel/data_event/data_bus.py new file mode 100644 index 0000000000..32c96b4766 --- /dev/null +++ b/nvflare/fuel/data_event/data_bus.py @@ -0,0 +1,103 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import threading +from typing import Any, Callable, List + +from nvflare.fuel.data_event.pub_sub import EventPubSub + + +class DataBus(EventPubSub): + """ + Singleton class for a simple data bus implementation. + + This class allows components to subscribe to topics, publish messages to topics, + and store/retrieve messages associated with specific keys and topics. + """ + + _instance = None + _lock = threading.Lock() + + def __new__(cls) -> "DataBus": + """ + Create a new instance of the DataBus class. + This method ensures that only one instance of the class is created (singleton pattern). + The databus + + + """ + with cls._lock: + if not cls._instance: + cls._instance = super(DataBus, cls).__new__(cls) + cls._instance.subscribers = {} + cls._instance.data_store = {} + return cls._instance + + def subscribe(self, topics: List[str], callback: Callable[[str, Any, "DataBus"], None]) -> None: + """ + Subscribe a callback function to one or more topics. + + Args: + topics (List[str]): A list of topics to subscribe to. + callback (Callable): The callback function to be called when messages are published to the subscribed topics. + """ + + if not topics: + raise ValueError("topics must non-empty") + + for topic in topics: + if topic.isspace(): + raise ValueError(f"topics {topics}contains white space topic") + + with self._lock: + if topic not in self.subscribers: + self.subscribers[topic] = [] + self.subscribers[topic].append(callback) + + def publish(self, topics: List[str], datum: Any) -> None: + """ + Publish a data to one or more topics, notifying all subscribed callbacks. + + Args: + topics (List[str]): A list of topics to publish the data to. + datum (Any): The data to be published to the specified topics. + """ + if topics: + for topic in topics: + with self._lock: + if topic in self.subscribers: + for callback in self.subscribers[topic]: + callback(topic, datum, self) + + def send_data(self, key: Any, datum: Any) -> None: + """ + Store a data associated with a key and topic. + + Args: + key (Any): The key to associate with the stored message. + datum (Any): The message to be stored. + """ + with self._lock: + self.data_store[key] = datum + + def receive_data(self, key: Any) -> Any: + """ + Retrieve a stored data associated with a key and topic. + + Args: + key (Any): The key associated with the stored message. + + Returns: + Any: The stored datum if found, or None if not found. + """ + return self.data_store.get(key) diff --git a/nvflare/fuel/data_event/event_manager.py b/nvflare/fuel/data_event/event_manager.py new file mode 100644 index 0000000000..6421f8bf4e --- /dev/null +++ b/nvflare/fuel/data_event/event_manager.py @@ -0,0 +1,45 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Optional + +from nvflare.fuel.data_event.data_bus import DataBus + + +class EventManager: + """ + Class for managing events by interacting with a DataBus. + + Args: + data_bus (DataBus): An instance of the DataBus class used for event communication. + """ + + def __init__(self, data_bus: "DataBus"): + """ + Initialize the EventManager with a DataBus instance. + + Args: + data_bus (DataBus): An instance of the DataBus class used for event communication. + """ + self.data_bus = data_bus + + def fire_event(self, event_name: str, event_data: Optional[Any] = None) -> None: + """ + Fire an event by publishing it to the DataBus. + + Args: + event_name (str): The name of the event to be fired. + event_data (Any, optional): Additional data associated with the event (default is None). + """ + self.data_bus.publish([event_name], event_data) diff --git a/nvflare/fuel/data_event/pub_sub.py b/nvflare/fuel/data_event/pub_sub.py new file mode 100644 index 0000000000..63583c8b13 --- /dev/null +++ b/nvflare/fuel/data_event/pub_sub.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Callable, List + + +class EventPubSub: + def subscribe(self, topics: List[str], callback: Callable[[str, Any, "DataBus"], None]) -> None: + """ + Subscribe a callback function to one or more topics. + + Args: + topics (List[str]): A list of topics to subscribe to. + callback (Callable): The callback function to be called when messages are published to the subscribed topics. + """ + + def publish(self, topics: List[str], datum: Any) -> None: + """ + Publish a message to one or more topics, notifying all subscribed callbacks. + + Args: + topics (List[str]): A list of topics to publish the message to. + datum (Any): The message to be published to the specified topics. + """ diff --git a/nvflare/fuel/message/data_bus.py b/nvflare/fuel/message/data_bus.py deleted file mode 100644 index b1bb265622..0000000000 --- a/nvflare/fuel/message/data_bus.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import threading -from typing import Callable, List - - -class DataBus: - _instance = None - _lock = threading.Lock() - - def __new__(cls): - with cls._lock: - if not cls._instance: - cls._instance = super(DataBus, cls).__new__(cls) - cls._instance.subscribers = {} - cls._instance.message_store = {} - return cls._instance - - def subscribe(self, topics: List[str], callback: Callable): - if topics: - for topic in topics: - if topic not in self.subscribers: - self.subscribers[topic] = [] - self.subscribers[topic].append(callback) - - def publish(self, topics: List[str], message: any): - if topics: - for topic in topics: - if topic in self.subscribers: - for callback in self.subscribers[topic]: - callback(message, topic) - - def send_message(self, key, message, topic: str = "default"): - if topic not in self.message_store: - self.message_store[topic] = {} - - self.message_store[topic][key] = message - - def receive_messages(self, key, topic: str = "default"): - return self.message_store.get(topic, {}).get(key) diff --git a/nvflare/fuel/message/event_manager.py b/nvflare/fuel/message/event_manager.py deleted file mode 100644 index 7b3dbd2623..0000000000 --- a/nvflare/fuel/message/event_manager.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from nvflare.fuel.message.data_bus import DataBus - - -class EventManager: - def __init__(self, data_bus: DataBus): - self.data_bus = data_bus - - def fire_event(self, event_name, event_data=None): - self.data_bus.publish([event_name], event_data) diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index 9072ea3f27..2c5009aa18 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -19,7 +19,7 @@ from nvflare.apis.responder import Responder from nvflare.app_common.wf_comm.wf_communicator import WFCommunicator from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec -from nvflare.fuel.message.data_bus import DataBus +from nvflare.fuel.data_event.data_bus import DataBus from nvflare.fuel.utils.argument_utils import parse_vars from nvflare.fuel.utils.component_builder import ComponentBuilder from nvflare.fuel.utils.config_service import ConfigService @@ -152,7 +152,7 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): communicator.set_strategy_config(strategy_config) communicator.register_serializers(strategy_config.get("serializers")) data_bus = DataBus() - data_bus.send_message("communicator", communicator) + data_bus.send_data("communicator", communicator) responder = communicator else: responder = component diff --git a/tests/unit_test/fuel/message/__init__.py b/tests/unit_test/fuel/message/__init__.py deleted file mode 100644 index 4fc50543f1..0000000000 --- a/tests/unit_test/fuel/message/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit_test/fuel/message/message_bus_test.py b/tests/unit_test/fuel/message/message_bus_test.py deleted file mode 100644 index 6135a74dfb..0000000000 --- a/tests/unit_test/fuel/message/message_bus_test.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from nvflare.fuel.message.event_manger import EventManager -from nvflare.fuel.message.message_bus import MessageBus - - -class TestMessageBus(unittest.TestCase): - def setUp(self): - self.message_bus = MessageBus() - self.event_manager = EventManager(self.message_bus) - - def test_subscribe_and_publish(self): - result = {"count": 0} - - def callback_function(message): - result["count"] += 1 - - self.message_bus.subscribe("test_topic", callback_function) - self.message_bus.publish("test_topic", "Test Message 1") - self.message_bus.publish("test_topic", "Test Message 2") - - self.assertEqual(result["count"], 2) - - def test_singleton_message_bus(self): - message_bus1 = MessageBus() - message_bus1.send_message("user_1", "Hello from User 1!") - user_1_message = message_bus1.receive_messages("user_1") - self.assertEqual(user_1_message, "Hello from User 1!") - - message_bus2 = MessageBus() - user_1_message = message_bus2.receive_messages("user_1") - self.assertEqual(user_1_message, "Hello from User 1!") - - def test_send_message_and_receive_messages(self): - self.message_bus.send_message("user_1", "Hello from User 1!") - self.message_bus.send_message("user_2", "Greetings from User 2!") - - user_1_message = self.message_bus.receive_messages("user_1") - user_2_message = self.message_bus.receive_messages("user_2") - - self.assertEqual(user_1_message, "Hello from User 1!") - self.assertEqual(user_2_message, "Greetings from User 2!") - - self.message_bus.send_message("user_1", "2nd greetings from User 1!") - user_1_message = self.message_bus.receive_messages("user_1") - self.assertEqual(user_1_message, "2nd greetings from User 1!") - - self.message_bus.send_message("user_1", "3rd greetings from User 1!", topic="channel-3") - user_1_message = self.message_bus.receive_messages("user_1") - self.assertEqual(user_1_message, "2nd greetings from User 1!") - - user_1_message = self.message_bus.receive_messages("user_1", topic="channel-3") - self.assertEqual(user_1_message, "3rd greetings from User 1!") - - def test_send_message_and_receive_messages_abnormal(self): - user_3_message = self.message_bus.receive_messages("user_3") - self.assertEqual(user_3_message, None) - - user_3_message = self.message_bus.receive_messages("user_3", topic="channel") - self.assertEqual(user_3_message, None) - - def test_fire_event(self): - result = {"event_received": False} - - def event_handler(data): - result["event_received"] = True - - self.message_bus.subscribe("test_event", event_handler) - self.event_manager.fire_event("test_event", {"key": "value"}) - - self.assertTrue(result["event_received"]) From b4da21f5ac990112878e4606af0a597d41df52b1 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 27 Jan 2024 20:06:27 -0800 Subject: [PATCH 30/41] fix the lock issue. --- .../app_common/executors/launcher_executor.py | 1 - nvflare/app_common/wf_comm/base_wf_comm.py | 5 +- nvflare/app_common/wf_comm/wf_comm_api.py | 83 +++++++++---------- nvflare/fuel/data_event/data_bus.py | 6 +- 4 files changed, 47 insertions(+), 48 deletions(-) diff --git a/nvflare/app_common/executors/launcher_executor.py b/nvflare/app_common/executors/launcher_executor.py index a9fd1fab34..e968728a93 100644 --- a/nvflare/app_common/executors/launcher_executor.py +++ b/nvflare/app_common/executors/launcher_executor.py @@ -257,7 +257,6 @@ def _execute_launcher_method_in_thread_executor(self, method_name: str, **kwargs future = self._thread_pool_executor.submit(getattr(self.launcher, method_name), **kwargs) result = future.result(timeout=self._launch_timeout) - return result except TimeoutError: self.log_warning( diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_comm.py index 7ec48b8f11..d151a3f275 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -102,7 +102,7 @@ def broadcast_to_peers_and_wait(self, pay_load): self.fl_ctx.set_prop("task_name", task.name) - print(f"call broadcast_and_wait to {targets} \n") + print(f"call broadcast_and_wait to {min_responses=}, {targets=} , {task.timeout=}\n") self.broadcast_and_wait( task=task, @@ -110,7 +110,7 @@ def broadcast_to_peers_and_wait(self, pay_load): min_responses=min_responses, wait_time_after_min_received=0, fl_ctx=self.fl_ctx, - abort_signal=abort_signal, + abort_signal=abort_signal ) print("\nafter broadcast_and_wait\n") self.fire_event(AppEventType.ROUND_DONE, self.fl_ctx) @@ -237,6 +237,7 @@ def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): fl_model = FLModelUtils.from_shareable(result) results[RESULT] = {client_name: fl_model} payload = {task_name: results} + print(f"fire_event to TASK_RESULT, {payload=}") self.event_manager.fire_event("TASK_RESULT", payload) else: self.handle_client_errors(rc, client_task, fl_ctx) diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index 68d7eded00..e8074c337f 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -57,21 +57,21 @@ def get_site_names(self): return self.meta.get(SITE_NAMES) def broadcast_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, ) -> Union[int, Dict[str, Dict[str, FLModel]]]: meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) self.register_callback(callback) - print("calling broadcast_to_peers_and_wait") + print("\ncalling broadcast_to_peers_and_wait\n") self.ctrl.broadcast_to_peers_and_wait(msg_payload) - print("after broadcast_to_peers_and_wait") + print("\nafter broadcast_to_peers_and_wait\n") if callback is None: return self._get_results(task_name) @@ -81,14 +81,14 @@ def register_callback(self, callback): self.event_manager.data_bus.subscribe(["POST_PROCESS_RESULT"], callback) def send_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - send_order: SendOrder = SendOrder.SEQUENTIAL, - targets: Optional[List[str]] = None, - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + send_order: SendOrder = SendOrder.SEQUENTIAL, + targets: Optional[List[str]] = None, + callback: Callable = None, ): meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) @@ -102,14 +102,14 @@ def send_and_wait( return self._get_results(task_name) def relay_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - relay_order: str = "sequential", - targets: Optional[List[str]] = None, - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + relay_order: str = "sequential", + targets: Optional[List[str]] = None, + callback: Callable = None, ) -> Dict[str, Dict[str, FLModel]]: meta = {} if meta is None else meta @@ -129,23 +129,23 @@ def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optio self.ctrl.broadcast_to_peers(pay_load=msg_payload) def send( - self, - task_name: str, - data: any, - meta: dict = None, - send_order: str = "sequential", - targets: Optional[List[str]] = None, + self, + task_name: str, + data: any, + meta: dict = None, + send_order: str = "sequential", + targets: Optional[List[str]] = None, ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.send_to_peers(pay_load=msg_payload, send_order=send_order) def relay( - self, - task_name: str, - data: any, - meta: dict = None, - send_order: str = "sequential", - targets: Optional[List[str]] = None, + self, + task_name: str, + data: any, + meta: dict = None, + send_order: str = "sequential", + targets: Optional[List[str]] = None, ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.relay_to_peers(msg_payload, send_order) @@ -167,6 +167,8 @@ def _get_results(self, task_name) -> Dict[str, Dict[str, FLModel]]: print("_get_results\n") batch_result: Dict = {} site_results = self.task_results.get(task_name) + if not site_results: + raise RuntimeError(f"not result for given task {task_name}") for i in range(len(site_results)): item = site_results[i] @@ -199,18 +201,13 @@ def _check_inputs(self): raise RuntimeError("missing Controller") def result_callback(self, topic, data, data_bus): - print("get callback, topic = ", topic) if topic == "TASK_RESULT": task, site_result = next(iter(data.items())) - print(task, site_result) # fire event with process data one_result = self._process_one_result(site_result) - print("fire_event to POST_PROCESS_RESULT") self.event_manager.fire_event("POST_PROCESS_RESULT", {task: one_result}) - site_task_results = self.task_results.get(task, []) site_task_results.append(site_result) - print("acc site_task_results \n") self.task_results[task] = site_task_results def _prepare_input_payload(self, task_name, data, meta, min_responses, targets): @@ -224,7 +221,7 @@ def _prepare_input_payload(self, task_name, data, meta, min_responses, targets): current_round = meta.get(CURRENT_ROUND, 0) num_rounds = meta.get(NUM_ROUNDS, 1) - resp_max_wait_time = meta.get(RESP_MAX_WAIT_TIME, 5) + resp_max_wait_time = meta.get(RESP_MAX_WAIT_TIME, 15) msg_payload = { TASK_NAME: task_name, diff --git a/nvflare/fuel/data_event/data_bus.py b/nvflare/fuel/data_event/data_bus.py index 32c96b4766..bf2587c827 100644 --- a/nvflare/fuel/data_event/data_bus.py +++ b/nvflare/fuel/data_event/data_bus.py @@ -74,8 +74,10 @@ def publish(self, topics: List[str], datum: Any) -> None: """ if topics: for topic in topics: - with self._lock: - if topic in self.subscribers: + print(f"{topic=}") + print(f"{self.subscribers=}", ) + if topic in self.subscribers: + with self._lock: for callback in self.subscribers[topic]: callback(topic, datum, self) From ae657f62adbfaa6c379714261ab709a75858d765 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 27 Jan 2024 20:39:10 -0800 Subject: [PATCH 31/41] define strategy.py in case it is needed. --- nvflare/app_common/abstract/strategy.py | 90 +++++++++++++++++++ nvflare/app_common/wf_comm/wf_comm_api.py | 6 +- .../app_common/wf_comm/wf_comm_api_spec.py | 6 +- nvflare/fuel/data_event/data_bus.py | 2 - 4 files changed, 96 insertions(+), 8 deletions(-) create mode 100644 nvflare/app_common/abstract/strategy.py diff --git a/nvflare/app_common/abstract/strategy.py b/nvflare/app_common/abstract/strategy.py new file mode 100644 index 0000000000..bdaf58cdd5 --- /dev/null +++ b/nvflare/app_common/abstract/strategy.py @@ -0,0 +1,90 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod +from typing import Optional, Callable, List + +from nvflare.app_common import wf_comm +from nvflare.app_common.wf_comm.wf_comm_api_spec import WFCommAPISpec + + +class Strategy(ABC, WFCommAPISpec): + + def __init__(self): + self.communicator = wf_comm.get_wf_comm_api() + + @abstractmethod + def run(self): + pass + + def broadcast_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, + ): + return self.communicator.broadcast_and_wait(task_name, min_responses, data, meta, targets, callback) + + def send_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + send_order: str = "sequential", + callback: Callable = None, + ): + return self.communicator.send_and_wait(task_name, min_responses, data, meta, targets, send_order, callback) + + def relay_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + relay_order: str = "sequential", + callback: Callable = None, + ): + return self.communicator.relay_and_wait(task_name, min_responses, data, meta, targets, relay_order, callback) + + def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): + return self.communicator.broadcast(task_name, data, meta, targets) + + def send( + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[str] = None, + send_order: str = "sequential", + ): + return self.communicator.send(task_name, data, meta, targets, send_order) + + def relay( + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + relay_order: str = "sequential", + ): + return self.communicator.send(task_name, data, meta, targets, relay_order) + + def get_site_names(self) -> List[str]: + return self.communicator.get_site_names() + diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index e8074c337f..d63eb41e7c 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -107,8 +107,8 @@ def relay_and_wait( min_responses: int, data: any, meta: dict = None, - relay_order: str = "sequential", targets: Optional[List[str]] = None, + relay_order: str = "sequential", callback: Callable = None, ) -> Dict[str, Dict[str, FLModel]]: @@ -133,8 +133,8 @@ def send( task_name: str, data: any, meta: dict = None, - send_order: str = "sequential", targets: Optional[List[str]] = None, + send_order: str = "sequential", ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.send_to_peers(pay_load=msg_payload, send_order=send_order) @@ -144,8 +144,8 @@ def relay( task_name: str, data: any, meta: dict = None, - send_order: str = "sequential", targets: Optional[List[str]] = None, + send_order: str = "sequential", ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.relay_to_peers(msg_payload, send_order) diff --git a/nvflare/app_common/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/wf_comm/wf_comm_api_spec.py index 461dcbc938..5f82be0e8e 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/wf_comm/wf_comm_api_spec.py @@ -66,8 +66,8 @@ def relay_and_wait( min_responses: int, data: any, meta: dict = None, - relay_order: str = "sequential", targets: Optional[List[str]] = None, + relay_order: str = "sequential", callback: Callable = None, ): pass @@ -82,8 +82,8 @@ def send( task_name: str, data: any, meta: dict = None, - send_order: str = "sequential", targets: Optional[str] = None, + send_order: str = "sequential", ): pass @@ -93,8 +93,8 @@ def relay( task_name: str, data: any, meta: dict = None, - relay_order: str = "sequential", targets: Optional[List[str]] = None, + relay_order: str = "sequential", ): pass diff --git a/nvflare/fuel/data_event/data_bus.py b/nvflare/fuel/data_event/data_bus.py index bf2587c827..5c2438eba6 100644 --- a/nvflare/fuel/data_event/data_bus.py +++ b/nvflare/fuel/data_event/data_bus.py @@ -74,8 +74,6 @@ def publish(self, topics: List[str], datum: Any) -> None: """ if topics: for topic in topics: - print(f"{topic=}") - print(f"{self.subscribers=}", ) if topic in self.subscribers: with self._lock: for callback in self.subscribers[topic]: From f0ae6e39f437f2551e006f391b836df8f3be026b Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sat, 27 Jan 2024 20:43:18 -0800 Subject: [PATCH 32/41] define strategy.py in case it is needed. --- nvflare/app_common/abstract/strategy.py | 74 +++++++++++----------- nvflare/app_common/wf_comm/base_wf_comm.py | 2 +- nvflare/app_common/wf_comm/wf_comm_api.py | 70 ++++++++++---------- 3 files changed, 72 insertions(+), 74 deletions(-) diff --git a/nvflare/app_common/abstract/strategy.py b/nvflare/app_common/abstract/strategy.py index bdaf58cdd5..14dad872ec 100644 --- a/nvflare/app_common/abstract/strategy.py +++ b/nvflare/app_common/abstract/strategy.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC, abstractmethod -from typing import Optional, Callable, List +from typing import Callable, List, Optional from nvflare.app_common import wf_comm from nvflare.app_common.wf_comm.wf_comm_api_spec import WFCommAPISpec class Strategy(ABC, WFCommAPISpec): - def __init__(self): self.communicator = wf_comm.get_wf_comm_api() @@ -28,37 +27,37 @@ def run(self): pass def broadcast_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, ): return self.communicator.broadcast_and_wait(task_name, min_responses, data, meta, targets, callback) def send_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - send_order: str = "sequential", - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + send_order: str = "sequential", + callback: Callable = None, ): return self.communicator.send_and_wait(task_name, min_responses, data, meta, targets, send_order, callback) def relay_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - relay_order: str = "sequential", - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + relay_order: str = "sequential", + callback: Callable = None, ): return self.communicator.relay_and_wait(task_name, min_responses, data, meta, targets, relay_order, callback) @@ -66,25 +65,24 @@ def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optio return self.communicator.broadcast(task_name, data, meta, targets) def send( - self, - task_name: str, - data: any, - meta: dict = None, - targets: Optional[str] = None, - send_order: str = "sequential", + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[str] = None, + send_order: str = "sequential", ): return self.communicator.send(task_name, data, meta, targets, send_order) def relay( - self, - task_name: str, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - relay_order: str = "sequential", + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + relay_order: str = "sequential", ): return self.communicator.send(task_name, data, meta, targets, relay_order) def get_site_names(self) -> List[str]: return self.communicator.get_site_names() - diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_comm.py index d151a3f275..ce747570ba 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -110,7 +110,7 @@ def broadcast_to_peers_and_wait(self, pay_load): min_responses=min_responses, wait_time_after_min_received=0, fl_ctx=self.fl_ctx, - abort_signal=abort_signal + abort_signal=abort_signal, ) print("\nafter broadcast_and_wait\n") self.fire_event(AppEventType.ROUND_DONE, self.fl_ctx) diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index d63eb41e7c..a14933c02b 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -57,13 +57,13 @@ def get_site_names(self): return self.meta.get(SITE_NAMES) def broadcast_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, ) -> Union[int, Dict[str, Dict[str, FLModel]]]: meta = {} if meta is None else meta @@ -81,14 +81,14 @@ def register_callback(self, callback): self.event_manager.data_bus.subscribe(["POST_PROCESS_RESULT"], callback) def send_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - send_order: SendOrder = SendOrder.SEQUENTIAL, - targets: Optional[List[str]] = None, - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + send_order: SendOrder = SendOrder.SEQUENTIAL, + targets: Optional[List[str]] = None, + callback: Callable = None, ): meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) @@ -102,14 +102,14 @@ def send_and_wait( return self._get_results(task_name) def relay_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - relay_order: str = "sequential", - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + relay_order: str = "sequential", + callback: Callable = None, ) -> Dict[str, Dict[str, FLModel]]: meta = {} if meta is None else meta @@ -129,23 +129,23 @@ def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optio self.ctrl.broadcast_to_peers(pay_load=msg_payload) def send( - self, - task_name: str, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - send_order: str = "sequential", + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + send_order: str = "sequential", ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.send_to_peers(pay_load=msg_payload, send_order=send_order) def relay( - self, - task_name: str, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - send_order: str = "sequential", + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + send_order: str = "sequential", ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.ctrl.relay_to_peers(msg_payload, send_order) From 207c13a1616b0dd4e1adc50fdd8115f8f709b866 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Sun, 28 Jan 2024 18:52:53 -0800 Subject: [PATCH 33/41] make sure the publish in parallel instead of sequential --- nvflare/app_common/abstract/strategy.py | 138 ++++++++++++------------ nvflare/fuel/data_event/data_bus.py | 5 +- 2 files changed, 73 insertions(+), 70 deletions(-) diff --git a/nvflare/app_common/abstract/strategy.py b/nvflare/app_common/abstract/strategy.py index 14dad872ec..a268a692fe 100644 --- a/nvflare/app_common/abstract/strategy.py +++ b/nvflare/app_common/abstract/strategy.py @@ -17,72 +17,72 @@ from nvflare.app_common import wf_comm from nvflare.app_common.wf_comm.wf_comm_api_spec import WFCommAPISpec - -class Strategy(ABC, WFCommAPISpec): - def __init__(self): - self.communicator = wf_comm.get_wf_comm_api() - - @abstractmethod - def run(self): - pass - - def broadcast_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - callback: Callable = None, - ): - return self.communicator.broadcast_and_wait(task_name, min_responses, data, meta, targets, callback) - - def send_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - send_order: str = "sequential", - callback: Callable = None, - ): - return self.communicator.send_and_wait(task_name, min_responses, data, meta, targets, send_order, callback) - - def relay_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - relay_order: str = "sequential", - callback: Callable = None, - ): - return self.communicator.relay_and_wait(task_name, min_responses, data, meta, targets, relay_order, callback) - - def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): - return self.communicator.broadcast(task_name, data, meta, targets) - - def send( - self, - task_name: str, - data: any, - meta: dict = None, - targets: Optional[str] = None, - send_order: str = "sequential", - ): - return self.communicator.send(task_name, data, meta, targets, send_order) - - def relay( - self, - task_name: str, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - relay_order: str = "sequential", - ): - return self.communicator.send(task_name, data, meta, targets, relay_order) - - def get_site_names(self) -> List[str]: - return self.communicator.get_site_names() +# +# class Strategy(ABC, WFCommAPISpec): +# def __init__(self): +# self.communicator = wf_comm.get_wf_comm_api() +# +# @abstractmethod +# def run(self): +# pass +# +# def broadcast_and_wait( +# self, +# task_name: str, +# min_responses: int, +# data: any, +# meta: dict = None, +# targets: Optional[List[str]] = None, +# callback: Callable = None, +# ): +# return self.communicator.broadcast_and_wait(task_name, min_responses, data, meta, targets, callback) +# +# def send_and_wait( +# self, +# task_name: str, +# min_responses: int, +# data: any, +# meta: dict = None, +# targets: Optional[List[str]] = None, +# send_order: str = "sequential", +# callback: Callable = None, +# ): +# return self.communicator.send_and_wait(task_name, min_responses, data, meta, targets, send_order, callback) +# +# def relay_and_wait( +# self, +# task_name: str, +# min_responses: int, +# data: any, +# meta: dict = None, +# targets: Optional[List[str]] = None, +# relay_order: str = "sequential", +# callback: Callable = None, +# ): +# return self.communicator.relay_and_wait(task_name, min_responses, data, meta, targets, relay_order, callback) +# +# def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): +# return self.communicator.broadcast(task_name, data, meta, targets) +# +# def send( +# self, +# task_name: str, +# data: any, +# meta: dict = None, +# targets: Optional[str] = None, +# send_order: str = "sequential", +# ): +# return self.communicator.send(task_name, data, meta, targets, send_order) +# +# def relay( +# self, +# task_name: str, +# data: any, +# meta: dict = None, +# targets: Optional[List[str]] = None, +# relay_order: str = "sequential", +# ): +# return self.communicator.send(task_name, data, meta, targets, relay_order) +# +# def get_site_names(self) -> List[str]: +# return self.communicator.get_site_names() diff --git a/nvflare/fuel/data_event/data_bus.py b/nvflare/fuel/data_event/data_bus.py index 5c2438eba6..f7c0f18459 100644 --- a/nvflare/fuel/data_event/data_bus.py +++ b/nvflare/fuel/data_event/data_bus.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import threading +from concurrent.futures import ThreadPoolExecutor from typing import Any, Callable, List from nvflare.fuel.data_event.pub_sub import EventPubSub @@ -76,8 +77,10 @@ def publish(self, topics: List[str], datum: Any) -> None: for topic in topics: if topic in self.subscribers: with self._lock: + executor = ThreadPoolExecutor(max_workers=len(self.subscribers[topic])) for callback in self.subscribers[topic]: - callback(topic, datum, self) + executor.submit(callback, topic, datum, self) + executor.shutdown() def send_data(self, key: Any, datum: Any) -> None: """ From a274a214ee2a6d56755668b1f3792939547f4da6 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Mon, 29 Jan 2024 21:43:03 -0800 Subject: [PATCH 34/41] ADD CODE TO ADDRESS THE NEW DESIGN CHANGES. There is a bug in the model update ( where the 2nd round missing keys) --- examples/hello-world/hello-fedavg/README.md | 172 ++++++++++++ .../fedavg/app/config/config_fed_client.conf | 77 ++++++ .../fedavg/app/config/config_fed_server.conf | 29 ++ .../jobs/fedavg/app/custom/cifar10.py | 138 ++++++++++ .../jobs/fedavg/app/custom/cifar10_fl.py | 138 ++++++++++ .../jobs/fedavg/app/custom/fedavg.py | 248 ++++++++++++++++++ .../jobs/fedavg/app/custom/fedavg_pt.py | 41 +++ .../jobs/fedavg/app/custom/net.py | 37 +++ .../hello-fedavg/jobs/fedavg/meta.conf | 7 + .../hello-world/hello-fedavg/requirements.txt | 0 nvflare/apis/wf_controller.py | 61 +++++ nvflare/app_common/app_constant.py | 8 + nvflare/app_common/wf_comm/base_wf_comm.py | 21 +- .../app_common/wf_comm/decomposer_register.py | 29 ++ nvflare/app_common/wf_comm/wf_comm_api.py | 13 +- .../wf_comm/wf_communicator_spec.py | 26 +- .../private/fed/server/server_json_config.py | 38 ++- 17 files changed, 1048 insertions(+), 35 deletions(-) create mode 100644 examples/hello-world/hello-fedavg/README.md create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf create mode 100644 examples/hello-world/hello-fedavg/requirements.txt create mode 100644 nvflare/apis/wf_controller.py create mode 100644 nvflare/app_common/wf_comm/decomposer_register.py diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md new file mode 100644 index 0000000000..76d43f16ed --- /dev/null +++ b/examples/hello-world/hello-fedavg/README.md @@ -0,0 +1,172 @@ +# FedAvg: simplified + +This example illustrates How to use the new Workflow Communication API to contract a workflow: no need to write a controller. + +## FLARE Workflow Communicator API + +The Flare workflow Communicator API only has small set methods + +``` + +class WFCommAPISpec(ABC): + @abstractmethod + def broadcast_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def send_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def relay_and_wait(self, msg_payload: Dict): + pass + + @abstractmethod + def broadcast(self, msg_payload: Dict): + pass + + @abstractmethod + def send(self, msg_payload: Dict): + pass + + @abstractmethod + def relay(self, msg_payload: Dict): + pass + + @abstractmethod + def get_site_names(self) -> List[str]: + pass + + @abstractmethod + def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: + pass + + @abstractmethod + def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: + pass + +``` + + +## Writing a new Workflow + +With this new API writing the new workflow is really simple: + +* Workflow (Server) + +``` +from nvflare.app_common.workflows import wf_comm as flare + +class FedAvg: + def __init__( + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + model_selection_rule: str = None, + ): + super(FedAvg, self).__init__() + + + + self.flare_comm = flare.get_wf_comm_api() + + def run(self): + self.logger.info("start Fed Avg Workflow\n \n") + + start = self.start_round + end = self.start_round + self.num_rounds + + model = self.init_model() + for current_round in range(start, end): + + self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") + self.current_round = current_round + + sag_results = self.scatter_and_gather(model, current_round) + + aggr_result = self.aggr_fn(sag_results) + + self.logger.info(f"aggregate metrics = {aggr_result.metrics}") + + model = update_model(model, aggr_result) + + self.select_best_model(model) + + self.save_model(self.best_model, self.output_path) + + self.logger.info("end Fed Avg Workflow\n \n") + + +``` +Scatter and Gather (SAG): + +SAG is simply ask WFController to broadcast the model to all clients + +``` + def scatter_and_gather(self, model: FLModel, current_round): + msg_payload = {"min_responses": self.min_clients, + "current_round": current_round, + "num_round": self.num_rounds, + "start_round": self.start_round, + "data": model} + + # (2) broadcast and wait + results = self.flare_comm.broadcast_and_wait(msg_payload) + return results +``` + +## Configurations + +### client-side configuration + +This is the same as FLARE Client API configuration + +### server-side configuration + + Server side controller is really simple, all we need is to use WFController with newly defined workflow class + + +``` +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "fed_avg" + path = "nvflare.app_opt.pt.wf_controller.PTWFController" + args { + comm_msg_pull_interval = 5 + task_name = "train" + wf_class_path = "fedavg_pt.PTFedAvg", + wf_args { + min_clients = 2 + num_rounds = 10 + output_path = "/tmp/nvflare/fedavg/mode.pth" + stop_cond = "accuracy >= 55" + model_selection_rule = "accuracy >=" + } + } + } + ] + + components = [] + +} + +``` + + +## Run the job + +assume current working directory is at ```hello-fedavg``` directory + +``` +nvflare simulator -n 2 -t 2 jobs/fedavg -w /tmp/fedavg + +``` diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf new file mode 100644 index 0000000000..06e7925cfa --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf @@ -0,0 +1,77 @@ +{ + format_version = 2 + app_script = "cifar10_fl.py" + app_config = "" + executors = [ + { + tasks = [ + "train" + ] + executor { + path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" + args { + launcher_id = "launcher" + pipe_id = "pipe" + heartbeat_timeout = 60 + params_exchange_format = "pytorch" + params_transfer_type = "DIFF" + train_with_evaluation = true + } + } + } + ] + task_data_filters = [] + task_result_filters = [] + components = [ + { + id = "launcher" + path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" + args { + script = "python3 custom/{app_script} {app_config} " + launch_once = true + } + } + { + id = "pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + } + { + id = "metrics_pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + } + { + id = "metric_relay" + path = "nvflare.app_common.widgets.metric_relay.MetricRelay" + args { + pipe_id = "metrics_pipe" + event_type = "fed.analytix_log_stats" + read_interval = 0.1 + } + } + { + id = "config_preparer" + path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" + args { + component_ids = [ + "metric_relay" + ] + } + } + ] +} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf new file mode 100644 index 0000000000..6af4160b39 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf @@ -0,0 +1,29 @@ +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "fed_avg" + path = "fedavg_pt.PTFedAvg" + args { + min_clients = 2 + num_rounds = 2 + output_path = "/tmp/nvflare/fedavg/mode.pth" + # stop_cond = "accuracy >= 55" + } + } + ] + + components = [ + { + id = "decomposer_register" + path = "nvflare.app_common.wf_comm.decomposer_register.DecomposerRegister" + args { + decomposers = [ "nvflare.app_opt.pt.decomposers.TensorDecomposer"] + } + } + ] +} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py new file mode 100644 index 0000000000..274142432f --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py @@ -0,0 +1,138 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.optim as optim +import torchvision +import torchvision.transforms as transforms +from net import Net + +# (1) import nvflare client API +import nvflare.client as flare + +# (optional) metrics +from nvflare.client.tracking import SummaryWriter + +# (optional) set a fix place so we don't need to download everytime +DATASET_PATH = "/tmp/nvflare/data" +# (optional) We change to use GPU to speed things up. +# if you want to use CPU, change DEVICE="cpu" +DEVICE = "cuda:0" +DEVICE = "cpu" + + +def main(): + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + + batch_size = 4 + epochs = 2 + + trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) + trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) + + testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) + testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) + + net = Net() + + # (2) initializes NVFlare client API + flare.init() + + summary_writer = SummaryWriter() + while flare.is_running(): + # (3) receives FLModel from NVFlare + input_model = flare.receive() + print(f"current_round={input_model.current_round}") + + # (4) loads model from NVFlare + net.load_state_dict(input_model.params) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) + + # (optional) use GPU to speed things up + net.to(DEVICE) + # (optional) calculate total steps + steps = epochs * len(trainloader) + for epoch in range(epochs): # loop over the dataset multiple times + + running_loss = 0.0 + for i, data in enumerate(trainloader, 0): + # get the inputs; data is a list of [inputs, labels] + # (optional) use GPU to speed things up + inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) + + # zero the parameter gradients + optimizer.zero_grad() + + # forward + backward + optimize + outputs = net(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # print statistics + running_loss += loss.item() + if i % 2000 == 1999: # print every 2000 mini-batches + print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") + global_step = input_model.current_round * steps + epoch * len(trainloader) + i + + summary_writer.add_scalar(tag="loss_for_each_batch", scalar=running_loss, global_step=global_step) + running_loss = 0.0 + + print("Finished Training") + + PATH = "./cifar_net.pth" + torch.save(net.state_dict(), PATH) + + # (5) wraps evaluation logic into a method to re-use for + # evaluation on both trained and received model + def evaluate(input_weights): + net = Net() + net.load_state_dict(input_weights) + # (optional) use GPU to speed things up + net.to(DEVICE) + + correct = 0 + total = 0 + # since we're not training, we don't need to calculate the gradients for our outputs + with torch.no_grad(): + for data in testloader: + # (optional) use GPU to speed things up + images, labels = data[0].to(DEVICE), data[1].to(DEVICE) + # calculate outputs by running images through the network + outputs = net(images) + # the class with the highest energy is what we choose as prediction + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") + return 100 * correct // total + + # (6) evaluate on received model for model selection + accuracy = evaluate(input_model.params) + # (7) construct trained FL model + output_model = flare.FLModel( + params=net.cpu().state_dict(), + metrics={"accuracy": accuracy}, + meta={"NUM_STEPS_CURRENT_ROUND": steps}, + ) + # (8) send model back to NVFlare + flare.send(output_model) + + +if __name__ == "__main__": + main() diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py new file mode 100644 index 0000000000..274142432f --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py @@ -0,0 +1,138 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.optim as optim +import torchvision +import torchvision.transforms as transforms +from net import Net + +# (1) import nvflare client API +import nvflare.client as flare + +# (optional) metrics +from nvflare.client.tracking import SummaryWriter + +# (optional) set a fix place so we don't need to download everytime +DATASET_PATH = "/tmp/nvflare/data" +# (optional) We change to use GPU to speed things up. +# if you want to use CPU, change DEVICE="cpu" +DEVICE = "cuda:0" +DEVICE = "cpu" + + +def main(): + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + + batch_size = 4 + epochs = 2 + + trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) + trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) + + testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) + testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) + + net = Net() + + # (2) initializes NVFlare client API + flare.init() + + summary_writer = SummaryWriter() + while flare.is_running(): + # (3) receives FLModel from NVFlare + input_model = flare.receive() + print(f"current_round={input_model.current_round}") + + # (4) loads model from NVFlare + net.load_state_dict(input_model.params) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) + + # (optional) use GPU to speed things up + net.to(DEVICE) + # (optional) calculate total steps + steps = epochs * len(trainloader) + for epoch in range(epochs): # loop over the dataset multiple times + + running_loss = 0.0 + for i, data in enumerate(trainloader, 0): + # get the inputs; data is a list of [inputs, labels] + # (optional) use GPU to speed things up + inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) + + # zero the parameter gradients + optimizer.zero_grad() + + # forward + backward + optimize + outputs = net(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # print statistics + running_loss += loss.item() + if i % 2000 == 1999: # print every 2000 mini-batches + print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") + global_step = input_model.current_round * steps + epoch * len(trainloader) + i + + summary_writer.add_scalar(tag="loss_for_each_batch", scalar=running_loss, global_step=global_step) + running_loss = 0.0 + + print("Finished Training") + + PATH = "./cifar_net.pth" + torch.save(net.state_dict(), PATH) + + # (5) wraps evaluation logic into a method to re-use for + # evaluation on both trained and received model + def evaluate(input_weights): + net = Net() + net.load_state_dict(input_weights) + # (optional) use GPU to speed things up + net.to(DEVICE) + + correct = 0 + total = 0 + # since we're not training, we don't need to calculate the gradients for our outputs + with torch.no_grad(): + for data in testloader: + # (optional) use GPU to speed things up + images, labels = data[0].to(DEVICE), data[1].to(DEVICE) + # calculate outputs by running images through the network + outputs = net(images) + # the class with the highest energy is what we choose as prediction + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") + return 100 * correct // total + + # (6) evaluate on received model for model selection + accuracy = evaluate(input_model.params) + # (7) construct trained FL model + output_model = flare.FLModel( + params=net.cpu().state_dict(), + metrics={"accuracy": accuracy}, + meta={"NUM_STEPS_CURRENT_ROUND": steps}, + ) + # (8) send model back to NVFlare + flare.send(output_model) + + +if __name__ == "__main__": + main() diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py new file mode 100644 index 0000000000..a3c9c1079d --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -0,0 +1,248 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +from typing import Callable, Dict, Optional + +from net import Net + +from nvflare.app_common.abstract.fl_model import FLModel, ParamsType +from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper +from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.utils.math_utils import parse_compare_criteria +from nvflare.security.logging import secure_format_traceback +from nvflare.apis.wf_controller import WFController + +update_model = FLModelUtils.update_model + + +# FedAvg Workflow + + +class FedAvg(WFController): + def __init__( + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + resp_max_wait_time: float = 5, + ): + super(FedAvg, self).__init__() + + self.logger = logging.getLogger(self.__class__.__name__) + self.task_name = "train" + self.output_path = output_path + self.min_clients = min_clients + self.resp_max_wait_time = resp_max_wait_time + self.num_rounds = num_rounds + self.start_round = start_round + self.current_round = start_round + self.best_model: Optional[FLModel] = None + self.aggr_params_helper = WeightedAggregationHelper() + self.aggr_metrics_helper = WeightedAggregationHelper() + self.params_type: Optional[ParamsType] = None + if stop_cond: + self.stop_criteria = parse_compare_criteria(stop_cond) + else: + self.stop_criteria = None + + + def run(self): + self.logger.info("start Fed Avg Workflow\n \n") + start = self.start_round + end = self.start_round + self.num_rounds + + model = self.init_model() + model.start_round = self.start_round + model.total_rounds = self.num_rounds + + for current_round in range(start, end): + + self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") + self.current_round = current_round + + if self.should_stop(model.metrics, self.stop_criteria): + self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") + break + + # no callback + sag_results = self.prepare_broadcast_and_wait(self.task_name, model) + aggr_result = self.aggr_fn(sag_results) + + # # with callback + # self.broadcast_and_wait(task_name=self.task_name, model=model, callback=self.callback) + # aggr_result = self.aggr_fn() + + self.logger.info(f"aggregate metrics = {aggr_result.metrics}") + + print("model size =", sys.getsizeof(model.params)) + + model = update_model(model, aggr_result) + + self.select_best_model(model) + + self.save_model(self.best_model, self.output_path) + + self.logger.info("end Fed Avg Workflow\n \n") + + def init_model(self): + net = Net() + model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) + return model + + def prepare_broadcast_and_wait(self, task_name, model: FLModel, callback=None): + # (2) broadcast and wait + model.current_round = self.current_round + results = self.broadcast_and_wait( + task_name=task_name, min_responses=self.min_clients, data=model, callback=callback + ) + if self.callback is None: + return results + + def callback(self, data, topic): + self.intime_agg_fn(data, self.aggr_params_helper, self.aggr_metrics_helper) + + def intime_agg_fn(self, data, aggr_params_helper, aggr_metrics_helper): + self.logger.info("\n fed avg intime_aggregate \n") + + if not data: + raise RuntimeError("input is None or empty") + task_name, task_result = next(iter(data.items())) + + try: + for site, fl_model in task_result.items(): + if self.params_type is None: + self.params_type = fl_model.params_type + + aggr_params_helper.add( + data=fl_model.params, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) + + self.logger.info(f"site={site} {fl_model.metrics=}") + + aggr_metrics_helper.add( + data=fl_model.metrics, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) + + except Exception as e: + raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") + + def aggr_fn(self, sag_result: Optional[Dict[str, Dict[str, FLModel]]] = None) -> FLModel: + + if self.callback and sag_result is None: + return self.get_aggr_result(self.aggr_params_helper, self.aggr_metrics_helper) + else: + + self.logger.info("fed avg aggregate \n") + + if not sag_result: + raise RuntimeError("input is None or empty") + + # we only have one task + task_name, task_result = next(iter(sag_result.items())) + self.logger.info(f"aggregating {len(task_result)} update(s) at round {self.current_round}") + + try: + aggr_params_helper = WeightedAggregationHelper() + aggr_metrics_helper = WeightedAggregationHelper() + params_type = None + for site, fl_model in task_result.items(): + if params_type is None: + params_type = fl_model.params_type + + aggr_params_helper.add( + data=fl_model.params, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) + + self.logger.info(f"site={site} {fl_model.metrics=}") + + aggr_metrics_helper.add( + data=fl_model.metrics, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) + + return self.get_aggr_result(aggr_params_helper, aggr_metrics_helper) + + except Exception as e: + raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") + + def select_best_model(self, curr_model: FLModel): + if self.best_model is None: + self.best_model = curr_model + return + + if self.stop_criteria: + metric, _, op_fn = self.stop_criteria + self.logger.info("compare models") + if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): + self.best_model = curr_model + else: + self.best_model = curr_model + + def save_model(self, model: FLModel, file_path: str): + pass + + def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[str] = None): + self.logger.info(f"stop_criteria, metrics = {stop_criteria=}, {metrics=}") + if stop_criteria is None or metrics is None: + return False + + key, target, op_fn = stop_criteria + value = metrics.get(key, None) + + if value is None: + raise RuntimeError(f"stop criteria key '{key}' doesn't exists in metrics") + + return op_fn(value, target) + + def is_curr_mode_better( + self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable + ) -> bool: + curr_metrics = curr_model.metrics + if curr_metrics is None: + return False + if target_metric not in curr_metrics: + return False + + best_metrics = best_model.metrics + return op_fn(curr_metrics.get(target_metric), best_metrics.get(target_metric)) + + def get_aggr_result(self, aggr_params_helper, aggr_metrics_helper): + aggr_params = aggr_params_helper.get_result() + aggr_metrics = aggr_metrics_helper.get_result() + + aggr_result = FLModel( + params=aggr_params, + params_type=self.params_type, + metrics=aggr_metrics, + meta={ + "num_rounds_aggregated": 1 + (self.current_round - self.start_round), + "current_round": self.current_round, + }, + ) + return aggr_result diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py new file mode 100644 index 0000000000..8a18abdd08 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import torch +from fedavg import FedAvg + +from nvflare.app_common.abstract.fl_model import FLModel + + +class PTFedAvg(FedAvg): + def __init__( + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + ): + super().__init__(min_clients, num_rounds, output_path, start_round, stop_cond) + + def save_model(self, model: FLModel, file_path: str): + if not file_path: + raise ValueError("invalid file path") + + dir_name = os.path.dirname(file_path) + os.makedirs(dir_name, exist_ok=True) + + self.logger.info(f"save best model to {file_path} \n") + torch.save(model.params, file_path) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py new file mode 100644 index 0000000000..031f84f432 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Net(nn.Module): + def __init__(self): + super().__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = torch.flatten(x, 1) # flatten all dimensions except batch + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf new file mode 100644 index 0000000000..1c27c4e99c --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf @@ -0,0 +1,7 @@ +{ + name = "fedavg" + deploy_map { + app = ["@ALL"] + } + min_clients = 2 +} diff --git a/examples/hello-world/hello-fedavg/requirements.txt b/examples/hello-world/hello-fedavg/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/apis/wf_controller.py b/nvflare/apis/wf_controller.py new file mode 100644 index 0000000000..cb17fb61c7 --- /dev/null +++ b/nvflare/apis/wf_controller.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import abstractmethod, ABC +from typing import Optional, Callable, List + +from nvflare.app_common import wf_comm + + +class WFController(ABC): + + def __init__(self): + self.communicator = wf_comm.get_wf_comm_api() + + @abstractmethod + def run(self): + pass + + def broadcast_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, + ): + return self.communicator.broadcast_and_wait(task_name, min_responses, data, meta, targets, callback) + + def send_and_wait(self, task_name: str, min_responses: int, data: any, meta: dict = None, + targets: Optional[List[str]] = None, send_order: str = "sequential", callback: Callable = None): + return self.communicator.send_and_wait(task_name, min_responses, data, meta, targets, send_order, callback) + + def relay_and_wait(self, task_name: str, min_responses: int, data: any, meta: dict = None, + targets: Optional[List[str]] = None, relay_order: str = "sequential", callback: Callable = None): + return self.communicator.relay_and_wait(task_name, min_responses, data, meta, targets, relay_order, callback) + + def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): + return self.communicator.broadcast(task_name, data, meta, targets) + + def send(self, task_name: str, data: any, meta: dict = None, targets: Optional[str] = None, + send_order: str = "sequential"): + return self.communicator.send(task_name, data, meta, targets, send_order) + + def relay(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None, + relay_order: str = "sequential"): + return self.communicator.send(task_name, data, meta, targets, relay_order) + + def get_site_names(self) -> List[str]: + return self.communicator.get_site_names() + diff --git a/nvflare/app_common/app_constant.py b/nvflare/app_common/app_constant.py index 73928fd95b..f56b767b24 100644 --- a/nvflare/app_common/app_constant.py +++ b/nvflare/app_common/app_constant.py @@ -212,3 +212,11 @@ class PSIConst(AppConstants): REQUEST_MSG = "PSI_REQUEST_MSG" REQUEST_MSG_SET = "PSI_REQUEST_MSG_SET" RESPONSE_MSG = "PSI_RESPONSE_MSG" + + +class CommConstants(object): + COMMUNICATOR = "communicator" + CONTROLLER = "controller" + TASK_RESULT = "TASK_RESULT" + POST_PROCESS_RESULT = "POST_PROCESS_RESULT" + diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_comm.py index ce747570ba..b33c83314a 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -25,6 +25,7 @@ from nvflare.app_common.app_constant import AppConstants from nvflare.app_common.app_event_type import AppEventType from nvflare.app_common.utils.fl_model_utils import FLModelUtils +from nvflare.app_common.wf_comm.decomposer_register import DecomposerRegister from nvflare.app_common.wf_comm.wf_comm_api import WFCommAPI from nvflare.app_common.wf_comm.wf_comm_api_spec import ( DATA, @@ -44,9 +45,9 @@ class BaseWFCommunicator(FLComponent, WFCommunicatorSpec, ControllerSpec, ABC): def __init__( - self, - task_timeout: int = 0, - result_pull_interval: float = 0.2, + self, + task_timeout: int = 0, + result_pull_interval: float = 0.2, ): super().__init__() self.strategy_fn_name = "run" @@ -67,10 +68,20 @@ def start_controller(self, fl_ctx: FLContext): self.event_manager = EventManager(self.data_bus) self.engine = self.fl_ctx.get_engine() + self.register_decomposers() + self.clients = self.engine.get_clients() self.publish_comm_api() self.log_info(fl_ctx, "workflow controller started") + def register_decomposers(self): + decomposer_register = self.engine.get_component("decomposer_register") + if decomposer_register: + if not isinstance(decomposer_register, DecomposerRegister): + raise ValueError( + f"decomposer_register component must be type of 'DecomposerRegister', got {type(decomposer_register)}.") + decomposer_register.register() + def publish_comm_api(self): comm_api = WFCommAPI() comm_api.meta.update({SITE_NAMES: self.get_site_names()}) @@ -79,7 +90,7 @@ def publish_comm_api(self): def start_workflow(self, abort_signal, fl_ctx): try: fl_ctx.set_prop("abort_signal", abort_signal) - func = getattr(self.get_strategy(), self.strategy_fn_name) + func = getattr(self.get_controller(), self.strategy_fn_name) func() except Exception as e: @@ -91,7 +102,7 @@ def stop_controller(self, fl_ctx: FLContext): pass def process_result_of_unknown_task( - self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext + self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext ): pass diff --git a/nvflare/app_common/wf_comm/decomposer_register.py b/nvflare/app_common/wf_comm/decomposer_register.py new file mode 100644 index 0000000000..c6f8ebac94 --- /dev/null +++ b/nvflare/app_common/wf_comm/decomposer_register.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +from nvflare.apis.fl_component import FLComponent +from nvflare.fuel.utils.class_utils import instantiate_class +from nvflare.fuel.utils.fobs import fobs + + +class DecomposerRegister(FLComponent): + def __init__(self, decomposers: List[str]): + super(DecomposerRegister, self).__init__() + self.decomposers = decomposers + + def register(self): + for class_path in self.decomposers: + d = instantiate_class(class_path, init_params=None) + fobs.register(d) diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index a14933c02b..41575edcfd 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -20,6 +20,7 @@ from nvflare.apis.controller_spec import SendOrder from nvflare.apis.fl_constant import ReturnCode from nvflare.app_common.abstract.fl_model import FLModel +from nvflare.app_common.app_constant import CommConstants from nvflare.app_common.wf_comm.wf_comm_api_spec import ( CURRENT_ROUND, DATA, @@ -47,10 +48,10 @@ def __init__(self): self.task_result_lock = threading.Lock() data_bus = DataBus() - data_bus.subscribe(topics=["TASK_RESULT"], callback=self.result_callback) + data_bus.subscribe(topics=[CommConstants.TASK_RESULT], callback=self.result_callback) self.event_manager = EventManager(data_bus) - self.ctrl = data_bus.receive_data("communicator") + self.ctrl = data_bus.receive_data(CommConstants.COMMUNICATOR) self._check_inputs() def get_site_names(self): @@ -69,16 +70,14 @@ def broadcast_and_wait( meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) self.register_callback(callback) - print("\ncalling broadcast_to_peers_and_wait\n") self.ctrl.broadcast_to_peers_and_wait(msg_payload) - print("\nafter broadcast_to_peers_and_wait\n") if callback is None: return self._get_results(task_name) def register_callback(self, callback): if callback: - self.event_manager.data_bus.subscribe(["POST_PROCESS_RESULT"], callback) + self.event_manager.data_bus.subscribe([CommConstants.POST_PROCESS_RESULT], callback) def send_and_wait( self, @@ -201,11 +200,11 @@ def _check_inputs(self): raise RuntimeError("missing Controller") def result_callback(self, topic, data, data_bus): - if topic == "TASK_RESULT": + if topic == CommConstants.TASK_RESULT: task, site_result = next(iter(data.items())) # fire event with process data one_result = self._process_one_result(site_result) - self.event_manager.fire_event("POST_PROCESS_RESULT", {task: one_result}) + self.event_manager.fire_event(CommConstants.POST_PROCESS_RESULT, {task: one_result}) site_task_results = self.task_results.get(task, []) site_task_results.append(site_result) self.task_results[task] = site_task_results diff --git a/nvflare/app_common/wf_comm/wf_communicator_spec.py b/nvflare/app_common/wf_comm/wf_communicator_spec.py index 15dbc0eea2..ef724a2a73 100644 --- a/nvflare/app_common/wf_comm/wf_communicator_spec.py +++ b/nvflare/app_common/wf_comm/wf_communicator_spec.py @@ -24,7 +24,7 @@ class WFCommunicatorSpec(ABC): def __init__(self): - self.strategy_config: Optional[Dict] = None + self.controller_config: Optional[Dict] = None @abstractmethod def broadcast_to_peers_and_wait(self, pay_load: Dict): @@ -50,23 +50,23 @@ def relay_to_peers_and_wait(self, pay_load: Dict, send_order: SendOrder = SendOr def relay_to_peers(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): pass - def set_strategy_config(self, strategy_config: Dict): - if strategy_config is None: - raise ValueError("strategy_config is None") + def set_controller_config(self, controller_config: Dict): + if controller_config is None: + raise ValueError("controller_config is None") - if not isinstance(strategy_config, dict): - raise ValueError(f"strategy_config should be Dict, found '{type(strategy_config)}'") + if not isinstance(controller_config, dict): + raise ValueError(f"controller_config should be Dict, found '{type(controller_config)}'") - self.strategy_config = strategy_config + self.controller_config = controller_config - def get_strategy(self): - strategy = None - if isinstance(self.strategy_config, dict): - strategy = ComponentBuilder().build_component(self.strategy_config) - if strategy is None: + def get_controller(self): + controller = None + if isinstance(self.controller_config, dict): + controller = ComponentBuilder().build_component(self.controller_config) + if controller is None: raise ValueError("strategy should provided, but get None") - return strategy + return controller def register_serializers(self, serializer_class_paths: List[str] = None): self.register_default_serializers() diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index 2c5009aa18..a681a4ac32 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -17,6 +17,7 @@ from nvflare.apis.fl_component import FLComponent from nvflare.apis.fl_constant import SystemConfigs, SystemVarName from nvflare.apis.responder import Responder +from nvflare.app_common.app_constant import CommConstants from nvflare.app_common.wf_comm.wf_communicator import WFCommunicator from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec from nvflare.fuel.data_event.data_bus import DataBus @@ -48,10 +49,23 @@ def __init__(self, id, responder: Responder, strategy=None): def enhance_workflow_config(element: dict): - if "strategy" in element: - strategy_config = element.get("strategy") - strategy_config["lazy_instantiate"] = True - element["strategy"] = strategy_config + if CommConstants.CONTROLLER in element: + controller_config = element.get(CommConstants.CONTROLLER) + controller_config["lazy_instantiate"] = True + element[CommConstants.CONTROLLER] = controller_config + elif CommConstants.COMMUNICATOR not in element: + controller_config = element.copy() + controller_config["lazy_instantiate"] = True + element = {CommConstants.CONTROLLER: controller_config} + else: + wf_config = element.copy() + comm_config = wf_config.pop(CommConstants.COMMUNICATOR) + controller_config = wf_config + controller_config["lazy_instantiate"] = True + element = {CommConstants.COMMUNICATOR: comm_config, + CommConstants.CONTROLLER: controller_config + } + return element @@ -139,20 +153,24 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): if re.search(r"^workflows\.#[0-9]+$", path): element = enhance_workflow_config(element) + + print("\n\n element =", element) component = self.authorize_and_build_component(element, config_ctx, node) + + # todo: fix: dependency graph is not right, we are now nvflare.private depending on the AppCommon class + # todo: refactoring the code into small methods if isinstance(component, dict): wf_config = component - communicator = wf_config.get("communicator") + communicator = wf_config.get(CommConstants.COMMUNICATOR) if communicator is None: communicator = WFCommunicator() if isinstance(communicator, WFCommunicatorSpec): - strategy_config = wf_config.get("strategy") - strategy_config["lazy_instantiate"] = False - communicator.set_strategy_config(strategy_config) - communicator.register_serializers(strategy_config.get("serializers")) + controller_config = wf_config.get(CommConstants.CONTROLLER) + controller_config["lazy_instantiate"] = False + communicator.set_controller_config(controller_config) data_bus = DataBus() - data_bus.send_data("communicator", communicator) + data_bus.send_data(CommConstants.COMMUNICATOR, communicator) responder = communicator else: responder = component From 7c4f62eb7d5f13eaaa5e4df76755563da1a24b83 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Mon, 29 Jan 2024 21:45:54 -0800 Subject: [PATCH 35/41] format code --- .../jobs/fedavg/app/custom/fedavg.py | 3 +- nvflare/apis/wf_controller.py | 64 ++++++++++---- nvflare/app_common/abstract/strategy.py | 88 ------------------- nvflare/app_common/app_constant.py | 1 - nvflare/app_common/wf_comm/base_wf_comm.py | 11 +-- .../private/fed/server/server_json_config.py | 4 +- 6 files changed, 53 insertions(+), 118 deletions(-) delete mode 100644 nvflare/app_common/abstract/strategy.py diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index a3c9c1079d..f9f19478f1 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -18,12 +18,12 @@ from net import Net +from nvflare.apis.wf_controller import WFController from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.utils.math_utils import parse_compare_criteria from nvflare.security.logging import secure_format_traceback -from nvflare.apis.wf_controller import WFController update_model = FLModelUtils.update_model @@ -60,7 +60,6 @@ def __init__( else: self.stop_criteria = None - def run(self): self.logger.info("start Fed Avg Workflow\n \n") start = self.start_round diff --git a/nvflare/apis/wf_controller.py b/nvflare/apis/wf_controller.py index cb17fb61c7..fde4968b47 100644 --- a/nvflare/apis/wf_controller.py +++ b/nvflare/apis/wf_controller.py @@ -11,14 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from abc import abstractmethod, ABC -from typing import Optional, Callable, List +from abc import ABC, abstractmethod +from typing import Callable, List, Optional from nvflare.app_common import wf_comm class WFController(ABC): - def __init__(self): self.communicator = wf_comm.get_wf_comm_api() @@ -27,35 +26,62 @@ def run(self): pass def broadcast_and_wait( - self, - task_name: str, - min_responses: int, - data: any, - meta: dict = None, - targets: Optional[List[str]] = None, - callback: Callable = None, + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + callback: Callable = None, ): return self.communicator.broadcast_and_wait(task_name, min_responses, data, meta, targets, callback) - def send_and_wait(self, task_name: str, min_responses: int, data: any, meta: dict = None, - targets: Optional[List[str]] = None, send_order: str = "sequential", callback: Callable = None): + def send_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + send_order: str = "sequential", + callback: Callable = None, + ): return self.communicator.send_and_wait(task_name, min_responses, data, meta, targets, send_order, callback) - def relay_and_wait(self, task_name: str, min_responses: int, data: any, meta: dict = None, - targets: Optional[List[str]] = None, relay_order: str = "sequential", callback: Callable = None): + def relay_and_wait( + self, + task_name: str, + min_responses: int, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + relay_order: str = "sequential", + callback: Callable = None, + ): return self.communicator.relay_and_wait(task_name, min_responses, data, meta, targets, relay_order, callback) def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): return self.communicator.broadcast(task_name, data, meta, targets) - def send(self, task_name: str, data: any, meta: dict = None, targets: Optional[str] = None, - send_order: str = "sequential"): + def send( + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[str] = None, + send_order: str = "sequential", + ): return self.communicator.send(task_name, data, meta, targets, send_order) - def relay(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None, - relay_order: str = "sequential"): + def relay( + self, + task_name: str, + data: any, + meta: dict = None, + targets: Optional[List[str]] = None, + relay_order: str = "sequential", + ): return self.communicator.send(task_name, data, meta, targets, relay_order) def get_site_names(self) -> List[str]: return self.communicator.get_site_names() - diff --git a/nvflare/app_common/abstract/strategy.py b/nvflare/app_common/abstract/strategy.py deleted file mode 100644 index a268a692fe..0000000000 --- a/nvflare/app_common/abstract/strategy.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from abc import ABC, abstractmethod -from typing import Callable, List, Optional - -from nvflare.app_common import wf_comm -from nvflare.app_common.wf_comm.wf_comm_api_spec import WFCommAPISpec - -# -# class Strategy(ABC, WFCommAPISpec): -# def __init__(self): -# self.communicator = wf_comm.get_wf_comm_api() -# -# @abstractmethod -# def run(self): -# pass -# -# def broadcast_and_wait( -# self, -# task_name: str, -# min_responses: int, -# data: any, -# meta: dict = None, -# targets: Optional[List[str]] = None, -# callback: Callable = None, -# ): -# return self.communicator.broadcast_and_wait(task_name, min_responses, data, meta, targets, callback) -# -# def send_and_wait( -# self, -# task_name: str, -# min_responses: int, -# data: any, -# meta: dict = None, -# targets: Optional[List[str]] = None, -# send_order: str = "sequential", -# callback: Callable = None, -# ): -# return self.communicator.send_and_wait(task_name, min_responses, data, meta, targets, send_order, callback) -# -# def relay_and_wait( -# self, -# task_name: str, -# min_responses: int, -# data: any, -# meta: dict = None, -# targets: Optional[List[str]] = None, -# relay_order: str = "sequential", -# callback: Callable = None, -# ): -# return self.communicator.relay_and_wait(task_name, min_responses, data, meta, targets, relay_order, callback) -# -# def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): -# return self.communicator.broadcast(task_name, data, meta, targets) -# -# def send( -# self, -# task_name: str, -# data: any, -# meta: dict = None, -# targets: Optional[str] = None, -# send_order: str = "sequential", -# ): -# return self.communicator.send(task_name, data, meta, targets, send_order) -# -# def relay( -# self, -# task_name: str, -# data: any, -# meta: dict = None, -# targets: Optional[List[str]] = None, -# relay_order: str = "sequential", -# ): -# return self.communicator.send(task_name, data, meta, targets, relay_order) -# -# def get_site_names(self) -> List[str]: -# return self.communicator.get_site_names() diff --git a/nvflare/app_common/app_constant.py b/nvflare/app_common/app_constant.py index f56b767b24..eaa14f54f2 100644 --- a/nvflare/app_common/app_constant.py +++ b/nvflare/app_common/app_constant.py @@ -219,4 +219,3 @@ class CommConstants(object): CONTROLLER = "controller" TASK_RESULT = "TASK_RESULT" POST_PROCESS_RESULT = "POST_PROCESS_RESULT" - diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_comm.py index b33c83314a..9223ac26e6 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -45,9 +45,9 @@ class BaseWFCommunicator(FLComponent, WFCommunicatorSpec, ControllerSpec, ABC): def __init__( - self, - task_timeout: int = 0, - result_pull_interval: float = 0.2, + self, + task_timeout: int = 0, + result_pull_interval: float = 0.2, ): super().__init__() self.strategy_fn_name = "run" @@ -79,7 +79,8 @@ def register_decomposers(self): if decomposer_register: if not isinstance(decomposer_register, DecomposerRegister): raise ValueError( - f"decomposer_register component must be type of 'DecomposerRegister', got {type(decomposer_register)}.") + f"decomposer_register component must be type of 'DecomposerRegister', got {type(decomposer_register)}." + ) decomposer_register.register() def publish_comm_api(self): @@ -102,7 +103,7 @@ def stop_controller(self, fl_ctx: FLContext): pass def process_result_of_unknown_task( - self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext + self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext ): pass diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index a681a4ac32..ed6c1c317d 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -62,9 +62,7 @@ def enhance_workflow_config(element: dict): comm_config = wf_config.pop(CommConstants.COMMUNICATOR) controller_config = wf_config controller_config["lazy_instantiate"] = True - element = {CommConstants.COMMUNICATOR: comm_config, - CommConstants.CONTROLLER: controller_config - } + element = {CommConstants.COMMUNICATOR: comm_config, CommConstants.CONTROLLER: controller_config} return element From 5fdd1c02188278a840e17f65191331978b855f03 Mon Sep 17 00:00:00 2001 From: chesterxgchen Date: Tue, 30 Jan 2024 09:37:39 -0800 Subject: [PATCH 36/41] fix the issue with return result --- .../jobs/fedavg/app/custom/cifar10_fl.py | 4 +- .../jobs/fedavg/app/custom/fedavg.py | 90 +++++++++---------- nvflare/app_common/wf_comm/base_wf_comm.py | 11 +-- nvflare/app_common/wf_comm/wf_comm_api.py | 4 +- 4 files changed, 51 insertions(+), 58 deletions(-) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py index 274142432f..7f7963a43c 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py @@ -36,7 +36,7 @@ def main(): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - batch_size = 4 + batch_size = 512 epochs = 2 trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) @@ -92,7 +92,7 @@ def main(): summary_writer.add_scalar(tag="loss_for_each_batch", scalar=running_loss, global_step=global_step) running_loss = 0.0 - print("Finished Training") + print(f"Finished Training for round {input_model.current_round}") PATH = "./cifar_net.pth" torch.save(net.state_dict(), PATH) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py index f9f19478f1..691e011f5a 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py @@ -17,7 +17,6 @@ from typing import Callable, Dict, Optional from net import Net - from nvflare.apis.wf_controller import WFController from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper @@ -33,13 +32,13 @@ class FedAvg(WFController): def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - resp_max_wait_time: float = 5, + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + resp_max_wait_time: float = 5, ): super(FedAvg, self).__init__() @@ -87,11 +86,7 @@ def run(self): # aggr_result = self.aggr_fn() self.logger.info(f"aggregate metrics = {aggr_result.metrics}") - - print("model size =", sys.getsizeof(model.params)) - model = update_model(model, aggr_result) - self.select_best_model(model) self.save_model(self.best_model, self.output_path) @@ -109,8 +104,10 @@ def prepare_broadcast_and_wait(self, task_name, model: FLModel, callback=None): results = self.broadcast_and_wait( task_name=task_name, min_responses=self.min_clients, data=model, callback=callback ) - if self.callback is None: + if callback is None: return results + else: + return None def callback(self, data, topic): self.intime_agg_fn(data, self.aggr_params_helper, self.aggr_metrics_helper) @@ -146,49 +143,50 @@ def intime_agg_fn(self, data, aggr_params_helper, aggr_metrics_helper): except Exception as e: raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") - def aggr_fn(self, sag_result: Optional[Dict[str, Dict[str, FLModel]]] = None) -> FLModel: - - if self.callback and sag_result is None: + def intime_aggr_fn(self, sag_result: Optional[Dict[str, Dict[str, FLModel]]] = None) -> FLModel: + if self.callback: return self.get_aggr_result(self.aggr_params_helper, self.aggr_metrics_helper) else: + raise ValueError("callback function needs to be defined") - self.logger.info("fed avg aggregate \n") + def aggr_fn(self, sag_result: Optional[Dict[str, Dict[str, FLModel]]] = None) -> FLModel: + self.logger.info("fed avg aggregate \n") - if not sag_result: - raise RuntimeError("input is None or empty") + if not sag_result: + raise RuntimeError("input is None or empty") - # we only have one task - task_name, task_result = next(iter(sag_result.items())) - self.logger.info(f"aggregating {len(task_result)} update(s) at round {self.current_round}") + # we only have one task + task_name, task_result = next(iter(sag_result.items())) + self.logger.info(f"aggregating {len(task_result)} update(s) at round {self.current_round}") - try: - aggr_params_helper = WeightedAggregationHelper() - aggr_metrics_helper = WeightedAggregationHelper() - params_type = None - for site, fl_model in task_result.items(): - if params_type is None: - params_type = fl_model.params_type + try: + aggr_params_helper = WeightedAggregationHelper() + aggr_metrics_helper = WeightedAggregationHelper() + params_type = None + for site, fl_model in task_result.items(): + if params_type is None: + params_type = fl_model.params_type - aggr_params_helper.add( - data=fl_model.params, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) + aggr_params_helper.add( + data=fl_model.params, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) - self.logger.info(f"site={site} {fl_model.metrics=}") + self.logger.info(f"site={site} {fl_model.metrics=}") - aggr_metrics_helper.add( - data=fl_model.metrics, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) + aggr_metrics_helper.add( + data=fl_model.metrics, + weight=self.current_round, + contributor_name=site, + contribution_round=self.current_round, + ) - return self.get_aggr_result(aggr_params_helper, aggr_metrics_helper) + return self.get_aggr_result(aggr_params_helper, aggr_metrics_helper) - except Exception as e: - raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") + except Exception as e: + raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") def select_best_model(self, curr_model: FLModel): if self.best_model is None: @@ -220,7 +218,7 @@ def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[st return op_fn(value, target) def is_curr_mode_better( - self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable + self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable ) -> bool: curr_metrics = curr_model.metrics if curr_metrics is None: diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_comm.py index 9223ac26e6..d7ae1147ec 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_comm.py @@ -22,7 +22,7 @@ from nvflare.apis.fl_context import FLContext from nvflare.apis.shareable import Shareable from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.app_constant import AppConstants +from nvflare.app_common.app_constant import AppConstants, CommConstants from nvflare.app_common.app_event_type import AppEventType from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.wf_comm.decomposer_register import DecomposerRegister @@ -114,17 +114,14 @@ def broadcast_to_peers_and_wait(self, pay_load): self.fl_ctx.set_prop("task_name", task.name) - print(f"call broadcast_and_wait to {min_responses=}, {targets=} , {task.timeout=}\n") - self.broadcast_and_wait( task=task, + fl_ctx=self.fl_ctx, targets=targets, min_responses=min_responses, wait_time_after_min_received=0, - fl_ctx=self.fl_ctx, abort_signal=abort_signal, ) - print("\nafter broadcast_and_wait\n") self.fire_event(AppEventType.ROUND_DONE, self.fl_ctx) self.log_info(self.fl_ctx, f"Round {current_round} finished.") @@ -243,14 +240,12 @@ def _result_received_cb(self, client_task: ClientTask, fl_ctx: FLContext): rc = result.get_return_code() results: Dict[str, any] = {STATUS: rc} - print(f"{rc=}") if rc == ReturnCode.OK: self.log_info(fl_ctx, f"Received result entries from client:{client_name} for task {task_name}") fl_model = FLModelUtils.from_shareable(result) results[RESULT] = {client_name: fl_model} payload = {task_name: results} - print(f"fire_event to TASK_RESULT, {payload=}") - self.event_manager.fire_event("TASK_RESULT", payload) + self.event_manager.fire_event(CommConstants.TASK_RESULT, payload) else: self.handle_client_errors(rc, client_task, fl_ctx) diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index 41575edcfd..357a1fa295 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -75,6 +75,8 @@ def broadcast_and_wait( if callback is None: return self._get_results(task_name) + return 0 + def register_callback(self, callback): if callback: self.event_manager.data_bus.subscribe([CommConstants.POST_PROCESS_RESULT], callback) @@ -163,7 +165,6 @@ def _process_one_result(self, site_result) -> Dict[str, FLModel]: return task_result def _get_results(self, task_name) -> Dict[str, Dict[str, FLModel]]: - print("_get_results\n") batch_result: Dict = {} site_results = self.task_results.get(task_name) if not site_results: @@ -179,7 +180,6 @@ def _get_results(self, task_name) -> Dict[str, Dict[str, FLModel]]: with self.task_result_lock: self.task_results[task_name] = [] - print("return batch_result=", batch_result) return batch_result def _check_result(self, site_result): From 5d09732b0cc726bcb4efb234f7b048af6ef35375 Mon Sep 17 00:00:00 2001 From: Sean Yang Date: Tue, 6 Feb 2024 15:59:43 -0800 Subject: [PATCH 37/41] cleanup, fix original controller parsing --- .../fedavg/app/config/config_fed_client.conf | 2 +- .../fedavg/app/config/config_fed_server.conf | 25 +-- .../jobs/fedavg/app/custom/cifar10.py | 138 ------------- .../jobs/fedavg/app/custom/cifar10_fl.py | 138 ------------- .../jobs/fedavg/app/custom/train.py | 188 ++++++++++++++++++ .../hello-world/hello-fedavg/requirements.txt | 3 + .../config_fed_client.conf | 116 +++++++++++ .../config_fed_server.conf | 41 ++++ job_templates/sag_pt_wf_controller/info.conf | 5 + job_templates/sag_pt_wf_controller/info.md | 11 + job_templates/sag_pt_wf_controller/meta.conf | 10 + nvflare/app_common/app_constant.py | 7 - ...ase_wf_comm.py => base_wf_communicator.py} | 7 +- nvflare/app_common/wf_comm/wf_comm_api.py | 18 +- .../app_common/wf_comm/wf_comm_api_spec.py | 6 - nvflare/app_common/wf_comm/wf_communicator.py | 3 +- .../wf_comm/wf_communicator_spec.py | 2 +- .../app_common/workflows/fed_avg.py | 22 +- .../app_common/workflows/fed_avg_pt.py | 2 +- nvflare/private/defs.py | 7 + .../private/fed/server/server_json_config.py | 95 ++++----- 21 files changed, 467 insertions(+), 379 deletions(-) delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py create mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py create mode 100644 job_templates/sag_pt_wf_controller/config_fed_client.conf create mode 100644 job_templates/sag_pt_wf_controller/config_fed_server.conf create mode 100644 job_templates/sag_pt_wf_controller/info.conf create mode 100644 job_templates/sag_pt_wf_controller/info.md create mode 100644 job_templates/sag_pt_wf_controller/meta.conf rename nvflare/app_common/wf_comm/{base_wf_comm.py => base_wf_communicator.py} (97%) rename examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py => nvflare/app_common/workflows/fed_avg.py (95%) rename examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py => nvflare/app_common/workflows/fed_avg_pt.py (95%) diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf index 06e7925cfa..74f9ab24b0 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf @@ -1,6 +1,6 @@ { format_version = 2 - app_script = "cifar10_fl.py" + app_script = "train.py" app_config = "" executors = [ { diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf index 6af4160b39..96ede86cea 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf @@ -5,25 +5,18 @@ task_result_filters = [] workflows = [ - { - id = "fed_avg" - path = "fedavg_pt.PTFedAvg" - args { - min_clients = 2 - num_rounds = 2 - output_path = "/tmp/nvflare/fedavg/mode.pth" - # stop_cond = "accuracy >= 55" - } + { + id = "fed_avg" + path = "nvflare.app_common.workflows.fed_avg_pt.PTFedAvg" + args { + min_clients = 2 + num_rounds = 2 + output_path = "/tmp/nvflare/fedavg/mode.pth" + # stop_cond = "accuracy >= 55" + } } ] components = [ - { - id = "decomposer_register" - path = "nvflare.app_common.wf_comm.decomposer_register.DecomposerRegister" - args { - decomposers = [ "nvflare.app_opt.pt.decomposers.TensorDecomposer"] - } - } ] } diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py deleted file mode 100644 index 274142432f..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn as nn -import torch.optim as optim -import torchvision -import torchvision.transforms as transforms -from net import Net - -# (1) import nvflare client API -import nvflare.client as flare - -# (optional) metrics -from nvflare.client.tracking import SummaryWriter - -# (optional) set a fix place so we don't need to download everytime -DATASET_PATH = "/tmp/nvflare/data" -# (optional) We change to use GPU to speed things up. -# if you want to use CPU, change DEVICE="cpu" -DEVICE = "cuda:0" -DEVICE = "cpu" - - -def main(): - transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - - batch_size = 4 - epochs = 2 - - trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) - trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) - - testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) - testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) - - net = Net() - - # (2) initializes NVFlare client API - flare.init() - - summary_writer = SummaryWriter() - while flare.is_running(): - # (3) receives FLModel from NVFlare - input_model = flare.receive() - print(f"current_round={input_model.current_round}") - - # (4) loads model from NVFlare - net.load_state_dict(input_model.params) - - criterion = nn.CrossEntropyLoss() - optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) - - # (optional) use GPU to speed things up - net.to(DEVICE) - # (optional) calculate total steps - steps = epochs * len(trainloader) - for epoch in range(epochs): # loop over the dataset multiple times - - running_loss = 0.0 - for i, data in enumerate(trainloader, 0): - # get the inputs; data is a list of [inputs, labels] - # (optional) use GPU to speed things up - inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) - - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = net(inputs) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # print statistics - running_loss += loss.item() - if i % 2000 == 1999: # print every 2000 mini-batches - print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") - global_step = input_model.current_round * steps + epoch * len(trainloader) + i - - summary_writer.add_scalar(tag="loss_for_each_batch", scalar=running_loss, global_step=global_step) - running_loss = 0.0 - - print("Finished Training") - - PATH = "./cifar_net.pth" - torch.save(net.state_dict(), PATH) - - # (5) wraps evaluation logic into a method to re-use for - # evaluation on both trained and received model - def evaluate(input_weights): - net = Net() - net.load_state_dict(input_weights) - # (optional) use GPU to speed things up - net.to(DEVICE) - - correct = 0 - total = 0 - # since we're not training, we don't need to calculate the gradients for our outputs - with torch.no_grad(): - for data in testloader: - # (optional) use GPU to speed things up - images, labels = data[0].to(DEVICE), data[1].to(DEVICE) - # calculate outputs by running images through the network - outputs = net(images) - # the class with the highest energy is what we choose as prediction - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels).sum().item() - - print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") - return 100 * correct // total - - # (6) evaluate on received model for model selection - accuracy = evaluate(input_model.params) - # (7) construct trained FL model - output_model = flare.FLModel( - params=net.cpu().state_dict(), - metrics={"accuracy": accuracy}, - meta={"NUM_STEPS_CURRENT_ROUND": steps}, - ) - # (8) send model back to NVFlare - flare.send(output_model) - - -if __name__ == "__main__": - main() diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py deleted file mode 100644 index 7f7963a43c..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/cifar10_fl.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn as nn -import torch.optim as optim -import torchvision -import torchvision.transforms as transforms -from net import Net - -# (1) import nvflare client API -import nvflare.client as flare - -# (optional) metrics -from nvflare.client.tracking import SummaryWriter - -# (optional) set a fix place so we don't need to download everytime -DATASET_PATH = "/tmp/nvflare/data" -# (optional) We change to use GPU to speed things up. -# if you want to use CPU, change DEVICE="cpu" -DEVICE = "cuda:0" -DEVICE = "cpu" - - -def main(): - transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - - batch_size = 512 - epochs = 2 - - trainset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=True, download=True, transform=transform) - trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) - - testset = torchvision.datasets.CIFAR10(root=DATASET_PATH, train=False, download=True, transform=transform) - testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) - - net = Net() - - # (2) initializes NVFlare client API - flare.init() - - summary_writer = SummaryWriter() - while flare.is_running(): - # (3) receives FLModel from NVFlare - input_model = flare.receive() - print(f"current_round={input_model.current_round}") - - # (4) loads model from NVFlare - net.load_state_dict(input_model.params) - - criterion = nn.CrossEntropyLoss() - optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) - - # (optional) use GPU to speed things up - net.to(DEVICE) - # (optional) calculate total steps - steps = epochs * len(trainloader) - for epoch in range(epochs): # loop over the dataset multiple times - - running_loss = 0.0 - for i, data in enumerate(trainloader, 0): - # get the inputs; data is a list of [inputs, labels] - # (optional) use GPU to speed things up - inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) - - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = net(inputs) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # print statistics - running_loss += loss.item() - if i % 2000 == 1999: # print every 2000 mini-batches - print(f"[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") - global_step = input_model.current_round * steps + epoch * len(trainloader) + i - - summary_writer.add_scalar(tag="loss_for_each_batch", scalar=running_loss, global_step=global_step) - running_loss = 0.0 - - print(f"Finished Training for round {input_model.current_round}") - - PATH = "./cifar_net.pth" - torch.save(net.state_dict(), PATH) - - # (5) wraps evaluation logic into a method to re-use for - # evaluation on both trained and received model - def evaluate(input_weights): - net = Net() - net.load_state_dict(input_weights) - # (optional) use GPU to speed things up - net.to(DEVICE) - - correct = 0 - total = 0 - # since we're not training, we don't need to calculate the gradients for our outputs - with torch.no_grad(): - for data in testloader: - # (optional) use GPU to speed things up - images, labels = data[0].to(DEVICE), data[1].to(DEVICE) - # calculate outputs by running images through the network - outputs = net(images) - # the class with the highest energy is what we choose as prediction - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels).sum().item() - - print(f"Accuracy of the network on the 10000 test images: {100 * correct // total} %") - return 100 * correct // total - - # (6) evaluate on received model for model selection - accuracy = evaluate(input_model.params) - # (7) construct trained FL model - output_model = flare.FLModel( - params=net.cpu().state_dict(), - metrics={"accuracy": accuracy}, - meta={"NUM_STEPS_CURRENT_ROUND": steps}, - ) - # (8) send model back to NVFlare - flare.send(output_model) - - -if __name__ == "__main__": - main() diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py new file mode 100644 index 0000000000..8a4bcfde27 --- /dev/null +++ b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py @@ -0,0 +1,188 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import torch +import torch.nn as nn +import torch.optim as optim +import torchvision +import torchvision.transforms as transforms +from net import Net + +# (1) import nvflare client API +import nvflare.client as flare +from nvflare.app_common.app_constant import ModelName + +# (optional) set a fix place so we don't need to download everytime +CIFAR10_ROOT = "/tmp/nvflare/data/cifar10" +# (optional) We change to use GPU to speed things up. +# if you want to use CPU, change DEVICE="cpu" +DEVICE = "cuda:0" + + +def define_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--dataset_path", type=str, default=CIFAR10_ROOT, nargs="?") + parser.add_argument("--batch_size", type=int, default=4, nargs="?") + parser.add_argument("--num_workers", type=int, default=1, nargs="?") + parser.add_argument("--local_epochs", type=int, default=2, nargs="?") + parser.add_argument("--model_path", type=str, default=f"{CIFAR10_ROOT}/cifar_net.pth", nargs="?") + return parser.parse_args() + + +def main(): + # define local parameters + args = define_parser() + + dataset_path = args.dataset_path + batch_size = args.batch_size + num_workers = args.num_workers + local_epochs = args.local_epochs + model_path = args.model_path + + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + trainset = torchvision.datasets.CIFAR10(root=dataset_path, train=True, download=True, transform=transform) + trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + testset = torchvision.datasets.CIFAR10(root=dataset_path, train=False, download=True, transform=transform) + testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=num_workers) + + net = Net() + best_accuracy = 0.0 + + # wraps evaluation logic into a method to re-use for + # evaluation on both trained and received model + def evaluate(input_weights): + net = Net() + net.load_state_dict(input_weights) + # (optional) use GPU to speed things up + net.to(DEVICE) + + correct = 0 + total = 0 + # since we're not training, we don't need to calculate the gradients for our outputs + with torch.no_grad(): + for data in testloader: + # (optional) use GPU to speed things up + images, labels = data[0].to(DEVICE), data[1].to(DEVICE) + # calculate outputs by running images through the network + outputs = net(images) + # the class with the highest energy is what we choose as prediction + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + return 100 * correct // total + + # (2) initialize NVFlare client API + flare.init() + + # (3) run continously when launch_once=true + while flare.is_running(): + + # (4) receive FLModel from NVFlare + input_model = flare.receive() + client_id = flare.get_site_name() + + # Based on different "task" we will do different things + # for "train" task (flare.is_train()) we use the received model to do training and/or evaluation + # and send back updated model and/or evaluation metrics, if the "train_with_evaluation" is specified as True + # in the config_fed_client we will need to do evaluation and include the evaluation metrics + # for "evaluate" task (flare.is_evaluate()) we use the received model to do evaluation + # and send back the evaluation metrics + # for "submit_model" task (flare.is_submit_model()) we just need to send back the local model + # (5) performing train task on received model + if flare.is_train(): + print(f"({client_id}) current_round={input_model.current_round}, total_rounds={input_model.total_rounds}") + + # (5.1) loads model from NVFlare + net.load_state_dict(input_model.params) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) + + # (optional) use GPU to speed things up + net.to(DEVICE) + # (optional) calculate total steps + steps = local_epochs * len(trainloader) + for epoch in range(local_epochs): # loop over the dataset multiple times + + running_loss = 0.0 + for i, data in enumerate(trainloader, 0): + # get the inputs; data is a list of [inputs, labels] + # (optional) use GPU to speed things up + inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) + + # zero the parameter gradients + optimizer.zero_grad() + + # forward + backward + optimize + outputs = net(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # print statistics + running_loss += loss.item() + if i % 2000 == 1999: # print every 2000 mini-batches + print(f"({client_id}) [{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") + running_loss = 0.0 + + print(f"({client_id}) Finished Training") + + # (5.2) evaluation on local trained model to save best model + local_accuracy = evaluate(net.state_dict()) + print(f"({client_id}) Evaluating local trained model. Accuracy on the 10000 test images: {local_accuracy}") + if local_accuracy > best_accuracy: + best_accuracy = local_accuracy + torch.save(net.state_dict(), model_path) + + # (5.3) evaluate on received model for model selection + accuracy = evaluate(input_model.params) + print( + f"({client_id}) Evaluating received model for model selection. Accuracy on the 10000 test images: {accuracy}" + ) + + # (5.4) construct trained FL model + output_model = flare.FLModel( + params=net.cpu().state_dict(), + metrics={"accuracy": accuracy}, + meta={"NUM_STEPS_CURRENT_ROUND": steps}, + ) + + # (5.5) send model back to NVFlare + flare.send(output_model) + + # (6) performing evaluate task on received model + elif flare.is_evaluate(): + accuracy = evaluate(input_model.params) + flare.send(flare.FLModel(metrics={"accuracy": accuracy})) + + # (7) performing submit_model task to obtain best local model + elif flare.is_submit_model(): + model_name = input_model.meta["submit_model_name"] + if model_name == ModelName.BEST_MODEL: + try: + weights = torch.load(model_path) + net = Net() + net.load_state_dict(weights) + flare.send(flare.FLModel(params=net.cpu().state_dict())) + except Exception as e: + raise ValueError("Unable to load best model") from e + else: + raise ValueError(f"Unknown model_type: {model_name}") + + +if __name__ == "__main__": + main() diff --git a/examples/hello-world/hello-fedavg/requirements.txt b/examples/hello-world/hello-fedavg/requirements.txt index e69de29bb2..265102f82c 100644 --- a/examples/hello-world/hello-fedavg/requirements.txt +++ b/examples/hello-world/hello-fedavg/requirements.txt @@ -0,0 +1,3 @@ +nvflare~=2.4.0rc +torch +torchvision diff --git a/job_templates/sag_pt_wf_controller/config_fed_client.conf b/job_templates/sag_pt_wf_controller/config_fed_client.conf new file mode 100644 index 0000000000..bd3ad468bb --- /dev/null +++ b/job_templates/sag_pt_wf_controller/config_fed_client.conf @@ -0,0 +1,116 @@ +{ + # version of the configuration + format_version = 2 + + # This is the application script which will be invoked. Client can replace this script with user's own training script. + app_script = "train.py" + + # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. + app_config = "" + + # Client Computing Executors. + executors = [ + { + # tasks the executors are defined to handle + tasks = ["train"] + + # This particular executor + executor { + + # This is an executor for Client API. The underline data exchange is using Pipe. + path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" + + args { + # launcher_id is used to locate the Launcher object in "components" + launcher_id = "launcher" + + # pipe_id is used to locate the Pipe object in "components" + pipe_id = "pipe" + + # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. + # Please refer to the class docstring for all available arguments + heartbeat_timeout = 60 + + # format of the exchange parameters + params_exchange_format = "pytorch" + + # if the transfer_type is FULL, then it will be sent directly + # if the transfer_type is DIFF, then we will calculate the + # difference VS received parameters and send the difference + params_transfer_type = "DIFF" + + # if train_with_evaluation is true, the executor will expect + # the custom code need to send back both the trained parameters and the evaluation metric + # otherwise only trained parameters are expected + train_with_evaluation = true + } + } + } + ], + + # this defined an array of task data filters. If provided, it will control the data from server controller to client executor + task_data_filters = [] + + # this defined an array of task result filters. If provided, it will control the result from client executor to server controller + task_result_filters = [] + + components = [ + { + # component id is "launcher" + id = "launcher" + + # the class path of this component + path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" + + args { + # the launcher will invoke the script + script = "python3 custom/{app_script} {app_config} " + # if launch_once is true, the SubprocessLauncher will launch once for the whole job + # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server + launch_once = true + } + } + { + id = "pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + } + { + id = "metrics_pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + }, + { + id = "metric_relay" + path = "nvflare.app_common.widgets.metric_relay.MetricRelay" + args { + pipe_id = "metrics_pipe" + event_type = "fed.analytix_log_stats" + # how fast should it read from the peer + read_interval = 0.1 + } + }, + { + # we use this component so the client api `flare.init()` can get required information + id = "config_preparer" + path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" + args { + component_ids = ["metric_relay"] + } + } + ] +} diff --git a/job_templates/sag_pt_wf_controller/config_fed_server.conf b/job_templates/sag_pt_wf_controller/config_fed_server.conf new file mode 100644 index 0000000000..1e35255c83 --- /dev/null +++ b/job_templates/sag_pt_wf_controller/config_fed_server.conf @@ -0,0 +1,41 @@ +{ + # version of the configuration + format_version = 2 + task_data_filters =[] + task_result_filters = [] + + workflows = [ + { + id = "fed_avg" + # PTFedAvg is a WFController. If the controller is a type WFController, a WFCommunicator will automatically be configured. + path = "fedavg_pt.PTFedAvg" + args { + min_clients = 2 + num_rounds = 2 + output_path = "/tmp/nvflare/fedavg/mode.pth" + # stop_cond = "accuracy >= 55" + } + } + # { + # # If using a WFController, can optionally pair with a `communicator` section (will be defaulted to WFCommunicator if not specified) + # communicator { + # path = "nvflare.app_common.wf_comm.wf_communicator.WFCommunicator" + # args = {} + # } + + # # If using a WFController, can be set under `controller` section if wanting to configure to pair with a communicator + # controller { + # path = "fedavg_pt.PTFedAvg" + # args { + # min_clients = 2 + # num_rounds = 2 + # output_path = "/tmp/nvflare/fedavg/mode.pth" + # # stop_cond = "accuracy >= 55" + # } + # } + # } + ] + + components = [ + ] +} \ No newline at end of file diff --git a/job_templates/sag_pt_wf_controller/info.conf b/job_templates/sag_pt_wf_controller/info.conf new file mode 100644 index 0000000000..5d384b8697 --- /dev/null +++ b/job_templates/sag_pt_wf_controller/info.conf @@ -0,0 +1,5 @@ +{ + description = "scatter & gather workflow using pytorch with wf controller" + execution_api_type = "client_api" + controller_type = "server" +} \ No newline at end of file diff --git a/job_templates/sag_pt_wf_controller/info.md b/job_templates/sag_pt_wf_controller/info.md new file mode 100644 index 0000000000..32cc9fcac7 --- /dev/null +++ b/job_templates/sag_pt_wf_controller/info.md @@ -0,0 +1,11 @@ +# Job Template Information Card + +## sag_pt_wf_controller + name = "sag_pt_wf_controller" + description = "Scatter and Gather Workflow using pytorch with wf controller" + class_name = "ScatterAndGather" + controller_type = "server" + executor_type = "launcher_executor" + contributor = "NVIDIA" + init_publish_date = "2024-02-08" + last_updated_date = "2024-02-08" # yyyy-mm-dd diff --git a/job_templates/sag_pt_wf_controller/meta.conf b/job_templates/sag_pt_wf_controller/meta.conf new file mode 100644 index 0000000000..cbbd7a293c --- /dev/null +++ b/job_templates/sag_pt_wf_controller/meta.conf @@ -0,0 +1,10 @@ +{ + name = "sag_pt_wf_controller" + resource_spec = {} + deploy_map { + # change deploy map as needed. + app = ["@ALL"] + } + min_clients = 2 + mandatory_clients = [] +} diff --git a/nvflare/app_common/app_constant.py b/nvflare/app_common/app_constant.py index eaa14f54f2..73928fd95b 100644 --- a/nvflare/app_common/app_constant.py +++ b/nvflare/app_common/app_constant.py @@ -212,10 +212,3 @@ class PSIConst(AppConstants): REQUEST_MSG = "PSI_REQUEST_MSG" REQUEST_MSG_SET = "PSI_REQUEST_MSG_SET" RESPONSE_MSG = "PSI_RESPONSE_MSG" - - -class CommConstants(object): - COMMUNICATOR = "communicator" - CONTROLLER = "controller" - TASK_RESULT = "TASK_RESULT" - POST_PROCESS_RESULT = "POST_PROCESS_RESULT" diff --git a/nvflare/app_common/wf_comm/base_wf_comm.py b/nvflare/app_common/wf_comm/base_wf_communicator.py similarity index 97% rename from nvflare/app_common/wf_comm/base_wf_comm.py rename to nvflare/app_common/wf_comm/base_wf_communicator.py index d7ae1147ec..e1c6d2584d 100644 --- a/nvflare/app_common/wf_comm/base_wf_comm.py +++ b/nvflare/app_common/wf_comm/base_wf_communicator.py @@ -22,7 +22,7 @@ from nvflare.apis.fl_context import FLContext from nvflare.apis.shareable import Shareable from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.app_constant import AppConstants, CommConstants +from nvflare.app_common.app_constant import AppConstants from nvflare.app_common.app_event_type import AppEventType from nvflare.app_common.utils.fl_model_utils import FLModelUtils from nvflare.app_common.wf_comm.decomposer_register import DecomposerRegister @@ -40,6 +40,7 @@ from nvflare.app_common.workflows.error_handle_utils import ABORT_WHEN_IN_ERROR from nvflare.fuel.data_event.data_bus import DataBus from nvflare.fuel.data_event.event_manager import EventManager +from nvflare.private.defs import CommConstants from nvflare.security.logging import secure_format_traceback @@ -50,7 +51,7 @@ def __init__( result_pull_interval: float = 0.2, ): super().__init__() - self.strategy_fn_name = "run" + self.wf_controller_fn_name = "run" self.clients = None self.task_timeout = task_timeout @@ -91,7 +92,7 @@ def publish_comm_api(self): def start_workflow(self, abort_signal, fl_ctx): try: fl_ctx.set_prop("abort_signal", abort_signal) - func = getattr(self.get_controller(), self.strategy_fn_name) + func = getattr(self.get_controller(), self.wf_controller_fn_name) func() except Exception as e: diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index 357a1fa295..a8d8615e04 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -20,16 +20,13 @@ from nvflare.apis.controller_spec import SendOrder from nvflare.apis.fl_constant import ReturnCode from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.app_constant import CommConstants +from nvflare.app_common.app_constant import AppConstants from nvflare.app_common.wf_comm.wf_comm_api_spec import ( - CURRENT_ROUND, DATA, MIN_RESPONSES, - NUM_ROUNDS, RESP_MAX_WAIT_TIME, RESULT, SITE_NAMES, - START_ROUND, STATUS, TARGET_SITES, TASK_NAME, @@ -37,6 +34,7 @@ ) from nvflare.fuel.data_event.data_bus import DataBus from nvflare.fuel.data_event.event_manager import EventManager +from nvflare.private.defs import CommConstants class WFCommAPI(WFCommAPISpec): @@ -216,9 +214,9 @@ def _prepare_input_payload(self, task_name, data, meta, min_responses, targets): current_round = data.current_round num_rounds = data.total_rounds else: - start_round = meta.get(START_ROUND, 0) - current_round = meta.get(CURRENT_ROUND, 0) - num_rounds = meta.get(NUM_ROUNDS, 1) + start_round = meta.get(AppConstants.START_ROUND, 0) + current_round = meta.get(AppConstants.CURRENT_ROUND, 0) + num_rounds = meta.get(AppConstants.NUM_ROUNDS, 1) resp_max_wait_time = meta.get(RESP_MAX_WAIT_TIME, 15) @@ -226,9 +224,9 @@ def _prepare_input_payload(self, task_name, data, meta, min_responses, targets): TASK_NAME: task_name, MIN_RESPONSES: min_responses, RESP_MAX_WAIT_TIME: resp_max_wait_time, - CURRENT_ROUND: current_round, - NUM_ROUNDS: num_rounds, - START_ROUND: start_round, + AppConstants.CURRENT_ROUND: current_round, + AppConstants.NUM_ROUNDS: num_rounds, + AppConstants.START_ROUND: start_round, DATA: data, TARGET_SITES: targets, } diff --git a/nvflare/app_common/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/wf_comm/wf_comm_api_spec.py index 5f82be0e8e..eb64293da8 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/wf_comm/wf_comm_api_spec.py @@ -18,14 +18,8 @@ SITE_NAMES = "SITE_NAMES" TASK_NAME = "TASK_NAME" -# note same as app_constant constant (todo: we only need one constant definition) MIN_RESPONSES = "min_responses" RESP_MAX_WAIT_TIME = "resp_max_wait_time" -START_ROUND = "start_round" -CURRENT_ROUND = "current_round" -CONTRIBUTION_ROUND = "contribution_round" -CONTRIBUTION_CLIENT = "contribution_client" -NUM_ROUNDS = "num_rounds" STATUS = "status" RESULT = "result" diff --git a/nvflare/app_common/wf_comm/wf_communicator.py b/nvflare/app_common/wf_comm/wf_communicator.py index 61bf60a677..f52c3188dc 100644 --- a/nvflare/app_common/wf_comm/wf_communicator.py +++ b/nvflare/app_common/wf_comm/wf_communicator.py @@ -15,12 +15,13 @@ from nvflare.apis.fl_context import FLContext from nvflare.apis.impl.controller import Controller from nvflare.apis.signal import Signal -from nvflare.app_common.wf_comm.base_wf_comm import BaseWFCommunicator +from nvflare.app_common.wf_comm.base_wf_communicator import BaseWFCommunicator class WFCommunicator(BaseWFCommunicator, Controller): def __init__(self): super().__init__() + self.register_serializers() def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.start_workflow(abort_signal, fl_ctx) diff --git a/nvflare/app_common/wf_comm/wf_communicator_spec.py b/nvflare/app_common/wf_comm/wf_communicator_spec.py index ef724a2a73..c353b7e13e 100644 --- a/nvflare/app_common/wf_comm/wf_communicator_spec.py +++ b/nvflare/app_common/wf_comm/wf_communicator_spec.py @@ -64,7 +64,7 @@ def get_controller(self): if isinstance(self.controller_config, dict): controller = ComponentBuilder().build_component(self.controller_config) if controller is None: - raise ValueError("strategy should provided, but get None") + raise ValueError("wf_controller should provided, but get None") return controller diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py b/nvflare/app_common/workflows/fed_avg.py similarity index 95% rename from examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py rename to nvflare/app_common/workflows/fed_avg.py index 691e011f5a..3a0bf65061 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg.py +++ b/nvflare/app_common/workflows/fed_avg.py @@ -13,10 +13,10 @@ # limitations under the License. import logging -import sys from typing import Callable, Dict, Optional from net import Net + from nvflare.apis.wf_controller import WFController from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper @@ -32,13 +32,13 @@ class FedAvg(WFController): def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - resp_max_wait_time: float = 5, + self, + min_clients: int, + num_rounds: int, + output_path: str, + start_round: int = 1, + stop_cond: str = None, + resp_max_wait_time: float = 5, ): super(FedAvg, self).__init__() @@ -196,7 +196,7 @@ def select_best_model(self, curr_model: FLModel): if self.stop_criteria: metric, _, op_fn = self.stop_criteria self.logger.info("compare models") - if self.is_curr_mode_better(self.best_model, curr_model, metric, op_fn): + if self.is_curr_model_better(self.best_model, curr_model, metric, op_fn): self.best_model = curr_model else: self.best_model = curr_model @@ -217,8 +217,8 @@ def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[st return op_fn(value, target) - def is_curr_mode_better( - self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable + def is_curr_model_better( + self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable ) -> bool: curr_metrics = curr_model.metrics if curr_metrics is None: diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py b/nvflare/app_common/workflows/fed_avg_pt.py similarity index 95% rename from examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py rename to nvflare/app_common/workflows/fed_avg_pt.py index 8a18abdd08..0f76ebfc3f 100644 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/fedavg_pt.py +++ b/nvflare/app_common/workflows/fed_avg_pt.py @@ -14,9 +14,9 @@ import os import torch -from fedavg import FedAvg from nvflare.app_common.abstract.fl_model import FLModel +from nvflare.app_common.workflows.fed_avg import FedAvg class PTFedAvg(FedAvg): diff --git a/nvflare/private/defs.py b/nvflare/private/defs.py index 6ae5c7fd8c..9318866016 100644 --- a/nvflare/private/defs.py +++ b/nvflare/private/defs.py @@ -181,6 +181,13 @@ class CellMessageHeaderKeys: ABORT_JOBS = "abort_jobs" +class CommConstants(object): + COMMUNICATOR = "communicator" + CONTROLLER = "controller" + TASK_RESULT = "TASK_RESULT" + POST_PROCESS_RESULT = "POST_PROCESS_RESULT" + + class JobFailureMsgKey: JOB_ID = "job_id" diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index ed6c1c317d..e8346a9411 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -17,14 +17,16 @@ from nvflare.apis.fl_component import FLComponent from nvflare.apis.fl_constant import SystemConfigs, SystemVarName from nvflare.apis.responder import Responder -from nvflare.app_common.app_constant import CommConstants +from nvflare.apis.wf_controller import WFController from nvflare.app_common.wf_comm.wf_communicator import WFCommunicator from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec from nvflare.fuel.data_event.data_bus import DataBus from nvflare.fuel.utils.argument_utils import parse_vars +from nvflare.fuel.utils.class_utils import instantiate_class from nvflare.fuel.utils.component_builder import ComponentBuilder from nvflare.fuel.utils.config_service import ConfigService from nvflare.fuel.utils.json_scanner import Node +from nvflare.private.defs import CommConstants from nvflare.private.fed_json_config import FedJsonConfigurator from nvflare.private.json_configer import ConfigContext, ConfigError @@ -35,34 +37,34 @@ class WorkFlow: - def __init__(self, id, responder: Responder, strategy=None): + def __init__(self, id, responder: Responder, wf_controller=None): """Workflow is a responder with ID. Args: id: identification responder (Responder): A responder or communicator - strategy: federated learning strategy. If None, the responder will implement the strategy + wf_controller: federated learning wf_controller. If None, the responder will implement the wf_controller """ self.id = id self.responder = responder - self.strategy = strategy + self.wf_controller = wf_controller -def enhance_workflow_config(element: dict): +def enhance_workflow_config(element: dict, class_path: str): if CommConstants.CONTROLLER in element: controller_config = element.get(CommConstants.CONTROLLER) controller_config["lazy_instantiate"] = True element[CommConstants.CONTROLLER] = controller_config - elif CommConstants.COMMUNICATOR not in element: - controller_config = element.copy() - controller_config["lazy_instantiate"] = True - element = {CommConstants.CONTROLLER: controller_config} - else: + elif CommConstants.COMMUNICATOR in element: wf_config = element.copy() comm_config = wf_config.pop(CommConstants.COMMUNICATOR) controller_config = wf_config controller_config["lazy_instantiate"] = True element = {CommConstants.COMMUNICATOR: comm_config, CommConstants.CONTROLLER: controller_config} + elif isinstance(instantiate_class(class_path, element.get("args", dict())), WFController): + controller_config = element.copy() + controller_config["lazy_instantiate"] = True + element = {CommConstants.CONTROLLER: controller_config} return element @@ -150,35 +152,11 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): return if re.search(r"^workflows\.#[0-9]+$", path): - element = enhance_workflow_config(element) + class_path = self.get_class_path(element) + element = enhance_workflow_config(element, class_path) - print("\n\n element =", element) component = self.authorize_and_build_component(element, config_ctx, node) - - # todo: fix: dependency graph is not right, we are now nvflare.private depending on the AppCommon class - # todo: refactoring the code into small methods - if isinstance(component, dict): - wf_config = component - communicator = wf_config.get(CommConstants.COMMUNICATOR) - if communicator is None: - communicator = WFCommunicator() - - if isinstance(communicator, WFCommunicatorSpec): - controller_config = wf_config.get(CommConstants.CONTROLLER) - controller_config["lazy_instantiate"] = False - communicator.set_controller_config(controller_config) - data_bus = DataBus() - data_bus.send_data(CommConstants.COMMUNICATOR, communicator) - responder = communicator - else: - responder = component - - if not isinstance(responder, Responder): - raise ConfigError( - '"workflow" must be a Responder or Controller or has a Responder object, but got {}'.format( - type(responder) - ) - ) + responder = self.get_responder(component) cid = element.get("id", None) if not cid: @@ -198,18 +176,43 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): self.components[cid] = responder return - def get_strategy(self, wf_config): - strategy_comp = wf_config.get("strategy") - strategy_comp["lazy_instantiate"] = False - if isinstance(strategy_comp, dict): - strategy = ComponentBuilder().build_component(strategy_comp) + def get_responder(self, component): + if isinstance(component, dict): + wf_config = component + communicator = wf_config.get(CommConstants.COMMUNICATOR) + if communicator is None: + communicator = WFCommunicator() + + if isinstance(communicator, WFCommunicatorSpec): + controller_config = wf_config.get(CommConstants.CONTROLLER) + controller_config["lazy_instantiate"] = False + communicator.set_controller_config(controller_config) + data_bus = DataBus() + data_bus.send_data(CommConstants.COMMUNICATOR, communicator) + responder = communicator + else: + responder = component + + if not isinstance(responder, Responder): + raise ConfigError( + '"workflow" must be a Responder or Controller or has a Responder object, but got {}'.format( + type(responder) + ) + ) + return responder + + def get_wf_controller(self, wf_config): + wf_controller_comp = wf_config.get("wf_controller") + wf_controller_comp["lazy_instantiate"] = False + if isinstance(wf_controller_comp, dict): + wf_controller = ComponentBuilder().build_component(wf_controller_comp) else: - strategy = strategy_comp + wf_controller = wf_controller_comp - if strategy is None: - raise ValueError("strategy should provided, but get None") + if wf_controller is None: + raise ValueError("wf_controller should provided, but get None") - return strategy + return wf_controller def _get_all_workflows_ids(self): ids = [] From 5b8420c2f8216a3834e84f2b11cfa43d4701bbb1 Mon Sep 17 00:00:00 2001 From: Sean Yang Date: Tue, 13 Feb 2024 16:06:46 -0800 Subject: [PATCH 38/41] fix format --- nvflare/fuel/data_event/data_bus.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nvflare/fuel/data_event/data_bus.py b/nvflare/fuel/data_event/data_bus.py index 7b60f08dbd..ef1fd4a4be 100644 --- a/nvflare/fuel/data_event/data_bus.py +++ b/nvflare/fuel/data_event/data_bus.py @@ -82,7 +82,6 @@ def publish(self, topics: List[str], datum: Any) -> None: executor.submit(callback, topic, datum, self) executor.shutdown() - def put_data(self, key: Any, datum: Any) -> None: """ Store a data associated with a key and topic. From a5f63ada0da69c608e804e792fa950294d222a93 Mon Sep 17 00:00:00 2001 From: Sean Yang Date: Wed, 14 Feb 2024 15:31:07 -0800 Subject: [PATCH 39/41] databus updates --- nvflare/app_common/wf_comm/__init__.py | 2 +- nvflare/app_common/wf_comm/base_wf_communicator.py | 2 +- nvflare/app_common/wf_comm/wf_comm_api.py | 2 +- nvflare/private/fed/server/server_json_config.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nvflare/app_common/wf_comm/__init__.py b/nvflare/app_common/wf_comm/__init__.py index 3291c2225f..d4658d3acf 100644 --- a/nvflare/app_common/wf_comm/__init__.py +++ b/nvflare/app_common/wf_comm/__init__.py @@ -19,4 +19,4 @@ def get_wf_comm_api() -> WFCommAPISpec: - return data_bus.receive_data("wf_comm_api") + return data_bus.get_data("wf_comm_api") diff --git a/nvflare/app_common/wf_comm/base_wf_communicator.py b/nvflare/app_common/wf_comm/base_wf_communicator.py index e1c6d2584d..30f204ecc2 100644 --- a/nvflare/app_common/wf_comm/base_wf_communicator.py +++ b/nvflare/app_common/wf_comm/base_wf_communicator.py @@ -87,7 +87,7 @@ def register_decomposers(self): def publish_comm_api(self): comm_api = WFCommAPI() comm_api.meta.update({SITE_NAMES: self.get_site_names()}) - self.data_bus.send_data("wf_comm_api", comm_api) + self.data_bus.put_data("wf_comm_api", comm_api) def start_workflow(self, abort_signal, fl_ctx): try: diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index a8d8615e04..5d8de3d076 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -49,7 +49,7 @@ def __init__(self): data_bus.subscribe(topics=[CommConstants.TASK_RESULT], callback=self.result_callback) self.event_manager = EventManager(data_bus) - self.ctrl = data_bus.receive_data(CommConstants.COMMUNICATOR) + self.ctrl = data_bus.get_data(CommConstants.COMMUNICATOR) self._check_inputs() def get_site_names(self): diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index e8346a9411..8dcb97896e 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -188,7 +188,7 @@ def get_responder(self, component): controller_config["lazy_instantiate"] = False communicator.set_controller_config(controller_config) data_bus = DataBus() - data_bus.send_data(CommConstants.COMMUNICATOR, communicator) + data_bus.put_data(CommConstants.COMMUNICATOR, communicator) responder = communicator else: responder = component From da30df2c063b0aaf2e86b507e8e2dbcba26d2e40 Mon Sep 17 00:00:00 2001 From: Sean Yang Date: Thu, 22 Feb 2024 11:27:56 -0800 Subject: [PATCH 40/41] add docstrings, address comments --- examples/hello-world/hello-fedavg/README.md | 172 ---------------- .../fedavg/app/config/config_fed_client.conf | 77 ------- .../fedavg/app/config/config_fed_server.conf | 22 -- .../jobs/fedavg/app/custom/net.py | 37 ---- .../jobs/fedavg/app/custom/train.py | 188 ------------------ .../hello-fedavg/jobs/fedavg/meta.conf | 7 - .../hello-world/hello-fedavg/requirements.txt | 3 - .../config_fed_server.conf | 4 +- nvflare/apis/wf_controller.py | 10 + .../app_common/executors/launcher_executor.py | 1 + nvflare/app_common/utils/fl_model_utils.py | 2 - nvflare/app_common/utils/math_utils.py | 4 +- nvflare/app_common/wf_comm/__init__.py | 2 +- .../wf_comm/base_wf_communicator.py | 37 +++- nvflare/app_common/wf_comm/wf_comm_api.py | 16 +- .../app_common/wf_comm/wf_comm_api_spec.py | 104 +++++++++- nvflare/app_common/wf_comm/wf_communicator.py | 2 +- .../wf_comm/wf_communicator_spec.py | 73 ++++--- .../workflows/error_handle_utils.py | 23 --- nvflare/app_common/workflows/fed_avg.py | 18 +- .../workflows => app_opt/pt}/fed_avg_pt.py | 2 +- .../app_common/utils/math_utils_test.py | 47 +++++ 22 files changed, 253 insertions(+), 598 deletions(-) delete mode 100644 examples/hello-world/hello-fedavg/README.md delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py delete mode 100644 examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf delete mode 100644 examples/hello-world/hello-fedavg/requirements.txt delete mode 100644 nvflare/app_common/workflows/error_handle_utils.py rename nvflare/{app_common/workflows => app_opt/pt}/fed_avg_pt.py (95%) create mode 100644 tests/unit_test/app_common/utils/math_utils_test.py diff --git a/examples/hello-world/hello-fedavg/README.md b/examples/hello-world/hello-fedavg/README.md deleted file mode 100644 index 76d43f16ed..0000000000 --- a/examples/hello-world/hello-fedavg/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# FedAvg: simplified - -This example illustrates How to use the new Workflow Communication API to contract a workflow: no need to write a controller. - -## FLARE Workflow Communicator API - -The Flare workflow Communicator API only has small set methods - -``` - -class WFCommAPISpec(ABC): - @abstractmethod - def broadcast_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def send_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def relay_and_wait(self, msg_payload: Dict): - pass - - @abstractmethod - def broadcast(self, msg_payload: Dict): - pass - - @abstractmethod - def send(self, msg_payload: Dict): - pass - - @abstractmethod - def relay(self, msg_payload: Dict): - pass - - @abstractmethod - def get_site_names(self) -> List[str]: - pass - - @abstractmethod - def wait_all(self, min_responses: int, resp_max_wait_time: Optional[float]) -> Dict[str, Dict[str, FLModel]]: - pass - - @abstractmethod - def wait_one(self, resp_max_wait_time: Optional[float] = None) -> Tuple[str, str, FLModel]: - pass - -``` - - -## Writing a new Workflow - -With this new API writing the new workflow is really simple: - -* Workflow (Server) - -``` -from nvflare.app_common.workflows import wf_comm as flare - -class FedAvg: - def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - model_selection_rule: str = None, - ): - super(FedAvg, self).__init__() - - - - self.flare_comm = flare.get_wf_comm_api() - - def run(self): - self.logger.info("start Fed Avg Workflow\n \n") - - start = self.start_round - end = self.start_round + self.num_rounds - - model = self.init_model() - for current_round in range(start, end): - - self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") - self.current_round = current_round - - sag_results = self.scatter_and_gather(model, current_round) - - aggr_result = self.aggr_fn(sag_results) - - self.logger.info(f"aggregate metrics = {aggr_result.metrics}") - - model = update_model(model, aggr_result) - - self.select_best_model(model) - - self.save_model(self.best_model, self.output_path) - - self.logger.info("end Fed Avg Workflow\n \n") - - -``` -Scatter and Gather (SAG): - -SAG is simply ask WFController to broadcast the model to all clients - -``` - def scatter_and_gather(self, model: FLModel, current_round): - msg_payload = {"min_responses": self.min_clients, - "current_round": current_round, - "num_round": self.num_rounds, - "start_round": self.start_round, - "data": model} - - # (2) broadcast and wait - results = self.flare_comm.broadcast_and_wait(msg_payload) - return results -``` - -## Configurations - -### client-side configuration - -This is the same as FLARE Client API configuration - -### server-side configuration - - Server side controller is really simple, all we need is to use WFController with newly defined workflow class - - -``` -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "fed_avg" - path = "nvflare.app_opt.pt.wf_controller.PTWFController" - args { - comm_msg_pull_interval = 5 - task_name = "train" - wf_class_path = "fedavg_pt.PTFedAvg", - wf_args { - min_clients = 2 - num_rounds = 10 - output_path = "/tmp/nvflare/fedavg/mode.pth" - stop_cond = "accuracy >= 55" - model_selection_rule = "accuracy >=" - } - } - } - ] - - components = [] - -} - -``` - - -## Run the job - -assume current working directory is at ```hello-fedavg``` directory - -``` -nvflare simulator -n 2 -t 2 jobs/fedavg -w /tmp/fedavg - -``` diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf deleted file mode 100644 index 74f9ab24b0..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_client.conf +++ /dev/null @@ -1,77 +0,0 @@ -{ - format_version = 2 - app_script = "train.py" - app_config = "" - executors = [ - { - tasks = [ - "train" - ] - executor { - path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" - args { - launcher_id = "launcher" - pipe_id = "pipe" - heartbeat_timeout = 60 - params_exchange_format = "pytorch" - params_transfer_type = "DIFF" - train_with_evaluation = true - } - } - } - ] - task_data_filters = [] - task_result_filters = [] - components = [ - { - id = "launcher" - path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" - args { - script = "python3 custom/{app_script} {app_config} " - launch_once = true - } - } - { - id = "pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - } - { - id = "metrics_pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - } - { - id = "metric_relay" - path = "nvflare.app_common.widgets.metric_relay.MetricRelay" - args { - pipe_id = "metrics_pipe" - event_type = "fed.analytix_log_stats" - read_interval = 0.1 - } - } - { - id = "config_preparer" - path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" - args { - component_ids = [ - "metric_relay" - ] - } - } - ] -} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf deleted file mode 100644 index 96ede86cea..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/config/config_fed_server.conf +++ /dev/null @@ -1,22 +0,0 @@ -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "fed_avg" - path = "nvflare.app_common.workflows.fed_avg_pt.PTFedAvg" - args { - min_clients = 2 - num_rounds = 2 - output_path = "/tmp/nvflare/fedavg/mode.pth" - # stop_cond = "accuracy >= 55" - } - } - ] - - components = [ - ] -} diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py deleted file mode 100644 index 031f84f432..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/net.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn as nn -import torch.nn.functional as F - - -class Net(nn.Module): - def __init__(self): - super().__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = torch.flatten(x, 1) # flatten all dimensions except batch - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return x diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py b/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py deleted file mode 100644 index 8a4bcfde27..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/app/custom/train.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - -import torch -import torch.nn as nn -import torch.optim as optim -import torchvision -import torchvision.transforms as transforms -from net import Net - -# (1) import nvflare client API -import nvflare.client as flare -from nvflare.app_common.app_constant import ModelName - -# (optional) set a fix place so we don't need to download everytime -CIFAR10_ROOT = "/tmp/nvflare/data/cifar10" -# (optional) We change to use GPU to speed things up. -# if you want to use CPU, change DEVICE="cpu" -DEVICE = "cuda:0" - - -def define_parser(): - parser = argparse.ArgumentParser() - parser.add_argument("--dataset_path", type=str, default=CIFAR10_ROOT, nargs="?") - parser.add_argument("--batch_size", type=int, default=4, nargs="?") - parser.add_argument("--num_workers", type=int, default=1, nargs="?") - parser.add_argument("--local_epochs", type=int, default=2, nargs="?") - parser.add_argument("--model_path", type=str, default=f"{CIFAR10_ROOT}/cifar_net.pth", nargs="?") - return parser.parse_args() - - -def main(): - # define local parameters - args = define_parser() - - dataset_path = args.dataset_path - batch_size = args.batch_size - num_workers = args.num_workers - local_epochs = args.local_epochs - model_path = args.model_path - - transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - trainset = torchvision.datasets.CIFAR10(root=dataset_path, train=True, download=True, transform=transform) - trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) - testset = torchvision.datasets.CIFAR10(root=dataset_path, train=False, download=True, transform=transform) - testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=num_workers) - - net = Net() - best_accuracy = 0.0 - - # wraps evaluation logic into a method to re-use for - # evaluation on both trained and received model - def evaluate(input_weights): - net = Net() - net.load_state_dict(input_weights) - # (optional) use GPU to speed things up - net.to(DEVICE) - - correct = 0 - total = 0 - # since we're not training, we don't need to calculate the gradients for our outputs - with torch.no_grad(): - for data in testloader: - # (optional) use GPU to speed things up - images, labels = data[0].to(DEVICE), data[1].to(DEVICE) - # calculate outputs by running images through the network - outputs = net(images) - # the class with the highest energy is what we choose as prediction - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels).sum().item() - - return 100 * correct // total - - # (2) initialize NVFlare client API - flare.init() - - # (3) run continously when launch_once=true - while flare.is_running(): - - # (4) receive FLModel from NVFlare - input_model = flare.receive() - client_id = flare.get_site_name() - - # Based on different "task" we will do different things - # for "train" task (flare.is_train()) we use the received model to do training and/or evaluation - # and send back updated model and/or evaluation metrics, if the "train_with_evaluation" is specified as True - # in the config_fed_client we will need to do evaluation and include the evaluation metrics - # for "evaluate" task (flare.is_evaluate()) we use the received model to do evaluation - # and send back the evaluation metrics - # for "submit_model" task (flare.is_submit_model()) we just need to send back the local model - # (5) performing train task on received model - if flare.is_train(): - print(f"({client_id}) current_round={input_model.current_round}, total_rounds={input_model.total_rounds}") - - # (5.1) loads model from NVFlare - net.load_state_dict(input_model.params) - - criterion = nn.CrossEntropyLoss() - optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) - - # (optional) use GPU to speed things up - net.to(DEVICE) - # (optional) calculate total steps - steps = local_epochs * len(trainloader) - for epoch in range(local_epochs): # loop over the dataset multiple times - - running_loss = 0.0 - for i, data in enumerate(trainloader, 0): - # get the inputs; data is a list of [inputs, labels] - # (optional) use GPU to speed things up - inputs, labels = data[0].to(DEVICE), data[1].to(DEVICE) - - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = net(inputs) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # print statistics - running_loss += loss.item() - if i % 2000 == 1999: # print every 2000 mini-batches - print(f"({client_id}) [{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}") - running_loss = 0.0 - - print(f"({client_id}) Finished Training") - - # (5.2) evaluation on local trained model to save best model - local_accuracy = evaluate(net.state_dict()) - print(f"({client_id}) Evaluating local trained model. Accuracy on the 10000 test images: {local_accuracy}") - if local_accuracy > best_accuracy: - best_accuracy = local_accuracy - torch.save(net.state_dict(), model_path) - - # (5.3) evaluate on received model for model selection - accuracy = evaluate(input_model.params) - print( - f"({client_id}) Evaluating received model for model selection. Accuracy on the 10000 test images: {accuracy}" - ) - - # (5.4) construct trained FL model - output_model = flare.FLModel( - params=net.cpu().state_dict(), - metrics={"accuracy": accuracy}, - meta={"NUM_STEPS_CURRENT_ROUND": steps}, - ) - - # (5.5) send model back to NVFlare - flare.send(output_model) - - # (6) performing evaluate task on received model - elif flare.is_evaluate(): - accuracy = evaluate(input_model.params) - flare.send(flare.FLModel(metrics={"accuracy": accuracy})) - - # (7) performing submit_model task to obtain best local model - elif flare.is_submit_model(): - model_name = input_model.meta["submit_model_name"] - if model_name == ModelName.BEST_MODEL: - try: - weights = torch.load(model_path) - net = Net() - net.load_state_dict(weights) - flare.send(flare.FLModel(params=net.cpu().state_dict())) - except Exception as e: - raise ValueError("Unable to load best model") from e - else: - raise ValueError(f"Unknown model_type: {model_name}") - - -if __name__ == "__main__": - main() diff --git a/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf b/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf deleted file mode 100644 index 1c27c4e99c..0000000000 --- a/examples/hello-world/hello-fedavg/jobs/fedavg/meta.conf +++ /dev/null @@ -1,7 +0,0 @@ -{ - name = "fedavg" - deploy_map { - app = ["@ALL"] - } - min_clients = 2 -} diff --git a/examples/hello-world/hello-fedavg/requirements.txt b/examples/hello-world/hello-fedavg/requirements.txt deleted file mode 100644 index 265102f82c..0000000000 --- a/examples/hello-world/hello-fedavg/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -nvflare~=2.4.0rc -torch -torchvision diff --git a/job_templates/sag_pt_wf_controller/config_fed_server.conf b/job_templates/sag_pt_wf_controller/config_fed_server.conf index 1e35255c83..1f2192897a 100644 --- a/job_templates/sag_pt_wf_controller/config_fed_server.conf +++ b/job_templates/sag_pt_wf_controller/config_fed_server.conf @@ -8,7 +8,7 @@ { id = "fed_avg" # PTFedAvg is a WFController. If the controller is a type WFController, a WFCommunicator will automatically be configured. - path = "fedavg_pt.PTFedAvg" + path = "nvflare.app_opt.pt.fedavg_pt.PTFedAvg" args { min_clients = 2 num_rounds = 2 @@ -25,7 +25,7 @@ # # If using a WFController, can be set under `controller` section if wanting to configure to pair with a communicator # controller { - # path = "fedavg_pt.PTFedAvg" + # path = "nvflare.app_opt.pt.fedavg_pt.PTFedAvg" # args { # min_clients = 2 # num_rounds = 2 diff --git a/nvflare/apis/wf_controller.py b/nvflare/apis/wf_controller.py index fde4968b47..175ee6827f 100644 --- a/nvflare/apis/wf_controller.py +++ b/nvflare/apis/wf_controller.py @@ -16,6 +16,16 @@ from nvflare.app_common import wf_comm +from .fl_constant import ReturnCode + +ABORT_WHEN_IN_ERROR = { + ReturnCode.EXECUTION_EXCEPTION: True, + ReturnCode.TASK_UNKNOWN: True, + ReturnCode.EXECUTION_RESULT_ERROR: False, + ReturnCode.TASK_DATA_FILTER_ERROR: True, + ReturnCode.TASK_RESULT_FILTER_ERROR: True, +} + class WFController(ABC): def __init__(self): diff --git a/nvflare/app_common/executors/launcher_executor.py b/nvflare/app_common/executors/launcher_executor.py index 64e3d7d946..6be65cbaee 100644 --- a/nvflare/app_common/executors/launcher_executor.py +++ b/nvflare/app_common/executors/launcher_executor.py @@ -258,6 +258,7 @@ def _execute_launcher_method_in_thread_executor(self, method_name: str, **kwargs future = self._thread_pool_executor.submit(getattr(self.launcher, method_name), **kwargs) result = future.result(timeout=self._launch_timeout) return result + except TimeoutError: self.log_warning( kwargs.get("fl_ctx"), diff --git a/nvflare/app_common/utils/fl_model_utils.py b/nvflare/app_common/utils/fl_model_utils.py index 486acec19b..7358070303 100644 --- a/nvflare/app_common/utils/fl_model_utils.py +++ b/nvflare/app_common/utils/fl_model_utils.py @@ -212,8 +212,6 @@ def update_model(model: FLModel, model_update: FLModel, replace_meta: bool = Tru else: model.meta.update(model_update.meta) - model.metrics = model_update.metrics - if model_update.params_type == ParamsType.FULL: model.params = model_update.params elif model_update.params_type == ParamsType.DIFF: diff --git a/nvflare/app_common/utils/math_utils.py b/nvflare/app_common/utils/math_utils.py index 6f5fb02240..3dde557a4c 100644 --- a/nvflare/app_common/utils/math_utils.py +++ b/nvflare/app_common/utils/math_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ def parse_compare_criteria(compare_expr: Optional[str] = None) -> Tuple[str, float, Callable]: """ Parse the compare expression into individual component - compare expression is in the format of string literal : " " such as accuracy >= 0.5 loss > 2.4 diff --git a/nvflare/app_common/wf_comm/__init__.py b/nvflare/app_common/wf_comm/__init__.py index d4658d3acf..c511d3e87b 100644 --- a/nvflare/app_common/wf_comm/__init__.py +++ b/nvflare/app_common/wf_comm/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nvflare/app_common/wf_comm/base_wf_communicator.py b/nvflare/app_common/wf_comm/base_wf_communicator.py index 30f204ecc2..e30c6f5071 100644 --- a/nvflare/app_common/wf_comm/base_wf_communicator.py +++ b/nvflare/app_common/wf_comm/base_wf_communicator.py @@ -21,6 +21,7 @@ from nvflare.apis.fl_constant import ReturnCode from nvflare.apis.fl_context import FLContext from nvflare.apis.shareable import Shareable +from nvflare.apis.wf_controller import ABORT_WHEN_IN_ERROR from nvflare.app_common.abstract.fl_model import FLModel from nvflare.app_common.app_constant import AppConstants from nvflare.app_common.app_event_type import AppEventType @@ -37,9 +38,12 @@ TASK_NAME, ) from nvflare.app_common.wf_comm.wf_communicator_spec import WFCommunicatorSpec -from nvflare.app_common.workflows.error_handle_utils import ABORT_WHEN_IN_ERROR from nvflare.fuel.data_event.data_bus import DataBus from nvflare.fuel.data_event.event_manager import EventManager +from nvflare.fuel.utils.class_utils import instantiate_class +from nvflare.fuel.utils.component_builder import ComponentBuilder +from nvflare.fuel.utils.fobs import fobs +from nvflare.fuel.utils.import_utils import optional_import from nvflare.private.defs import CommConstants from nvflare.security.logging import secure_format_traceback @@ -267,3 +271,34 @@ def handle_client_errors(self, rc: str, client_task: ClientTask, fl_ctx: FLConte self.log_error(fl_ctx, f"Execution failed for {client_task.client.name}") else: raise ValueError(f"Execution result is not received for {client_task.client.name}") + + def set_controller_config(self, controller_config: Dict): + if controller_config is None: + raise ValueError("controller_config is None") + + if not isinstance(controller_config, dict): + raise ValueError(f"controller_config should be Dict, found '{type(controller_config)}'") + + self.controller_config = controller_config + + def get_controller(self): + controller = None + if isinstance(self.controller_config, dict): + controller = ComponentBuilder().build_component(self.controller_config) + if controller is None: + raise ValueError("wf_controller should provided, but get None") + + return controller + + def register_serializers(self, serializer_class_paths: List[str] = None): + self.register_default_serializers() + if serializer_class_paths: + for class_path in serializer_class_paths: + fobs.register(instantiate_class(class_path, {})) + + def register_default_serializers(self): + torch, flag = optional_import("torch") + if flag: + from nvflare.app_opt.pt.decomposers import TensorDecomposer + + fobs.register(TensorDecomposer) diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index 5d8de3d076..2a1743c713 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -49,7 +49,7 @@ def __init__(self): data_bus.subscribe(topics=[CommConstants.TASK_RESULT], callback=self.result_callback) self.event_manager = EventManager(data_bus) - self.ctrl = data_bus.get_data(CommConstants.COMMUNICATOR) + self.comm = data_bus.get_data(CommConstants.COMMUNICATOR) self._check_inputs() def get_site_names(self): @@ -68,7 +68,7 @@ def broadcast_and_wait( meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) self.register_callback(callback) - self.ctrl.broadcast_to_peers_and_wait(msg_payload) + self.comm.broadcast_to_peers_and_wait(msg_payload) if callback is None: return self._get_results(task_name) @@ -95,7 +95,7 @@ def send_and_wait( if callback is not None: self.register_callback(callback) - self.ctrl.send_to_peers_and_wait(msg_payload, send_order) + self.comm.send_to_peers_and_wait(msg_payload, send_order) if callback is not None: return self._get_results(task_name) @@ -116,7 +116,7 @@ def relay_and_wait( self.register_callback(callback) - self.ctrl.relay_to_peers_and_wait(msg_payload, SendOrder(relay_order)) + self.comm.relay_to_peers_and_wait(msg_payload, SendOrder(relay_order)) if callback is None: return self._get_results(task_name) @@ -125,7 +125,7 @@ def relay_and_wait( def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) - self.ctrl.broadcast_to_peers(pay_load=msg_payload) + self.comm.broadcast_to_peers(pay_load=msg_payload) def send( self, @@ -136,7 +136,7 @@ def send( send_order: str = "sequential", ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) - self.ctrl.send_to_peers(pay_load=msg_payload, send_order=send_order) + self.comm.send_to_peers(pay_load=msg_payload, send_order=send_order) def relay( self, @@ -147,7 +147,7 @@ def relay( send_order: str = "sequential", ): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) - self.ctrl.relay_to_peers(msg_payload, send_order) + self.comm.relay_to_peers(msg_payload, send_order) def _process_one_result(self, site_result) -> Dict[str, FLModel]: self._check_result(site_result) @@ -194,7 +194,7 @@ def _check_result(self, site_result): raise RuntimeError(f"expecting all keys {keys} present in site_result") def _check_inputs(self): - if self.ctrl is None: + if self.comm is None: raise RuntimeError("missing Controller") def result_callback(self, topic, data, data_bus): diff --git a/nvflare/app_common/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/wf_comm/wf_comm_api_spec.py index eb64293da8..f248ddfffc 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/wf_comm/wf_comm_api_spec.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,6 +38,20 @@ def broadcast_and_wait( targets: Optional[List[str]] = None, callback: Callable = None, ): + """Communication interface for the blocking version of the 'broadcast' method. + + First, the task is scheduled for broadcast (see the broadcast method); + It then waits until the task is completed. + + Args: + task_name: the name of the task to be sent. + min_responses: the min number of responses expected. If 0, must get responses from + all clients that the task has been sent to. + data: the data to be sent in the task. + meta: the meta to be sent in the task. + targets: list of destination clients. If None, all clients. + callback: callback to be registered. + """ pass @abstractmethod @@ -51,6 +65,21 @@ def send_and_wait( send_order: str = "sequential", callback: Callable = None, ): + """Communication interface for the blocking version of the 'send' method. + + First, the task is scheduled for send (see the 'send' method); + It then waits until the task is completed and returns the task completion status and collected result. + + Args: + task_name: the name of the task to be sent. + min_responses: the min number of responses expected. If 0, must get responses from + all clients that the task has been sent to. + data: the data to be sent in the task. + meta: the meta to be sent in the task. + targets: list of destination clients. + send_order: order for choosing the next client. + callback: callback to be registered. + """ pass @abstractmethod @@ -64,10 +93,52 @@ def relay_and_wait( relay_order: str = "sequential", callback: Callable = None, ): + """Communication interface to schedule a task to be done sequentially by the clients in the targets list. This is a non-blocking call. + + Args: + task_name: the name of the task to be sent. + min_responses: the min number of responses expected. If 0, must get responses from + all clients that the task has been sent to. + data: the data to be sent in the task. + meta: the meta to be sent in the task. + targets: list of destination clients. If None, all clients. + relay_order: order for choosing the next client. + callback: callback to be registered. + """ pass @abstractmethod def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): + """Communication interface to schedule to broadcast the task to specified targets. + + This is a non-blocking call. + + The task is standing until one of the following conditions comes true: + - if timeout is specified (> 0), and the task has been standing for more than the specified time + - the controller has received the specified min_responses results for this task, and all target clients + are done. + - the controller has received the specified min_responses results for this task, and has waited + for wait_time_after_min_received. + + While the task is standing: + - Before sending the task to a client, the before_task_sent CB (if specified) is called; + - When a result is received from a client, the result_received CB (if specified) is called; + + After the task is done, the task_done CB (if specified) is called: + - If result_received CB is specified, the 'result' in the ClientTask of each + client is produced by the result_received CB; + - Otherwise, the 'result' contains the original result submitted by the clients; + + NOTE: if the targets is None, the actual broadcast target clients will be dynamic, because the clients + could join/disconnect at any moment. While the task is standing, any client that joins automatically + becomes a target for this broadcast. + + Args: + task_name: the name of the task to be sent. + data: the data to be sent in the task. + meta: the meta to be sent in the task. + targets: list of destination clients. If None, all clients. + """ pass @abstractmethod @@ -79,6 +150,27 @@ def send( targets: Optional[str] = None, send_order: str = "sequential", ): + """Communication interface to schedule to send the task to a single target client. + + This is a non-blocking call. + + In ANY order, the target client is the first target that asks for task. + In SEQUENTIAL order, the controller will try its best to send the task to the first client + in the targets list. If can't, it will try the next target, and so on. + + NOTE: if the 'targets' is None, the actual target clients will be dynamic, because the clients + could join/disconnect at any moment. While the task is standing, any client that joins automatically + becomes a target for this task. + + If the send_order is SEQUENTIAL, the targets must be a non-empty list of client names. + + Args: + task_name: the name of the task to be sent. + data: the data to be sent in the task. + meta: the meta to be sent in the task. + targets: list of destination clients. If None, all clients. + send_order: order for choosing the next client. + """ pass @abstractmethod @@ -90,8 +182,18 @@ def relay( targets: Optional[List[str]] = None, relay_order: str = "sequential", ): + """Communication interface to schedule a task to be done sequentially by the clients in the targets list. This is a non-blocking call. + + Args: + task_name: the name of the task to be sent. + data: the data to be sent in the task. + meta: the meta to be sent in the task. + targets: list of destination clients. + relay_order: order for choosing the next client. + """ pass @abstractmethod def get_site_names(self) -> List[str]: + """Get list of site names.""" pass diff --git a/nvflare/app_common/wf_comm/wf_communicator.py b/nvflare/app_common/wf_comm/wf_communicator.py index f52c3188dc..ed6d15d103 100644 --- a/nvflare/app_common/wf_comm/wf_communicator.py +++ b/nvflare/app_common/wf_comm/wf_communicator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nvflare/app_common/wf_comm/wf_communicator_spec.py b/nvflare/app_common/wf_comm/wf_communicator_spec.py index c353b7e13e..a198b31e92 100644 --- a/nvflare/app_common/wf_comm/wf_communicator_spec.py +++ b/nvflare/app_common/wf_comm/wf_communicator_spec.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,13 +13,9 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import Dict, List, Optional +from typing import Dict, Optional from nvflare.apis.controller_spec import SendOrder -from nvflare.fuel.utils.class_utils import instantiate_class -from nvflare.fuel.utils.component_builder import ComponentBuilder -from nvflare.fuel.utils.fobs import fobs -from nvflare.fuel.utils.import_utils import optional_import class WFCommunicatorSpec(ABC): @@ -28,55 +24,58 @@ def __init__(self): @abstractmethod def broadcast_to_peers_and_wait(self, pay_load: Dict): + """Convert pay_load and call Controller's 'broadcast_and_wait' method. + + Args: + pay_load: the name of the task to be sent. + """ pass @abstractmethod def broadcast_to_peers(self, pay_load: Dict): + """Convert pay_load and call Controller's 'broadcast' method. + + Args: + pay_load: the name of the task to be sent. + """ pass @abstractmethod def send_to_peers(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): + """Convert pay_load and call Controller's 'send' method. + + Args: + pay_load: the name of the task to be sent. + send_order: order for choosing the next client. + """ pass @abstractmethod def send_to_peers_and_wait(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): + """Convert pay_load and call Controller's 'send_and_wait' method. + + Args: + pay_load: the name of the task to be sent. + send_order: order for choosing the next client. + """ pass @abstractmethod def relay_to_peers_and_wait(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): + """Convert pay_load and call Controller's 'relay_and_wait' method. + + Args: + pay_load: the name of the task to be sent. + send_order: order for choosing the next client. + """ pass @abstractmethod def relay_to_peers(self, pay_load: Dict, send_order: SendOrder = SendOrder.SEQUENTIAL): - pass - - def set_controller_config(self, controller_config: Dict): - if controller_config is None: - raise ValueError("controller_config is None") + """Convert pay_load and call Controller's 'relay' method. - if not isinstance(controller_config, dict): - raise ValueError(f"controller_config should be Dict, found '{type(controller_config)}'") - - self.controller_config = controller_config - - def get_controller(self): - controller = None - if isinstance(self.controller_config, dict): - controller = ComponentBuilder().build_component(self.controller_config) - if controller is None: - raise ValueError("wf_controller should provided, but get None") - - return controller - - def register_serializers(self, serializer_class_paths: List[str] = None): - self.register_default_serializers() - if serializer_class_paths: - for class_path in serializer_class_paths: - fobs.register(instantiate_class(class_path, {})) - - def register_default_serializers(self): - torch, flag = optional_import("torch") - if flag: - from nvflare.app_opt.pt.decomposers import TensorDecomposer - - fobs.register(TensorDecomposer) + Args: + pay_load: the name of the task to be sent. + send_order: order for choosing the next client. + """ + pass diff --git a/nvflare/app_common/workflows/error_handle_utils.py b/nvflare/app_common/workflows/error_handle_utils.py deleted file mode 100644 index 18f7515f47..0000000000 --- a/nvflare/app_common/workflows/error_handle_utils.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from nvflare.apis.fl_constant import ReturnCode - -ABORT_WHEN_IN_ERROR = { - ReturnCode.EXECUTION_EXCEPTION: True, - ReturnCode.TASK_UNKNOWN: True, - ReturnCode.EXECUTION_RESULT_ERROR: False, - ReturnCode.TASK_DATA_FILTER_ERROR: True, - ReturnCode.TASK_RESULT_FILTER_ERROR: True, -} diff --git a/nvflare/app_common/workflows/fed_avg.py b/nvflare/app_common/workflows/fed_avg.py index 3a0bf65061..835b5ca0e3 100644 --- a/nvflare/app_common/workflows/fed_avg.py +++ b/nvflare/app_common/workflows/fed_avg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -78,7 +78,10 @@ def run(self): break # no callback - sag_results = self.prepare_broadcast_and_wait(self.task_name, model) + model.current_round = self.current_round + sag_results = self.broadcast_and_wait( + task_name=self.task_name, min_responses=self.min_clients, data=model, callback=None + ) aggr_result = self.aggr_fn(sag_results) # # with callback @@ -98,17 +101,6 @@ def init_model(self): model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) return model - def prepare_broadcast_and_wait(self, task_name, model: FLModel, callback=None): - # (2) broadcast and wait - model.current_round = self.current_round - results = self.broadcast_and_wait( - task_name=task_name, min_responses=self.min_clients, data=model, callback=callback - ) - if callback is None: - return results - else: - return None - def callback(self, data, topic): self.intime_agg_fn(data, self.aggr_params_helper, self.aggr_metrics_helper) diff --git a/nvflare/app_common/workflows/fed_avg_pt.py b/nvflare/app_opt/pt/fed_avg_pt.py similarity index 95% rename from nvflare/app_common/workflows/fed_avg_pt.py rename to nvflare/app_opt/pt/fed_avg_pt.py index 0f76ebfc3f..340fe33168 100644 --- a/nvflare/app_common/workflows/fed_avg_pt.py +++ b/nvflare/app_opt/pt/fed_avg_pt.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit_test/app_common/utils/math_utils_test.py b/tests/unit_test/app_common/utils/math_utils_test.py new file mode 100644 index 0000000000..dc1ad768bc --- /dev/null +++ b/tests/unit_test/app_common/utils/math_utils_test.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator + +import pytest + +from nvflare.app_common.utils.math_utils import parse_compare_criteria + +TEST_CASES = [ + ("accuracy >= 50", ("accuracy", 50, operator.ge)), + ("accuracy <= 50", ("accuracy", 50, operator.le)), + ("accuracy > 50", ("accuracy", 50, operator.gt)), + ("accuracy < 50", ("accuracy", 50, operator.lt)), + ("accuracy = 50", ("accuracy", 50, operator.eq)), + ("loss < 0.1", ("loss", 0.1, operator.lt)), + ("50 >= 50", ("50", 50, operator.ge)), +] + +INVALID_TEST_CASES = [ + ("50 >= accuracy", None), + ("accuracy >== 50", None), + ("accuracy >= accuracy", None), + (50, None), +] + + +class TestMathUtils: + @pytest.mark.parametrize("compare_expr,compare_tuple", TEST_CASES + INVALID_TEST_CASES) + def test_parse_compare_criteria(self, compare_expr, compare_tuple): + if compare_tuple is None: + with pytest.raises(Exception): + result_tuple = parse_compare_criteria(compare_expr) + else: + result_tuple = parse_compare_criteria(compare_expr) + assert result_tuple == compare_tuple From d37cd234123eb0a269b723096baebd006c0a2c8d Mon Sep 17 00:00:00 2001 From: Sean Yang Date: Fri, 23 Feb 2024 11:33:47 -0800 Subject: [PATCH 41/41] fix communicator pairing, remove temp example --- .../config_fed_client.conf | 116 --------- .../config_fed_server.conf | 41 --- job_templates/sag_pt_wf_controller/info.conf | 5 - job_templates/sag_pt_wf_controller/info.md | 11 - job_templates/sag_pt_wf_controller/meta.conf | 10 - .../app_common/executors/launcher_executor.py | 2 +- .../wf_comm/base_wf_communicator.py | 8 +- nvflare/app_common/wf_comm/wf_comm_api.py | 16 +- .../app_common/wf_comm/wf_comm_api_spec.py | 17 +- nvflare/app_common/workflows/fed_avg.py | 237 ------------------ nvflare/app_opt/pt/fed_avg_pt.py | 41 --- .../private/fed/server/server_json_config.py | 52 ++-- 12 files changed, 49 insertions(+), 507 deletions(-) delete mode 100644 job_templates/sag_pt_wf_controller/config_fed_client.conf delete mode 100644 job_templates/sag_pt_wf_controller/config_fed_server.conf delete mode 100644 job_templates/sag_pt_wf_controller/info.conf delete mode 100644 job_templates/sag_pt_wf_controller/info.md delete mode 100644 job_templates/sag_pt_wf_controller/meta.conf delete mode 100644 nvflare/app_common/workflows/fed_avg.py delete mode 100644 nvflare/app_opt/pt/fed_avg_pt.py diff --git a/job_templates/sag_pt_wf_controller/config_fed_client.conf b/job_templates/sag_pt_wf_controller/config_fed_client.conf deleted file mode 100644 index bd3ad468bb..0000000000 --- a/job_templates/sag_pt_wf_controller/config_fed_client.conf +++ /dev/null @@ -1,116 +0,0 @@ -{ - # version of the configuration - format_version = 2 - - # This is the application script which will be invoked. Client can replace this script with user's own training script. - app_script = "train.py" - - # Additional arguments needed by the training code. For example, in lightning, these can be --trainer.batch_size=xxx. - app_config = "" - - # Client Computing Executors. - executors = [ - { - # tasks the executors are defined to handle - tasks = ["train"] - - # This particular executor - executor { - - # This is an executor for Client API. The underline data exchange is using Pipe. - path = "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor" - - args { - # launcher_id is used to locate the Launcher object in "components" - launcher_id = "launcher" - - # pipe_id is used to locate the Pipe object in "components" - pipe_id = "pipe" - - # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. - # Please refer to the class docstring for all available arguments - heartbeat_timeout = 60 - - # format of the exchange parameters - params_exchange_format = "pytorch" - - # if the transfer_type is FULL, then it will be sent directly - # if the transfer_type is DIFF, then we will calculate the - # difference VS received parameters and send the difference - params_transfer_type = "DIFF" - - # if train_with_evaluation is true, the executor will expect - # the custom code need to send back both the trained parameters and the evaluation metric - # otherwise only trained parameters are expected - train_with_evaluation = true - } - } - } - ], - - # this defined an array of task data filters. If provided, it will control the data from server controller to client executor - task_data_filters = [] - - # this defined an array of task result filters. If provided, it will control the result from client executor to server controller - task_result_filters = [] - - components = [ - { - # component id is "launcher" - id = "launcher" - - # the class path of this component - path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" - - args { - # the launcher will invoke the script - script = "python3 custom/{app_script} {app_config} " - # if launch_once is true, the SubprocessLauncher will launch once for the whole job - # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server - launch_once = true - } - } - { - id = "pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - } - { - id = "metrics_pipe" - path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" - args { - mode = "PASSIVE" - site_name = "{SITE_NAME}" - token = "{JOB_ID}" - root_url = "{ROOT_URL}" - secure_mode = "{SECURE_MODE}" - workspace_dir = "{WORKSPACE}" - } - }, - { - id = "metric_relay" - path = "nvflare.app_common.widgets.metric_relay.MetricRelay" - args { - pipe_id = "metrics_pipe" - event_type = "fed.analytix_log_stats" - # how fast should it read from the peer - read_interval = 0.1 - } - }, - { - # we use this component so the client api `flare.init()` can get required information - id = "config_preparer" - path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" - args { - component_ids = ["metric_relay"] - } - } - ] -} diff --git a/job_templates/sag_pt_wf_controller/config_fed_server.conf b/job_templates/sag_pt_wf_controller/config_fed_server.conf deleted file mode 100644 index 1f2192897a..0000000000 --- a/job_templates/sag_pt_wf_controller/config_fed_server.conf +++ /dev/null @@ -1,41 +0,0 @@ -{ - # version of the configuration - format_version = 2 - task_data_filters =[] - task_result_filters = [] - - workflows = [ - { - id = "fed_avg" - # PTFedAvg is a WFController. If the controller is a type WFController, a WFCommunicator will automatically be configured. - path = "nvflare.app_opt.pt.fedavg_pt.PTFedAvg" - args { - min_clients = 2 - num_rounds = 2 - output_path = "/tmp/nvflare/fedavg/mode.pth" - # stop_cond = "accuracy >= 55" - } - } - # { - # # If using a WFController, can optionally pair with a `communicator` section (will be defaulted to WFCommunicator if not specified) - # communicator { - # path = "nvflare.app_common.wf_comm.wf_communicator.WFCommunicator" - # args = {} - # } - - # # If using a WFController, can be set under `controller` section if wanting to configure to pair with a communicator - # controller { - # path = "nvflare.app_opt.pt.fedavg_pt.PTFedAvg" - # args { - # min_clients = 2 - # num_rounds = 2 - # output_path = "/tmp/nvflare/fedavg/mode.pth" - # # stop_cond = "accuracy >= 55" - # } - # } - # } - ] - - components = [ - ] -} \ No newline at end of file diff --git a/job_templates/sag_pt_wf_controller/info.conf b/job_templates/sag_pt_wf_controller/info.conf deleted file mode 100644 index 5d384b8697..0000000000 --- a/job_templates/sag_pt_wf_controller/info.conf +++ /dev/null @@ -1,5 +0,0 @@ -{ - description = "scatter & gather workflow using pytorch with wf controller" - execution_api_type = "client_api" - controller_type = "server" -} \ No newline at end of file diff --git a/job_templates/sag_pt_wf_controller/info.md b/job_templates/sag_pt_wf_controller/info.md deleted file mode 100644 index 32cc9fcac7..0000000000 --- a/job_templates/sag_pt_wf_controller/info.md +++ /dev/null @@ -1,11 +0,0 @@ -# Job Template Information Card - -## sag_pt_wf_controller - name = "sag_pt_wf_controller" - description = "Scatter and Gather Workflow using pytorch with wf controller" - class_name = "ScatterAndGather" - controller_type = "server" - executor_type = "launcher_executor" - contributor = "NVIDIA" - init_publish_date = "2024-02-08" - last_updated_date = "2024-02-08" # yyyy-mm-dd diff --git a/job_templates/sag_pt_wf_controller/meta.conf b/job_templates/sag_pt_wf_controller/meta.conf deleted file mode 100644 index cbbd7a293c..0000000000 --- a/job_templates/sag_pt_wf_controller/meta.conf +++ /dev/null @@ -1,10 +0,0 @@ -{ - name = "sag_pt_wf_controller" - resource_spec = {} - deploy_map { - # change deploy map as needed. - app = ["@ALL"] - } - min_clients = 2 - mandatory_clients = [] -} diff --git a/nvflare/app_common/executors/launcher_executor.py b/nvflare/app_common/executors/launcher_executor.py index 6be65cbaee..90cc6a3d0a 100644 --- a/nvflare/app_common/executors/launcher_executor.py +++ b/nvflare/app_common/executors/launcher_executor.py @@ -257,8 +257,8 @@ def _execute_launcher_method_in_thread_executor(self, method_name: str, **kwargs future = self._thread_pool_executor.submit(getattr(self.launcher, method_name), **kwargs) result = future.result(timeout=self._launch_timeout) - return result + return result except TimeoutError: self.log_warning( kwargs.get("fl_ctx"), diff --git a/nvflare/app_common/wf_comm/base_wf_communicator.py b/nvflare/app_common/wf_comm/base_wf_communicator.py index e30c6f5071..172e7b98b7 100644 --- a/nvflare/app_common/wf_comm/base_wf_communicator.py +++ b/nvflare/app_common/wf_comm/base_wf_communicator.py @@ -18,7 +18,7 @@ from nvflare.apis.controller_spec import ClientTask, ControllerSpec, OperatorMethod, SendOrder, Task, TaskOperatorKey from nvflare.apis.dxo import DXO, DataKind from nvflare.apis.fl_component import FLComponent -from nvflare.apis.fl_constant import ReturnCode +from nvflare.apis.fl_constant import FLContextKey, ReturnCode from nvflare.apis.fl_context import FLContext from nvflare.apis.shareable import Shareable from nvflare.apis.wf_controller import ABORT_WHEN_IN_ERROR @@ -76,7 +76,7 @@ def start_controller(self, fl_ctx: FLContext): self.register_decomposers() self.clients = self.engine.get_clients() - self.publish_comm_api() + self.publish_comm_api(fl_ctx) self.log_info(fl_ctx, "workflow controller started") def register_decomposers(self): @@ -88,8 +88,8 @@ def register_decomposers(self): ) decomposer_register.register() - def publish_comm_api(self): - comm_api = WFCommAPI() + def publish_comm_api(self, fl_ctx: FLContext): + comm_api = WFCommAPI(cid=fl_ctx.get_prop(FLContextKey.WORKFLOW, "")) comm_api.meta.update({SITE_NAMES: self.get_site_names()}) self.data_bus.put_data("wf_comm_api", comm_api) diff --git a/nvflare/app_common/wf_comm/wf_comm_api.py b/nvflare/app_common/wf_comm/wf_comm_api.py index 2a1743c713..0b9ffaea5b 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api.py +++ b/nvflare/app_common/wf_comm/wf_comm_api.py @@ -15,7 +15,7 @@ import logging import threading -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, List, Optional from nvflare.apis.controller_spec import SendOrder from nvflare.apis.fl_constant import ReturnCode @@ -38,7 +38,7 @@ class WFCommAPI(WFCommAPISpec): - def __init__(self): + def __init__(self, cid: str = ""): self.meta = {SITE_NAMES: []} self.logger = logging.getLogger(self.__class__.__name__) @@ -49,7 +49,7 @@ def __init__(self): data_bus.subscribe(topics=[CommConstants.TASK_RESULT], callback=self.result_callback) self.event_manager = EventManager(data_bus) - self.comm = data_bus.get_data(CommConstants.COMMUNICATOR) + self.comm = data_bus.get_data(cid + CommConstants.COMMUNICATOR) self._check_inputs() def get_site_names(self): @@ -63,7 +63,7 @@ def broadcast_and_wait( meta: dict = None, targets: Optional[List[str]] = None, callback: Callable = None, - ) -> Union[int, Dict[str, Dict[str, FLModel]]]: + ) -> Dict[str, Dict[str, FLModel]]: meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) @@ -73,8 +73,6 @@ def broadcast_and_wait( if callback is None: return self._get_results(task_name) - return 0 - def register_callback(self, callback): if callback: self.event_manager.data_bus.subscribe([CommConstants.POST_PROCESS_RESULT], callback) @@ -88,7 +86,7 @@ def send_and_wait( send_order: SendOrder = SendOrder.SEQUENTIAL, targets: Optional[List[str]] = None, callback: Callable = None, - ): + ) -> Dict[str, Dict[str, FLModel]]: meta = {} if meta is None else meta msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses, targets) @@ -97,7 +95,7 @@ def send_and_wait( self.comm.send_to_peers_and_wait(msg_payload, send_order) - if callback is not None: + if callback is None: return self._get_results(task_name) def relay_and_wait( @@ -121,8 +119,6 @@ def relay_and_wait( if callback is None: return self._get_results(task_name) - return self._get_results(task_name) - def broadcast(self, task_name: str, data: any, meta: dict = None, targets: Optional[List[str]] = None): msg_payload = self._prepare_input_payload(task_name, data, meta, min_responses=0, targets=targets) self.comm.broadcast_to_peers(pay_load=msg_payload) diff --git a/nvflare/app_common/wf_comm/wf_comm_api_spec.py b/nvflare/app_common/wf_comm/wf_comm_api_spec.py index f248ddfffc..f30f92e15d 100644 --- a/nvflare/app_common/wf_comm/wf_comm_api_spec.py +++ b/nvflare/app_common/wf_comm/wf_comm_api_spec.py @@ -13,7 +13,7 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import Callable, List, Optional +from typing import Callable, Dict, List, Optional SITE_NAMES = "SITE_NAMES" TASK_NAME = "TASK_NAME" @@ -37,7 +37,7 @@ def broadcast_and_wait( meta: dict = None, targets: Optional[List[str]] = None, callback: Callable = None, - ): + ) -> Dict[str, any]: """Communication interface for the blocking version of the 'broadcast' method. First, the task is scheduled for broadcast (see the broadcast method); @@ -51,6 +51,9 @@ def broadcast_and_wait( meta: the meta to be sent in the task. targets: list of destination clients. If None, all clients. callback: callback to be registered. + + Returns: + result dict if callback is None """ pass @@ -64,7 +67,7 @@ def send_and_wait( targets: Optional[List[str]] = None, send_order: str = "sequential", callback: Callable = None, - ): + ) -> Dict[str, any]: """Communication interface for the blocking version of the 'send' method. First, the task is scheduled for send (see the 'send' method); @@ -79,6 +82,9 @@ def send_and_wait( targets: list of destination clients. send_order: order for choosing the next client. callback: callback to be registered. + + Returns: + result dict if callback is None """ pass @@ -92,7 +98,7 @@ def relay_and_wait( targets: Optional[List[str]] = None, relay_order: str = "sequential", callback: Callable = None, - ): + ) -> Dict[str, any]: """Communication interface to schedule a task to be done sequentially by the clients in the targets list. This is a non-blocking call. Args: @@ -104,6 +110,9 @@ def relay_and_wait( targets: list of destination clients. If None, all clients. relay_order: order for choosing the next client. callback: callback to be registered. + + Returns: + result dict if callback is None """ pass diff --git a/nvflare/app_common/workflows/fed_avg.py b/nvflare/app_common/workflows/fed_avg.py deleted file mode 100644 index 835b5ca0e3..0000000000 --- a/nvflare/app_common/workflows/fed_avg.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import Callable, Dict, Optional - -from net import Net - -from nvflare.apis.wf_controller import WFController -from nvflare.app_common.abstract.fl_model import FLModel, ParamsType -from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper -from nvflare.app_common.utils.fl_model_utils import FLModelUtils -from nvflare.app_common.utils.math_utils import parse_compare_criteria -from nvflare.security.logging import secure_format_traceback - -update_model = FLModelUtils.update_model - - -# FedAvg Workflow - - -class FedAvg(WFController): - def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - resp_max_wait_time: float = 5, - ): - super(FedAvg, self).__init__() - - self.logger = logging.getLogger(self.__class__.__name__) - self.task_name = "train" - self.output_path = output_path - self.min_clients = min_clients - self.resp_max_wait_time = resp_max_wait_time - self.num_rounds = num_rounds - self.start_round = start_round - self.current_round = start_round - self.best_model: Optional[FLModel] = None - self.aggr_params_helper = WeightedAggregationHelper() - self.aggr_metrics_helper = WeightedAggregationHelper() - self.params_type: Optional[ParamsType] = None - if stop_cond: - self.stop_criteria = parse_compare_criteria(stop_cond) - else: - self.stop_criteria = None - - def run(self): - self.logger.info("start Fed Avg Workflow\n \n") - start = self.start_round - end = self.start_round + self.num_rounds - - model = self.init_model() - model.start_round = self.start_round - model.total_rounds = self.num_rounds - - for current_round in range(start, end): - - self.logger.info(f"Round {current_round}/{self.num_rounds} started. {start=}, {end=}") - self.current_round = current_round - - if self.should_stop(model.metrics, self.stop_criteria): - self.logger.info(f"stop at {current_round}/{self.num_rounds}, early stop condition satisfied.") - break - - # no callback - model.current_round = self.current_round - sag_results = self.broadcast_and_wait( - task_name=self.task_name, min_responses=self.min_clients, data=model, callback=None - ) - aggr_result = self.aggr_fn(sag_results) - - # # with callback - # self.broadcast_and_wait(task_name=self.task_name, model=model, callback=self.callback) - # aggr_result = self.aggr_fn() - - self.logger.info(f"aggregate metrics = {aggr_result.metrics}") - model = update_model(model, aggr_result) - self.select_best_model(model) - - self.save_model(self.best_model, self.output_path) - - self.logger.info("end Fed Avg Workflow\n \n") - - def init_model(self): - net = Net() - model = FLModel(params=net.state_dict(), params_type=ParamsType.FULL) - return model - - def callback(self, data, topic): - self.intime_agg_fn(data, self.aggr_params_helper, self.aggr_metrics_helper) - - def intime_agg_fn(self, data, aggr_params_helper, aggr_metrics_helper): - self.logger.info("\n fed avg intime_aggregate \n") - - if not data: - raise RuntimeError("input is None or empty") - task_name, task_result = next(iter(data.items())) - - try: - for site, fl_model in task_result.items(): - if self.params_type is None: - self.params_type = fl_model.params_type - - aggr_params_helper.add( - data=fl_model.params, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) - - self.logger.info(f"site={site} {fl_model.metrics=}") - - aggr_metrics_helper.add( - data=fl_model.metrics, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) - - except Exception as e: - raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") - - def intime_aggr_fn(self, sag_result: Optional[Dict[str, Dict[str, FLModel]]] = None) -> FLModel: - if self.callback: - return self.get_aggr_result(self.aggr_params_helper, self.aggr_metrics_helper) - else: - raise ValueError("callback function needs to be defined") - - def aggr_fn(self, sag_result: Optional[Dict[str, Dict[str, FLModel]]] = None) -> FLModel: - self.logger.info("fed avg aggregate \n") - - if not sag_result: - raise RuntimeError("input is None or empty") - - # we only have one task - task_name, task_result = next(iter(sag_result.items())) - self.logger.info(f"aggregating {len(task_result)} update(s) at round {self.current_round}") - - try: - aggr_params_helper = WeightedAggregationHelper() - aggr_metrics_helper = WeightedAggregationHelper() - params_type = None - for site, fl_model in task_result.items(): - if params_type is None: - params_type = fl_model.params_type - - aggr_params_helper.add( - data=fl_model.params, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) - - self.logger.info(f"site={site} {fl_model.metrics=}") - - aggr_metrics_helper.add( - data=fl_model.metrics, - weight=self.current_round, - contributor_name=site, - contribution_round=self.current_round, - ) - - return self.get_aggr_result(aggr_params_helper, aggr_metrics_helper) - - except Exception as e: - raise RuntimeError(f"Exception in aggregate call: {secure_format_traceback()}") - - def select_best_model(self, curr_model: FLModel): - if self.best_model is None: - self.best_model = curr_model - return - - if self.stop_criteria: - metric, _, op_fn = self.stop_criteria - self.logger.info("compare models") - if self.is_curr_model_better(self.best_model, curr_model, metric, op_fn): - self.best_model = curr_model - else: - self.best_model = curr_model - - def save_model(self, model: FLModel, file_path: str): - pass - - def should_stop(self, metrics: Optional[Dict] = None, stop_criteria: Optional[str] = None): - self.logger.info(f"stop_criteria, metrics = {stop_criteria=}, {metrics=}") - if stop_criteria is None or metrics is None: - return False - - key, target, op_fn = stop_criteria - value = metrics.get(key, None) - - if value is None: - raise RuntimeError(f"stop criteria key '{key}' doesn't exists in metrics") - - return op_fn(value, target) - - def is_curr_model_better( - self, best_model: FLModel, curr_model: FLModel, target_metric: str, op_fn: Callable - ) -> bool: - curr_metrics = curr_model.metrics - if curr_metrics is None: - return False - if target_metric not in curr_metrics: - return False - - best_metrics = best_model.metrics - return op_fn(curr_metrics.get(target_metric), best_metrics.get(target_metric)) - - def get_aggr_result(self, aggr_params_helper, aggr_metrics_helper): - aggr_params = aggr_params_helper.get_result() - aggr_metrics = aggr_metrics_helper.get_result() - - aggr_result = FLModel( - params=aggr_params, - params_type=self.params_type, - metrics=aggr_metrics, - meta={ - "num_rounds_aggregated": 1 + (self.current_round - self.start_round), - "current_round": self.current_round, - }, - ) - return aggr_result diff --git a/nvflare/app_opt/pt/fed_avg_pt.py b/nvflare/app_opt/pt/fed_avg_pt.py deleted file mode 100644 index 340fe33168..0000000000 --- a/nvflare/app_opt/pt/fed_avg_pt.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -import torch - -from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.workflows.fed_avg import FedAvg - - -class PTFedAvg(FedAvg): - def __init__( - self, - min_clients: int, - num_rounds: int, - output_path: str, - start_round: int = 1, - stop_cond: str = None, - ): - super().__init__(min_clients, num_rounds, output_path, start_round, stop_cond) - - def save_model(self, model: FLModel, file_path: str): - if not file_path: - raise ValueError("invalid file path") - - dir_name = os.path.dirname(file_path) - os.makedirs(dir_name, exist_ok=True) - - self.logger.info(f"save best model to {file_path} \n") - torch.save(model.params, file_path) diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py index 8dcb97896e..a97e982c28 100644 --- a/nvflare/private/fed/server/server_json_config.py +++ b/nvflare/private/fed/server/server_json_config.py @@ -50,25 +50,6 @@ def __init__(self, id, responder: Responder, wf_controller=None): self.wf_controller = wf_controller -def enhance_workflow_config(element: dict, class_path: str): - if CommConstants.CONTROLLER in element: - controller_config = element.get(CommConstants.CONTROLLER) - controller_config["lazy_instantiate"] = True - element[CommConstants.CONTROLLER] = controller_config - elif CommConstants.COMMUNICATOR in element: - wf_config = element.copy() - comm_config = wf_config.pop(CommConstants.COMMUNICATOR) - controller_config = wf_config - controller_config["lazy_instantiate"] = True - element = {CommConstants.COMMUNICATOR: comm_config, CommConstants.CONTROLLER: controller_config} - elif isinstance(instantiate_class(class_path, element.get("args", dict())), WFController): - controller_config = element.copy() - controller_config["lazy_instantiate"] = True - element = {CommConstants.CONTROLLER: controller_config} - - return element - - class ServerJsonConfigurator(FedJsonConfigurator): def __init__(self, config_file_name: str, args, app_root: str, kv_list=None, exclude_libs=True): """This class parses server config from json file. @@ -152,15 +133,10 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): return if re.search(r"^workflows\.#[0-9]+$", path): - class_path = self.get_class_path(element) - element = enhance_workflow_config(element, class_path) + element = self.enhance_workflow_config(element) component = self.authorize_and_build_component(element, config_ctx, node) - responder = self.get_responder(component) - cid = element.get("id", None) - if not cid: - cid = type(responder).__name__ if not isinstance(cid, str): raise ConfigError('"id" must be str but got {}'.format(type(cid))) @@ -171,12 +147,34 @@ def process_config_element(self, config_ctx: ConfigContext, node: Node): if cid in self.components: raise ConfigError('duplicate component id "{}"'.format(cid)) + responder = self.get_responder(component, cid) + workflow = WorkFlow(cid, responder) self.workflows.append(workflow) self.components[cid] = responder return - def get_responder(self, component): + def enhance_workflow_config(self, element: dict): + if CommConstants.CONTROLLER in element: + controller_config = element.get(CommConstants.CONTROLLER) + controller_config["lazy_instantiate"] = True + element[CommConstants.CONTROLLER] = controller_config + elif CommConstants.COMMUNICATOR in element: + wf_config = element.copy() + comm_config = wf_config.pop(CommConstants.COMMUNICATOR) + controller_config = wf_config + controller_config["lazy_instantiate"] = True + id = controller_config.pop("id") + element = {"id": id, CommConstants.COMMUNICATOR: comm_config, CommConstants.CONTROLLER: controller_config} + elif isinstance(instantiate_class(self.get_class_path(element), element.get("args", dict())), WFController): + controller_config = element.copy() + controller_config["lazy_instantiate"] = True + id = controller_config.pop("id") + element = {"id": id, CommConstants.CONTROLLER: controller_config} + + return element + + def get_responder(self, component, cid): if isinstance(component, dict): wf_config = component communicator = wf_config.get(CommConstants.COMMUNICATOR) @@ -188,7 +186,7 @@ def get_responder(self, component): controller_config["lazy_instantiate"] = False communicator.set_controller_config(controller_config) data_bus = DataBus() - data_bus.put_data(CommConstants.COMMUNICATOR, communicator) + data_bus.put_data(cid + CommConstants.COMMUNICATOR, communicator) responder = communicator else: responder = component