From 2802f42e612982d633315b75b617859da7da5c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fwbmqwang=28=E7=8E=8B=E7=9B=9F=E5=8D=BF=29?= Date: Wed, 28 Feb 2024 09:37:39 +0800 Subject: [PATCH 1/4] config jsonConverToYaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: v_wbmqwang(王盟卿) --- .../board/services/JobManagerService.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/fedai/fate/board/services/JobManagerService.java b/src/main/java/org/fedai/fate/board/services/JobManagerService.java index 2656f856..29f7c536 100644 --- a/src/main/java/org/fedai/fate/board/services/JobManagerService.java +++ b/src/main/java/org/fedai/fate/board/services/JobManagerService.java @@ -20,6 +20,8 @@ import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; import com.alibaba.fastjson.serializer.SerializerFeature; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.fedai.fate.board.exceptions.LogicException; @@ -39,6 +41,8 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.springframework.util.concurrent.ListenableFuture; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; import javax.servlet.http.HttpServletResponse; import java.io.*; @@ -319,8 +323,10 @@ public ResponseResult download(DownloadQO downloadQO, HttpServletResponse respon response.setContentType("application/force-download"); response.setHeader("Content-Disposition", "attachment;fileName=" + fileOutputName); try { + String yamlStr = jsonConverToYaml(dagInfo.toJSONString()); OutputStream os = response.getOutputStream(); - os.write(JSON.toJSONBytes(dagInfo, SerializerFeature.PrettyFormat, SerializerFeature.WriteMapNullValue, SerializerFeature.WriteDateUseDateFormat)); +// os.write(JSON.toJSONBytes(dagInfo, SerializerFeature.PrettyFormat, SerializerFeature.WriteMapNullValue, SerializerFeature.WriteDateUseDateFormat)); + os.write(yamlStr.getBytes()); log.info("download success,file :{}", fileOutputName); } catch (Exception e) { log.error("download failed", e); @@ -329,6 +335,17 @@ public ResponseResult download(DownloadQO downloadQO, HttpServletResponse respon return null; } + private String jsonConverToYaml(String jsonString) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + Object obj = objectMapper.readValue(jsonString, Object.class); + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(options); + String yamlStr = yaml.dumpAsMap(obj); + return yamlStr; + } + + //host端需过滤掉其他方信息 private JSONObject getHostConfig(JSONObject runtime_confObject) { if (runtime_confObject != null) { From 23fce7d9285362da49c2bb196f5c441b70ff5021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fwbmqwang=28=E7=8E=8B=E7=9B=9F=E5=8D=BF=29?= Date: Wed, 28 Feb 2024 09:44:51 +0800 Subject: [PATCH 2/4] fix bin file starting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: v_wbmqwang(王盟卿) --- bin/service.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/service.sh b/bin/service.sh index 78f018cd..2ac286de 100644 --- a/bin/service.sh +++ b/bin/service.sh @@ -81,9 +81,11 @@ eval action=\$$# main_class=org.fedai.fate.board.bootstrap.Bootstrap module=fateboard version=2.0.0 +start_type=background if [ $action = starting ];then action=start + start_type=front elif [ $action = restarting ];then action=restart fi @@ -252,7 +254,7 @@ start() { cmd="$JAVA_HOME/bin/java -Dspring.config.location=$configpath/application.properties -Dssh_config_file=$basepath/ssh/ -Xmx2048m -Xms2048m -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -cp $libpath/*:$basepath/${module}-${version}.jar ${main_class}" print_info "The command is: $cmd" - if [[ $1 == "front" ]]; then + if [[ ${start_type} == "front" ]]; then exec $cmd >> ${BOARD_HOME}/logs/bootstrap.${module}.out 2>>${BOARD_HOME}/logs/bootstrap.${module}.err else exec $cmd >> ${BOARD_HOME}/logs/bootstrap.${module}.out 2>>${BOARD_HOME}/logs/bootstrap.${module}.err & From be2a876386aebafd946395c4ca433e9bc6cc0542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fwbmqwang=28=E7=8E=8B=E7=9B=9F=E5=8D=BF=29?= Date: Wed, 28 Feb 2024 11:08:39 +0800 Subject: [PATCH 3/4] fix bin file starting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: v_wbmqwang(王盟卿) --- bin/service.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/service.sh b/bin/service.sh index 2ac286de..f39fbeff 100644 --- a/bin/service.sh +++ b/bin/service.sh @@ -80,7 +80,7 @@ eval action=\$$# main_class=org.fedai.fate.board.bootstrap.Bootstrap module=fateboard -version=2.0.0 +version=2.1.0 start_type=background if [ $action = starting ];then From 6a1685db37fa2de38763106373186a498524cb20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fwbmqwang=28=E7=8E=8B=E7=9B=9F=E5=8D=BF=29?= Date: Thu, 29 Feb 2024 09:31:06 +0800 Subject: [PATCH 4/4] update component yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: v_wbmqwang(王盟卿) --- static/evaluation.yaml | 3 ++- static/hetero_nn.yaml | 11 ++++++----- static/hetero_secureboost.yaml | 9 +++++---- static/homo_lr.yaml | 6 +++--- static/homo_nn.yaml | 11 ++++++----- static/union.yaml | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/static/evaluation.yaml b/static/evaluation.yaml index 740bbb22..34950949 100644 --- a/static/evaluation.yaml +++ b/static/evaluation.yaml @@ -54,7 +54,7 @@ component: use 'label' in the input dataframe input_artifacts: data: - input_data: + input_datas: types: - dataframe optional: false @@ -82,3 +82,4 @@ component: description: metric, invisible for user is_multi: false schema_version: v1 + diff --git a/static/hetero_nn.yaml b/static/hetero_nn.yaml index 634baa23..8ec901a8 100644 --- a/static/hetero_nn.yaml +++ b/static/hetero_nn.yaml @@ -87,7 +87,7 @@ component: description: '' is_multi: false model: - train_model_input: + warm_start_model: types: - model_directory optional: true @@ -98,7 +98,7 @@ component: - host description: '' is_multi: false - predict_model_input: + input_model: types: - model_directory optional: false @@ -111,7 +111,7 @@ component: is_multi: false output_artifacts: data: - train_data_output: + train_output_data: types: - dataframe optional: true @@ -122,7 +122,7 @@ component: - host description: '' is_multi: false - predict_data_output: + test_output_data: types: - dataframe optional: true @@ -134,7 +134,7 @@ component: description: '' is_multi: false model: - train_model_output: + output_model: types: - model_directory optional: true @@ -155,3 +155,4 @@ component: description: metric, invisible for user is_multi: false schema_version: v1 + diff --git a/static/hetero_secureboost.yaml b/static/hetero_secureboost.yaml index c371a8e8..f2fc0837 100644 --- a/static/hetero_secureboost.yaml +++ b/static/hetero_secureboost.yaml @@ -316,7 +316,7 @@ component: description: '' is_multi: false model: - train_model_input: + warm_start_model: types: - json_model optional: true @@ -327,7 +327,7 @@ component: - host description: '' is_multi: false - predict_model_input: + input_model: types: - json_model optional: false @@ -340,7 +340,7 @@ component: is_multi: false output_artifacts: data: - train_data_output: + train_output_data: types: - dataframe optional: true @@ -374,7 +374,7 @@ component: description: '' is_multi: true model: - train_model_output: + output_model: types: - json_model optional: true @@ -395,3 +395,4 @@ component: description: metric, invisible for user is_multi: false schema_version: v1 + diff --git a/static/homo_lr.yaml b/static/homo_lr.yaml index 88409352..07a93da6 100644 --- a/static/homo_lr.yaml +++ b/static/homo_lr.yaml @@ -206,7 +206,7 @@ component: description: '' is_multi: false model: - train_input_model: + warm_start_model: types: - json_model optional: true @@ -217,7 +217,7 @@ component: - host description: '' is_multi: false - predict_input_model: + input_model: types: - json_model optional: false @@ -253,7 +253,7 @@ component: description: '' is_multi: false model: - train_output_model: + output_model: types: - json_model optional: false diff --git a/static/homo_nn.yaml b/static/homo_nn.yaml index d68745fb..13474c7d 100644 --- a/static/homo_nn.yaml +++ b/static/homo_nn.yaml @@ -88,7 +88,7 @@ component: description: '' is_multi: false model: - train_model_input: + warm_start_model: types: - model_directory optional: true @@ -99,7 +99,7 @@ component: - host description: '' is_multi: false - predict_model_input: + input_model: types: - model_directory optional: false @@ -112,7 +112,7 @@ component: is_multi: false output_artifacts: data: - train_data_output: + train_output_data: types: - dataframe optional: true @@ -123,7 +123,7 @@ component: - host description: '' is_multi: false - predict_data_output: + test_output_data: types: - dataframe optional: true @@ -135,7 +135,7 @@ component: description: '' is_multi: false model: - train_model_output: + output_model: types: - model_directory optional: true @@ -156,3 +156,4 @@ component: description: metric, invisible for user is_multi: false schema_version: v1 + diff --git a/static/union.yaml b/static/union.yaml index f1ff1d42..5b049164 100644 --- a/static/union.yaml +++ b/static/union.yaml @@ -10,7 +10,7 @@ component: parameters: { } input_artifacts: data: - input_data_list: + input_datas: types: - dataframe optional: false