fix params sync multi times problem (#45406)

PaddlePaddle · Aug 25, 2022 · 20d3866 · 20d3866
1 parent 9ac27ac
commit 20d3866
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/paddle/fluid/inference/analysis/passes/convert_to_mixed_precision.cc b/paddle/fluid/inference/analysis/passes/convert_to_mixed_precision.cc
@@ -368,6 +368,7 @@ void ProcessInputNode(
         in_var_type == framework::proto::VarType::FP32) {
       if (WeightsShouldNotConvert(in_node)) return;
       in_var->SetDataType(to_type);
+      in_var_type = to_type;
     } else if (!in_var->Persistable() && IsFloatVarType(in_var_type) &&
                in_var_type != to_type) {
       AddCastOp(graph,

diff --git a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc
@@ -14,6 +14,7 @@
 
 #include "paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.h"
 
+#include <string>
 #include <unordered_set>
 
 #include "paddle/fluid/framework/data_layout.h"
@@ -113,6 +114,7 @@ void IrParamsSyncAmongDevicesPass::CopyParamsToGpu(Argument *argument) {
     reserve_cpu_weights = true;
   }
 
+  std::unordered_set<std::string> visited;
   for (auto *node : paddle::framework::ir::TopologySortOperations(graph)) {
     if (!node->IsOp()) continue;
     if (node->Op()->Type() == "feed" || node->Op()->Type() == "fetch") continue;
@@ -126,6 +128,8 @@ void IrParamsSyncAmongDevicesPass::CopyParamsToGpu(Argument *argument) {
         }
         continue;
       }
+      if (visited.count(var_name)) continue;
+      visited.insert(var_name);
       auto *var = scope->FindLocalVar(var_name);
       PADDLE_ENFORCE_NOT_NULL(var,
                               platform::errors::PreconditionNotMet(