[Benchmark] add maskformer and pphumanseg_server to train benchmark (P…

…addlePaddle#3068)

* fix bisenetv2 and fastscnn config for benchmark

* [Benchmark] Align the benchmark test environment of each model, such as num_workers, log_iters, epoch, not_eval, to_static_train, etc.

* [Benchmark] add pphumanseg_server to train benchmark

* [Benchmark] add maskformer to train benchmark

---------

Co-authored-by: juncaipeng <[email protected]>

test_tipc/configs/maskformer/train_infer_python.txt

@@ -50,9 +50,11 @@ inference:deploy/python/infer.py
 --benchmark:True
 --save_dir:
 --model_name:maskformer
-===========================disable_infer_benchmark==========================
-random_infer_input:[{float32,[3,1024,512]}]
-===========================disable_train_benchmark==========================
+===========================infer_benchmark_params==========================
+random_infer_input:[{float32,[3,512,512]}]
+===========================to_static_train_benchmark_params===========================
+to_static_train:--opts to_static_training=True
+===========================train_benchmark_params==========================
 batch_size:4
 fp_items:fp32|fp16
 epoch:400

test_tipc/configs/pphumanseg_server/human_pp_humansegv1_server.yml

@@ -0,0 +1,74 @@
+batch_size: 8
+iters: 1000
+
+train_dataset:
+  type: Dataset
+  dataset_root: test_tipc/data/mini_supervisely
+  train_path: test_tipc/data/mini_supervisely/train.txt
+  num_classes: 2
+  transforms:
+    - type: Resize
+      target_size: [512, 512]
+    - type: ResizeStepScaling
+      scale_step_size: 0
+    - type: RandomRotation
+    - type: RandomPaddingCrop
+      crop_size: [512, 512]
+    - type: RandomHorizontalFlip
+    - type: RandomDistort
+    - type: RandomBlur
+      prob: 0.3
+    - type: Normalize
+  mode: train
+
+val_dataset:
+  type: Dataset
+  dataset_root: test_tipc/data/mini_supervisely
+  val_path: test_tipc/data/mini_supervisely/val.txt
+  num_classes: 2
+  transforms:
+    - type: Resize
+      target_size: [512, 512]
+    - type: Normalize
+  mode: val
+
+export:
+  transforms:
+    - type: Resize
+      target_size: [512, 512]
+    - type: Normalize
+
+
+optimizer:
+  type: sgd
+  momentum: 0.9
+  weight_decay: 0.0005
+
+lr_scheduler:
+  type: PolynomialDecay
+  learning_rate: 0.001
+  end_lr: 0
+  power: 0.9
+
+loss:
+  types:
+    - type: MixedLoss
+      losses:
+        - type: CrossEntropyLoss
+        - type: LovaszSoftmaxLoss
+      coef: [0.8, 0.2]
+  coef: [1]
+
+model:
+  type: DeepLabV3P
+  backbone:
+    type: ResNet50_vd
+    output_stride: 8
+    multi_grid: [1, 2, 4]
+    pretrained: https://bj.bcebos.com/paddleseg/dygraph/resnet50_vd_ssld_v2.tar.gz
+  num_classes: 2
+  backbone_indices: [0, 3]
+  aspp_ratios: [1, 12, 24, 36]
+  aspp_out_channels: 256
+  align_corners: False
+  pretrained: null

test_tipc/configs/pphumanseg_server/train_infer_python.txt

@@ -0,0 +1,63 @@
+===========================train_params===========================
+model_name:pphumanseg_server
+python:python3.7
+gpu_list:0|0,1
+Global.use_gpu:null|null
+Global.auto_cast:null
+--iters:lite_train_lite_infer=20|lite_train_whole_infer=20|whole_train_whole_infer=1000
+--save_dir:
+--batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=8
+--model_path:null
+train_model_name:best_model/model.pdparams
+train_infer_img_dir:test_tipc/data/mini_supervisely/test.txt
+--profiler_options:null
+##
+trainer:norm_train
+norm_train:tools/train.py --config test_tipc/configs/pphumanseg_server/human_pp_humansegv1_server.yml --device gpu --save_interval 500 --seed 100 --num_workers 8
+pact_train:null
+fpgm_train:null
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params===========================
+eval:null
+null:null
+##
+===========================export_params===========================
+--save_dir:
+--model_path:
+norm_export:tools/export.py --config test_tipc/configs/pphumanseg_server/human_pp_humansegv1_server.yml
+quant_export:null
+fpgm_export:null
+distill_export:null
+export1:null
+export2:null
+===========================infer_params===========================
+infer_model:./test_tipc/output/pphumanseg_server/pphumanseg_server_generic_192x192/model.pdparams
+infer_export:tools/export.py --config test_tipc/configs/pphumanseg_server/human_pp_humansegv1_server.yml
+infer_quant:False
+inference:deploy/python/infer.py
+--device:cpu|gpu
+--enable_mkldnn:True
+--cpu_threads:6
+--batch_size:1
+--use_trt:False
+--precision:fp32
+--config:
+--image_path:./test_tipc/data/mini_supervisely/test.txt
+--save_log_path:null
+--benchmark:True
+--save_dir:
+--model_name:pphumanseg_server
+===========================infer_benchmark_params==========================
+random_infer_input:[{float32,[3,512,512]}]
+===========================to_static_train_benchmark_params===========================
+to_static_train:--opts to_static_training=True
+===========================train_benchmark_params==========================
+batch_size:2|4
+fp_items:fp32|fp16
+epoch:400
+--profiler_options:'batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile'  
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096;FLAGS_cudnn_deterministic=False
+log_iters:15;skip_iters:2;repeats:500

test_tipc/prepare.sh

@@ -134,6 +134,7 @@ mkdir -p ./test_tipc/data
 if [ ${MODE} = "benchmark_train" ]; then
     if [ ${model_name} = 'fcn_hrnetw18_small' ] \
         || [ ${model_name} = 'pphumanseg_lite' ] \
+        || [ ${model_name} = 'pphumanseg_server' ] \
         || [ ${model_name} = 'deeplabv3p_resnet50' ] \
         || [ ${model_name} = 'pp_humanseg_lite_KL' ] \
         || [ ${model_name} = 'fcn_hrnetw18_small_KL' ] \
@@ -149,6 +150,10 @@ if [ ${MODE} = "benchmark_train" ]; then
         rm -rf ./test_tipc/data/cityscapes
         wget https://paddleseg.bj.bcebos.com/dataset/cityscapes.tar -O ./test_tipc/data/cityscapes.tar --no-check-certificate
         tar -xf ./test_tipc/data/cityscapes.tar -C ./test_tipc/data/
+    elif [ ${model_name} = 'maskformer' ] || [ ${model_name} = 'mask2former' ]; then
+        rm -rf ./test_tipc/data/ADE20k-20
+        wget -nc -P  ./test_tipc/data/  https://bj.bcebos.com/paddleseg/tipc/data/ADE20k-20.zip --no-check-certificate
+        cd ./test_tipc/data/ && unzip ADE20k-20.zip && cd -
     else
         rm -rf ./test_tipc/data/cityscapes
         wget https://paddleseg.bj.bcebos.com/tipc/data/cityscapes_300imgs.tar.gz \
@@ -160,16 +165,16 @@ elif [ ${MODE} = "serving_infer" ]; then
     wget -nc -P ./test_tipc/data https://paddleseg.bj.bcebos.com/dygraph/demo/cityscapes_small.png --no-check-certificate
 elif [ ${MODE} = "lite_train_lite_infer" ] || [ ${MODE} = "lite_train_whole_infer" ] || [ ${MODE} = "whole_train_whole_infer" ] || [ ${MODE} = "whole_infer" ]; then
 
-    if [ ${model_name} = "fcn_hrnetw18_small" ] || [ ${model_name} = "pphumanseg_lite" ] || [ ${model_name} = "deeplabv3p_resnet50" ] || [ ${model_name} = "pp_humanseg_lite_KL" ] || [ ${model_name} = "fcn_hrnetw18_small_KL" ] || [ ${model_name} = "deeplabv3p_resnet50_KL" ]; then
+    if [ ${model_name} = "fcn_hrnetw18_small" ] || [ ${model_name} = "pphumanseg_lite" ] || [ ${model_name} = "deeplabv3p_resnet50" ] || [ ${model_name} = "pp_humanseg_lite_KL" ] || [ ${model_name} = "fcn_hrnetw18_small_KL" ] || [ ${model_name} = "deeplabv3p_resnet50_KL" ] || [ ${model_name} = "pphumanseg_server" ]; then
         rm -rf ./test_tipc/data/mini_supervisely
         wget -nc -P ./test_tipc/data/ https://paddleseg.bj.bcebos.com/humanseg/data/mini_supervisely.zip --no-check-certificate
         cd ./test_tipc/data/ && unzip mini_supervisely.zip && cd -
     elif [ ${model_name} = "ppmatting" ]; then
         rm -rf ./test_tipc/data/PPM-100
         wget -nc -P ./test_tipc/data/ https://paddleseg.bj.bcebos.com/matting/datasets/PPM-100.zip --no-check-certificate
         cd ./test_tipc/data/ && unzip PPM-100.zip && cd -
-    elif [ ${model_name} == "maskformer" ]; then
-        rm -rf ./test_tipc/data/ADE20K-20
+    elif [ ${model_name} == "maskformer" ] || [ ${model_name} == "mask2former" ]; then
+        rm -rf ./test_tipc/data/ADE20k-20
         wget -nc -P  ./test_tipc/data/  https://bj.bcebos.com/paddleseg/tipc/data/ADE20k-20.zip --no-check-certificate
         cd ./test_tipc/data/ && unzip ADE20k-20.zip && cd -
     else