refine

Author: wanghuancoder

test_pipline/V2/run_gpu_bigtensor_regr_EBRL.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Script to run engineV2.py
+# Usage: ./run.sh
+
+# 配置参数
+# NUM_GPUS!=0 时，engineV2 不受外部 "CUDA_VISIBLE_DEVICES" 影响
+FILE_INPUT="path_to_EBRL/EBRL_config.txt"
+# FILE_PATTERN="report/big_tensor_gpu/error_config.txt"
+LOG_DIR="tester/api_config/test_log_gpu_bigtensor_regr_EBRL"
+NUM_GPUS=-1
+NUM_WORKERS_PER_GPU=1
+GPU_IDS="-1"
+# REQUIRED_MEMORY=10
+
+TEST_MODE_ARGS=(
+	--accuracy=True
+	# --paddle_only=True
+    # --paddle_cinn=True
+	# --test_amp=True
+	# --test_cpu=True
+	--use_cached_numpy=False
+)
+
+IN_OUT_ARGS=(
+    --api_config_file="$FILE_INPUT"
+    # --api_config_file_pattern="$FILE_PATTERN"
+    --log_dir="$LOG_DIR"
+)
+
+PARALLEL_ARGS=(
+    --num_gpus="$NUM_GPUS"
+    --num_workers_per_gpu="$NUM_WORKERS_PER_GPU"
+    --gpu_ids="$GPU_IDS"
+    # --required_memory="$REQUIRED_MEMORY"
+)
+
+mkdir -p "$LOG_DIR" || {
+    echo "错误：无法创建日志目录 '$LOG_DIR'"
+    exit 1
+}
+
+# 执行程序
+LOG_FILE="$LOG_DIR/log_$(date +%Y%m%d_%H%M%S).log"
+nohup python engineV2.py \
+        "${TEST_MODE_ARGS[@]}" \
+        "${IN_OUT_ARGS[@]}" \
+        "${PARALLEL_ARGS[@]}" \
+        >> "$LOG_FILE" 2>&1 &
+
+PYTHON_PID=$!
+
+sleep 1
+if ! ps -p "$PYTHON_PID" > /dev/null; then
+    echo "错误：engineV2 启动失败，请检查 $LOG_FILE"
+    exit 1
+fi
+
+echo -e "\n\033[32m执行中... 另开终端运行监控:\033[0m"
+echo -e "1. GPU使用:   watch -n 1 nvidia-smi"
+echo -e "2. 日志目录:  ls -lh $LOG_DIR"
+echo -e "3. 详细日志:  tail -f $LOG_FILE"
+echo -e "4. 终止任务:  kill $PYTHON_PID"
+echo -e "\n进程已在后台运行，关闭终端不会影响进程执行"
+
+exit 0
+
+# watch -n 1 nvidia-smi --query-compute-apps=pid,process_name,used_memory,gpu_uuid --format=csv

tester/api_config/bittensor_config_filter.py

@@ -0,0 +1,60 @@
+from config_analyzer import TensorConfig, APIConfig, analyse_configs
+from tqdm import tqdm
+import random
+
+def is_0_size_tensor(tensor_config):
+    for i in tensor_config.shape:
+        if i == 0:
+            return True
+    return False
+
+def is_0D_tensor(tensor_config):
+    return len(tensor_config.shape) == 0
+
+def tensor_numel(tensor_config):
+    numel = 1
+    for i in tensor_config.shape:
+        numel = numel * i
+    return numel
+
+def get_tensor_configs(api_config):
+    tensor_configs = []
+    for arg_config in api_config.args:
+        if isinstance(arg_config, TensorConfig):
+            tensor_configs.append(arg_config)
+        elif isinstance(arg_config, list):
+            for j in range(len(arg_config)):
+                if isinstance(arg_config[j], TensorConfig):
+                    tensor_configs.append(arg_config[j])
+        elif isinstance(arg_config, tuple):
+            for j in range(len(arg_config)):
+                if isinstance(arg_config[j], TensorConfig):
+                    tensor_configs.append(arg_config[j])
+
+    for key, arg_config in api_config.kwargs.items():
+        if isinstance(arg_config, TensorConfig):
+            tensor_configs.append(arg_config)
+        elif isinstance(arg_config, list):
+            for j in range(len(arg_config)):
+                if isinstance(arg_config[j], TensorConfig):
+                    tensor_configs.append(arg_config[j])
+        elif isinstance(arg_config, tuple):
+            for j in range(len(arg_config)):
+                if isinstance(arg_config[j], TensorConfig):
+                    tensor_configs.append(arg_config[j])
+    return tensor_configs
+
+file_list = [
+    "/host_home/wanghuan29/APItest/PaddleAPITest/EB45/EBRL.txt",
+]
+
+if __name__ == '__main__':
+    for file in file_list:
+        api_configs = analyse_configs(file)
+        for api_config in tqdm(api_configs):
+            tensor_configs = get_tensor_configs(api_config)
+            for tensor_config in tensor_configs:
+                if tensor_numel(tensor_config) >= 2147483647:
+                    print(api_config.config)
+                    break
+