diff --git a/build.sh b/build.sh index efd8c637f..211994ebc 100755 --- a/build.sh +++ b/build.sh @@ -181,6 +181,16 @@ function build_memory_saver() cd - } +function create_deepep_cmake() +{ + cd csrc || exit + chmod +x deepep_cmake_build.sh + chmod +x deepep/build.sh + chmod +x deepep/compile_ascend_proj.sh + ./deepep_cmake_build.sh + cd - +} + function make_deepep_package() { cd python/deep_ep || exit @@ -209,7 +219,7 @@ function make_sgl_kernel_npu_package() function main() { - + create_deepep_cmake build_kernels build_deepep_kernels if pip3 show wheel;then diff --git a/csrc/deepep/AddCustom.json b/csrc/deepep/AddCustom.json new file mode 100644 index 000000000..d4162b478 --- /dev/null +++ b/csrc/deepep/AddCustom.json @@ -0,0 +1,40 @@ +[ + { + "op": "AddCustom", + "language": "cpp", + "input_desc": [ + { + "name": "x", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float16" + ] + }, + { + "name": "y", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float16" + ] + } + ], + "output_desc": [ + { + "name": "z", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float16" + ] + } + ] + } +] diff --git a/csrc/deepep/build.sh b/csrc/deepep/build.sh new file mode 100755 index 000000000..81e1bf760 --- /dev/null +++ b/csrc/deepep/build.sh @@ -0,0 +1,123 @@ +#!/bin/bash +export MODULE_NAME="deepep" +export MODULE_SRC_PATH="${SRC_PATH}/${MODULE_NAME}" +export MODULE_SCRIPTS_PATH="${SCRIPTS_PATH}/${MODULE_NAME}" +export MODULE_BUILD_OUT_PATH="${BUILD_OUT_PATH}/${MODULE_NAME}" +export MODULE_TEST_PATH="${TEST_PATH}/${MODULE_NAME}" +IS_EXTRACT=0 +SOC_VERSION="all" +ENABLE_UT_BUILD=0 +ENABLE_PYBIND_BUILD=0 +ENABLE_SRC_BUILD=1 + +BuildPybind() { + DIST_OUT_PATH=$MODULE_BUILD_OUT_PATH + if [ -d $DIST_OUT_PATH/dist ]; then + rm -rf $DIST_OUT_PATH/dist + fi + EXT_PATH=$MODULE_SRC_PATH + cd $EXT_PATH + sh build.sh + DIST_GEN_PATH=$EXT_PATH/dist/ + if [ -d $DIST_GEN_PATH ]; then + echo "copy $DIST_GEN_PATH to $DIST_OUT_PATH/" + cp -rf $DIST_GEN_PATH $DIST_OUT_PATH/ + else + echo $DIST_GEN_PATH does not exist + echo "BuildPybind fail" + return 1 + fi +} + +BuildTest() { + cd ${MODULE_TEST_PATH}/ut_gtest + if [ -d "./build" ]; then + rm -rf "./build" + fi + mkdir ./build + cd build + cmake .. && make -j && make install + if [ $? -ne 0 ]; then + echo "BuildTest fail" + return 1 + fi +} + +PrintHelp() { + echo " +./build.sh comm_operator ... +-x Extract the run package +-c Target SOC VERSION + Support Soc: [ascend910_93, ascend910b4] +-d Enable debug +-t enable UT build +-p enable pybind build +-r enable code coverage +" +} + +while getopts "c:xdtprh" opt; do + case $opt in + c) + SOC_VERSION=$OPTARG + ;; + x) + IS_EXTRACT=1 + ;; + d) + export BUILD_TYPE="Debug" + ;; + t) + ENABLE_UT_BUILD=1 + ENABLE_SRC_BUILD=0 + ;; + p) + ENABLE_PYBIND_BUILD=1 + ENABLE_SRC_BUILD=0 + ;; + r) + export BUILD_TYPE="Debug" + export ENABLE_COV=1 + ;; + h) + PrintHelp + exit 0 + ;; + esac +done + +echo "Start creating the CMake file" + +if [ ! -d "$BUILD_OUT_PATH/${MODULE_NAME}" ]; then + echo "mkdir $BUILD_OUT_PATH/${MODULE_NAME}" + mkdir $BUILD_OUT_PATH/${MODULE_NAME} +fi + +if [ $ENABLE_SRC_BUILD -eq 1 ]; then + if [[ "$SOC_VERSION" == "all" ]]; then + echo "$MODULE_SCRIPTS_PATH/compile_ascend_proj.sh $MODULE_SRC_PATH Ascend910_9382 $IS_EXTRACT $BUILD_TYPE" + bash $MODULE_SCRIPTS_PATH/compile_ascend_proj.sh $MODULE_SRC_PATH Ascend910_9382 $IS_EXTRACT $BUILD_TYPE + else + echo "$MODULE_SCRIPTS_PATH/compile_ascend_proj.sh $MODULE_SRC_PATH $SOC_VERSION $IS_EXTRACT $BUILD_TYPE" + bash $MODULE_SCRIPTS_PATH/compile_ascend_proj.sh $MODULE_SRC_PATH $SOC_VERSION $IS_EXTRACT $BUILD_TYPE + fi + if [ $? -ne 0 ]; then + exit 1 + fi +fi + +if [ $ENABLE_PYBIND_BUILD -eq 1 ]; then + echo "Start to BuildPybind" + BuildPybind + if [ $? -ne 0 ]; then + exit 1 + fi +fi + +if [ $ENABLE_UT_BUILD -eq 1 ]; then + echo "Start to BuildTest" + BuildTest + if [ $? -ne 0 ]; then + exit 1 + fi +fi diff --git a/csrc/deepep/compile_ascend_proj.sh b/csrc/deepep/compile_ascend_proj.sh new file mode 100755 index 000000000..a373d70de --- /dev/null +++ b/csrc/deepep/compile_ascend_proj.sh @@ -0,0 +1,82 @@ +#!/bin/bash +CopyOps() { + local src_dir="$1" # Path of Source Directory A + local dst_dir="$2" # Path of Target Directory B + + if [ -d "$src_dir/op_host" ]; then + cp -rf "$src_dir/op_host/"* "$dst_dir/op_host/" + echo "cp -rf $src_dir/op_host/* $dst_dir/op_host/" + fi + + if [ -d "$src_dir/op_kernel" ]; then + cp -rf "$src_dir/op_kernel/"* "$dst_dir/op_kernel/" + echo "cp -rf $src_dir/op_kernel/* $dst_dir/op_kernel/" + fi + + # Ensure that op_host and op_kernel exist in the target directory + mkdir -p "$dst_dir/op_host" "$dst_dir/op_kernel" + + # Traverse all direct subdirectories under the source directory (including directory names with spaces) + find "$src_dir" -mindepth 1 -maxdepth 1 -type d -print0 | while IFS= read -r -d '' subdir; do + # Check whether the subdirectory exists (although find has already filtered it, double-checking is safer) + if [ -d "$subdir" ]; then + # Processing the op_host directory + if [ -d "$subdir/op_host" ]; then + cp -rf "$subdir/op_host/"* "$dst_dir/op_host/" + echo "cp -rf $subdir/op_host/* $dst_dir/op_host/" + fi + + # Processing the op_kernel directory + if [ -d "$subdir/op_kernel" ]; then + cp -rf "$subdir/op_kernel/"* "$dst_dir/op_kernel/" + echo "cp -rf $subdir/op_kernel/* $dst_dir/op_kernel/" + fi + fi + done + echo "${src_dir} built successfully" +} + +BuildOps() { + local proj_name=$1 + echo "${proj_name}" + local soc_version=$2 + + if [ -d "./${proj_name}" ]; then + rm -rf ${proj_name}/cmake + rm -rf ${proj_name}/op_host + rm -rf ${proj_name}/op_kernel + rm -rf ${proj_name}/scripts + rm -rf ${proj_name}/build.sh + rm -rf ${proj_name}/CMakeLists.txt + rm -rf ${proj_name}/CMakePresets.json + rm -rf ${proj_name}/framework + fi + + echo "msopgen gen -i ./AddCustom.json -c ai_core-${soc_version} -f pytorch -lan cpp -out ${proj_name}" + msopgen gen -i ./AddCustom.json -c ai_core-${soc_version} -f pytorch -lan cpp -out ${proj_name} + rm -rf ./${proj_name}/op_host/add_custom* + rm -rf ./${proj_name}/op_kernel/add_custom* +} + +# Build the operator project and transfer its output to the specified location +BuildAscendProj() { + local os_id=$(grep ^ID= /etc/os-release | cut -d= -f2 | tr -d '"') + local arch=$(uname -m) + local soc_version=$2 + local is_extract=$3 + local build_type=$4 + # Modify the default operator name + export OPS_PROJECT_NAME=aclnnInner + # Enter the compilation path + echo "cd $1" + cd $1 + + BuildOps "ops_${soc_version}" ${soc_version} + BuildOps "ops2_${soc_version}" ${soc_version} + CopyOps "./ops" "./ops_${soc_version}" + CopyOps "./ops2" "./ops2_${soc_version}" + cp -r ./ops_${soc_version}/cmake ./ops + cp -r ./ops2_${soc_version}/cmake ./ops2 +} + +BuildAscendProj $1 $2 $3 $4 diff --git a/csrc/deepep/ops/CMakeLists.txt b/csrc/deepep/ops/CMakeLists.txt index b3211072b..ab80c26dd 100644 --- a/csrc/deepep/ops/CMakeLists.txt +++ b/csrc/deepep/ops/CMakeLists.txt @@ -9,10 +9,34 @@ else() message(FATAL_ERROR "Unsupported host processor: ${CMAKE_SYSTEM_PROCESSOR}. Please specify a valid architecture.") endif() +set(CANN82_PATH "${ASCEND_CANN_PACKAGE_PATH}/include/experiment/platform/platform/platform_infos_def.h") +set(CANN83_PATH "${ASCEND_CANN_PACKAGE_PATH}/include/platform/platform_infos_def.h") + +if (EXISTS "${CANN83_PATH}") + set(CANN_VERSION_MACRO "USE_CANN83_PATH") + message(STATUS "Detected CANN 8.3+ include structure, using ${CANN83_PATH}") +elseif (EXISTS "${CANN82_PATH}") + set(CANN_VERSION_MACRO "USE_CANN82_PATH") + message(STATUS "Detected CANN 8.2 include structure, using ${CANN82_PATH}") +else() + message(FATAL_ERROR + "Cannot find 'platform_infos_def.h' in expected locations.\n" + "Please verify that CANN is correctly installed and that either:\n" + " ${CANN83_PATH}\n" + " or\n" + " ${CANN82_PATH}\n" + "exists and contains the header file." + ) +endif() + include(cmake/config.cmake) -include(cmake/func.cmake) +include(func.cmake) include(cmake/intf.cmake) +if (CANN_VERSION_MACRO) + add_compile_definitions(${CANN_VERSION_MACRO}) +endif() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) @@ -39,6 +63,35 @@ add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_ ) add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) +if (DEFINED CANN_VERSION_MACRO AND NOT "${CANN_VERSION_MACRO}" STREQUAL "") + set(CANN_VERSION_FLAG "-D${CANN_VERSION_MACRO}") + message(STATUS "opbuild: Detected CANN_VERSION_MACRO = ${CANN_VERSION_MACRO}") +else() + set(CANN_VERSION_FLAG "") + message(WARNING "opbuild: No CANN_VERSION_MACRO defined! Possible #error in .cc files.") +endif() + +execute_process( + COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 + ${OPBUILD_OPS_SRC} + -D_GLIBCXX_USE_CXX11_ABI=0 + ${CANN_VERSION_FLAG} + -I ${ASCEND_CANN_PACKAGE_PATH}/include + -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel + -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 + -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR +) +if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") +endif() + get_system_info(SYSTEM_INFO) # gen version.info diff --git a/csrc/deepep/ops/cmake/config.cmake b/csrc/deepep/ops/cmake/config.cmake deleted file mode 100644 index d990cd312..000000000 --- a/csrc/deepep/ops/cmake/config.cmake +++ /dev/null @@ -1,42 +0,0 @@ - -set(CMAKE_CXX_FLAGS_DEBUG "") -set(CMAKE_CXX_FLAGS_RELEASE "") - -if (NOT DEFINED CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release CACHE STRING "") -endif() -if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/build_out" CACHE PATH "" FORCE) -endif() -if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) - set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") -endif() -if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) - set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") -endif() -if (NOT DEFINED ASCEND_COMPUTE_UNIT) - set(ASCEND_COMPUTE_UNIT ascend910_93 CACHE STRING "") -endif() -if (NOT DEFINED ENABLE_TEST) - set(ENABLE_TEST FALSE CACHE BOOL "") -endif() -if (NOT DEFINED ENABLE_CROSS_COMPILE) - set(ENABLE_CROSS_COMPILE FALSE CACHE BOOL "") -endif() -if (NOT DEFINED CMAKE_CROSS_PLATFORM_COMPILER) - set(CMAKE_CROSS_PLATFORM_COMPILER "/your/cross/compiler/path" CACHE PATH "") -endif() -if (NOT DEFINED CMAKE_CROSS_LIBRARY_PATH) - set(CMAKE_CROSS_LIBRARY_PATH "" CACHE PATH "") -endif() -if (NOT DEFINED ASCEND_PACK_SHARED_LIBRARY) - set(ASCEND_PACK_SHARED_LIBRARY False CACHE BOOL "") -endif() -set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) -set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) -set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) -file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) -set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") -set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") -file(WRITE ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} "") -file(WRITE ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} "") diff --git a/csrc/deepep/ops/cmake/device_task.cmake b/csrc/deepep/ops/cmake/device_task.cmake deleted file mode 100644 index 3b7c0a136..000000000 --- a/csrc/deepep/ops/cmake/device_task.cmake +++ /dev/null @@ -1,48 +0,0 @@ -message(STATUS "TILING SINK TASK BEGIN") -message(STATUS "TARGET: ${TARGET}") -message(STATUS "OPTION: ${OPTION}") -message(STATUS "SRC: ${SRC}") -message(STATUS "VENDOR: ${VENDOR_NAME}") - -set(CMAKE_CXX_COMPILER ${ASCEND_CANN_PACKAGE_PATH}/toolkit/toolchain/hcc/bin/aarch64-target-linux-gnu-g++) -set(CMAKE_C_COMPILER ${ASCEND_CANN_PACKAGE_PATH}/toolkit/toolchain/hcc/bin/aarch64-target-linux-gnu-gcc) - -string(REPLACE " " ";" SRC "${SRC}") -add_library(${TARGET} ${OPTION} - ${SRC} -) -target_compile_definitions(${TARGET} PRIVATE - DEVICE_OP_TILING_LIB - _FORTIFY_SOURCE=2 - google=ascend_private -) -target_include_directories(${TARGET} PRIVATE - ${ASCEND_CANN_PACKAGE_PATH}/include -) -target_compile_options(${TARGET} PRIVATE - -fPIC - -fstack-protector-strong - -fstack-protector-all - -O2 - -std=c++11 - -fvisibility-inlines-hidden - -fvisibility=hidden -) -target_link_libraries(${TARGET} PRIVATE - -Wl,--whole-archive - device_register - c_sec - mmpa - tiling_api - platform_static - ascend_protobuf - exe_meta_device - -Wl,--no-whole-archive -) -target_link_directories(${TARGET} PRIVATE - ${ASCEND_CANN_PACKAGE_PATH}/lib64/device/lib64 - ${ASCEND_CANN_PACKAGE_PATH}/compiler/lib64 -) -set_target_properties(${TARGET} PROPERTIES - OUTPUT_NAME cust_opmaster -) diff --git a/csrc/deepep/ops/cmake/intf.cmake b/csrc/deepep/ops/cmake/intf.cmake deleted file mode 100644 index d2643bbc7..000000000 --- a/csrc/deepep/ops/cmake/intf.cmake +++ /dev/null @@ -1,28 +0,0 @@ - -add_library(intf_pub INTERFACE) -target_compile_options(intf_pub INTERFACE - -fPIC - -fvisibility=hidden - -fvisibility-inlines-hidden - $<$:-O2> - $<$:-O0 -g> - $<$:-std=c++11> - $<$,$>:-ftrapv -fstack-check> - $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> - $,-fstack-protector-strong,-fstack-protector-all> -) -target_compile_definitions(intf_pub INTERFACE - _GLIBCXX_USE_CXX11_ABI=0 - $<$:_FORTIFY_SOURCE=2> -) -target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include - ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel -) -target_link_options(intf_pub INTERFACE - $<$,EXECUTABLE>:-pie> - $<$:-s> - -Wl,-z,relro - -Wl,-z,now - -Wl,-z,noexecstack -) -target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/csrc/deepep/ops/cmake/makeself.cmake b/csrc/deepep/ops/cmake/makeself.cmake deleted file mode 100644 index 82c8da7eb..000000000 --- a/csrc/deepep/ops/cmake/makeself.cmake +++ /dev/null @@ -1,31 +0,0 @@ -execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh - --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh - --help-header ./help.info - --gzip --complevel 4 --nomd5 --sha256 - ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - RESULT_VARIABLE EXEC_RESULT - ERROR_VARIABLE EXEC_ERROR -) - -if (NOT "${EXEC_RESULT}x" STREQUAL "0x") - message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") -endif() - -execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ - COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - ) - -if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") - execute_process( - COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - ) - - execute_process( - COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ - COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - ) -endif() diff --git a/csrc/deepep/ops/cmake/util/__init__.py b/csrc/deepep/ops/cmake/util/__init__.py deleted file mode 100755 index 364083fab..000000000 --- a/csrc/deepep/ops/cmake/util/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(PYF_PATH) diff --git a/csrc/deepep/ops/cmake/util/ascendc_bin_param_build.py b/csrc/deepep/ops/cmake/util/ascendc_bin_param_build.py deleted file mode 100755 index 388575cde..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_bin_param_build.py +++ /dev/null @@ -1,531 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import copy -import hashlib -import json -import os -import re -import sys -from collections import defaultdict -from typing import Dict, List, NamedTuple, Set, Tuple - -import const_var -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class ParamInfo(NamedTuple): - dtype_list: list - format_list: list - dtype_for_bin_list: dict - format_for_bin_list: dict - - -class BinParamBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - self.soc = "" - self.out_path = "" - self.tiling_keys = set() - self.op_debug_config = "" - - def set_soc_version(self: any, soc: str): - self.soc = soc - - def set_out_path(self: any, out_path: str): - self.out_path = out_path - - def set_tiling_key(self: any, tiling_key_info: Set): - if tiling_key_info: - self.tiling_keys.update(tiling_key_info) - - def set_op_debug_config(self: any, op_debug_config: str): - if op_debug_config: - self.op_debug_config = op_debug_config - - def get_full_list(self: any): - dtype_list = [] - for dtype_in in self.input_dtype: - dtype_list.append(dtype_in.split(",")) - for dtype_out in self.output_dtype: - dtype_list.append(dtype_out.split(",")) - - format_list = [] - for fmt_in in self.input_fmt: - format_list.append(fmt_in.split(",")) - for fmt_out in self.output_fmt: - format_list.append(fmt_out.split(",")) - - dtype_for_bin_list = [ - [] for _ in range(len(self.input_dtype) + len(self.output_dtype)) - ] - format_for_bin_list = copy.deepcopy(dtype_for_bin_list) - - for key, value in self.input_dtype_for_bin.items(): - dtype_for_bin_list[key] = value.split(",") - for key, value in self.output_dtype_for_bin.items(): - dtype_for_bin_list[key + len(self.input_dtype)] = value.split(",") - for key, value in self.input_fmt_for_bin.items(): - format_for_bin_list[key] = value.split(",") - for key, value in self.output_fmt_for_bin.items(): - format_for_bin_list[key + len(self.input_dtype)] = value.split(",") - - return ParamInfo( - dtype_list, format_list, dtype_for_bin_list, format_for_bin_list - ) - - def gen_bin_cprs_list(self: any, param_info: ParamInfo): - combine_dict = {} - origin_combine_dict = {} - for cob_idx in range(0, len(self.input_dtype[0].split(","))): - origin_combine = "" - combine = "" - for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): - if param_info.dtype_for_bin_list[param_idx]: - combine += param_info.dtype_for_bin_list[param_idx][cob_idx] - else: - combine += param_info.dtype_list[param_idx][cob_idx] - origin_combine += param_info.dtype_list[param_idx][cob_idx] - if param_info.format_for_bin_list[param_idx]: - combine += param_info.format_for_bin_list[param_idx][cob_idx] - else: - combine += param_info.format_list[param_idx][cob_idx] - origin_combine += param_info.format_list[param_idx][cob_idx] - if combine not in combine_dict: - combine_dict[combine] = [] - combine_dict[combine].append(cob_idx) - origin_combine_dict[origin_combine] = cob_idx - for key, value in combine_dict.items(): - if key not in origin_combine_dict: - print(f"WARNING: ForBinQuery {key} not in origin combine") - self.bin_save_list += value - continue - if len(value) == 1 and value[0] == origin_combine_dict[key]: - self.bin_save_list += value - continue - self.bin_cprs_head.append(origin_combine_dict[key]) - self.bin_cprs_list.append(value) - for index, sub_list in enumerate(self.bin_cprs_list): - if self.bin_cprs_head[index] not in self.bin_save_list: - continue - sub_list.append(self.bin_cprs_head[index]) - self.bin_save_list += self.bin_cprs_head - - def gen_for_bin_list(self: any, param_info: ParamInfo): - combine_size = len(self.input_dtype[0].split(",")) - input_size = len(self.input_dtype) - output_size = len(self.output_dtype) - - self.input_dtype_for_bin_list = [[] for _ in range(input_size)] - self.output_dtype_for_bin_list = [[] for _ in range(output_size)] - for i in range(0, input_size): - self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] - for i in range(0, output_size): - self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] - self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) - self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) - - for index, sub_list in enumerate(self.bin_cprs_list): - head_idx = self.bin_cprs_head[index] - for cmb_idx in sub_list: - for i in range(0, input_size): - self.input_dtype_for_bin_list[i][head_idx].append( - param_info.dtype_list[i][cmb_idx] - ) - self.input_fmt_for_bin_list[i][head_idx].append( - param_info.format_list[i][cmb_idx] - ) - for i in range(0, output_size): - self.output_dtype_for_bin_list[i][head_idx].append( - param_info.dtype_list[i + input_size][cmb_idx] - ) - self.output_fmt_for_bin_list[i][head_idx].append( - param_info.format_list[i + input_size][cmb_idx] - ) - - def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): - for i in range(0, input_size): - self.input_dtype_for_bin_list[i] = [ - element - for index, element in enumerate(self.input_dtype_for_bin_list[i]) - if index in self.bin_save_list - ] - self.input_fmt_for_bin_list[i] = [ - element - for index, element in enumerate(self.input_fmt_for_bin_list[i]) - if index in self.bin_save_list - ] - new_dtype_list = [ - element - for index, element in enumerate(dtype_list[i]) - if index in self.bin_save_list - ] - new_dtype_str = "" - for dtype in new_dtype_list: - new_dtype_str += f"{dtype}," - self.input_dtype[i] = new_dtype_str[:-1] - new_format_list = [ - element - for index, element in enumerate(format_list[i]) - if index in self.bin_save_list - ] - new_format_str = "" - for fmt in new_format_list: - new_format_str += f"{fmt}," - self.input_fmt[i] = new_format_str[:-1] - for i in range(0, output_size): - self.output_dtype_for_bin_list[i] = [ - element - for index, element in enumerate(self.output_dtype_for_bin_list[i]) - if index in self.bin_save_list - ] - self.output_fmt_for_bin_list[i] = [ - element - for index, element in enumerate(self.output_fmt_for_bin_list[i]) - if index in self.bin_save_list - ] - new_dtype_list = [ - element - for index, element in enumerate(dtype_list[i + input_size]) - if index in self.bin_save_list - ] - new_dtype_str = "" - for dtype in new_dtype_list: - new_dtype_str += f"{dtype}," - self.output_dtype[i] = new_dtype_str[:-1] - new_format_list = [ - element - for index, element in enumerate(format_list[i + input_size]) - if index in self.bin_save_list - ] - new_format_str = "" - for fmt in new_format_list: - new_format_str += f"{fmt}," - self.output_fmt[i] = new_format_str[:-1] - - def is_set_for_bin_query(self: any): - return any( - [ - self.input_dtype_for_bin, - self.output_dtype_for_bin, - self.input_fmt_for_bin, - self.output_fmt_for_bin, - ] - ) - - def for_bin_list_match(self: any): - if not self.is_set_for_bin_query(): - return - input_size = len(self.input_dtype) - output_size = len(self.output_dtype) - param_info = self.get_full_list() - self.gen_bin_cprs_list(param_info) - self.gen_for_bin_list(param_info) - if len(self.bin_save_list) == len(self.input_dtype[0].split(",")): - print( - f"WARNING: ForBinQuery can not compress number of bin file with this set, please check!!." - ) - return - self.rm_cprs_cmb( - param_info.dtype_list, param_info.format_list, input_size, output_size - ) - - def gen_input_json(self: any, auto_gen_path: str): - key_map = {} - self.for_bin_list_match() - count = len(self.input_dtype[0].split(",")) - required_parameters = set() - index_value = -1 - - for i in range(0, count): - inputs = [] - outputs = [] - attrs = [] - required_parameter = [] - op_node = {} - - for idx in range(0, len(self.input_name)): - idtypes = self.input_dtype[idx].split(",") - ifmts = self.input_fmt[idx].split(",") - itype = self.input_type[idx] - para = {} - para["name"] = self.input_name[idx][:-5] - para["index"] = idx - para["dtype"] = idtypes[i] - if ( - self.is_set_for_bin_query() - and self.input_dtype_for_bin_list[idx][i] - ): - para["dtypeForBinQuery"] = self.input_dtype_for_bin_list[idx][i] - para["format"] = ifmts[i] - if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: - para["formatForBinQuery"] = self.input_fmt_for_bin_list[idx][i] - para["paramType"] = itype - para["shape"] = [-2] - para["format_match_mode"] = "FormatAgnostic" - - input_parameter_key = (idtypes[i], ifmts[i]) - if itype == "dynamic": - inputs.append([para]) - required_parameter.append(input_parameter_key) - elif itype == "required": - inputs.append(para) - required_parameter.append(input_parameter_key) - else: - inputs.append(para) - - for idx in range(0, len(self.output_name)): - odtypes = self.output_dtype[idx].split(",") - ofmts = self.output_fmt[idx].split(",") - otype = self.output_type[idx] - para = {} - para["name"] = self.output_name[idx][:-5] - para["index"] = idx - para["dtype"] = odtypes[i] - if ( - self.is_set_for_bin_query() - and self.output_dtype_for_bin_list[idx][i] - ): - para["dtypeForBinQuery"] = self.output_dtype_for_bin_list[idx][i] - para["format"] = ofmts[i] - if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: - para["formatForBinQuery"] = self.output_fmt_for_bin_list[idx][i] - para["paramType"] = otype - para["shape"] = [-2] - para["format_match_mode"] = "FormatAgnostic" - output_parameter_key = (odtypes[i], ofmts[i]) - if otype == "dynamic": - outputs.append([para]) - required_parameter.append(output_parameter_key) - elif otype == "required": - outputs.append(para) - required_parameter.append(output_parameter_key) - else: - outputs.append(para) - - for attr in self.attr_list: - att = {} - att["name"] = attr - atype = self.attr_val.get(attr).get("type").lower() - att["dtype"] = atype - att["value"] = const_var.ATTR_DEF_VAL.get(atype) - attrs.append(att) - - required_parameter_tuple = tuple(required_parameter) - if required_parameter_tuple in required_parameters: - continue - else: - required_parameters.add(required_parameter_tuple) - index_value += 1 - - op_node["bin_filename"] = "" - op_node["inputs"] = inputs - op_node["outputs"] = outputs - if len(attrs) > 0: - op_node["attrs"] = attrs - - param = {} - param["op_type"] = self.op_type - param["op_list"] = [op_node] - objstr = json.dumps(param, indent=" ") - md5sum = hashlib.md5(objstr.encode("utf-8")).hexdigest() - while key_map.get(md5sum) is not None: - objstr += "1" - md5sum = hashlib.md5(objstr.encode("utf-8")).hexdigest() - key_map[md5sum] = md5sum - bin_file = self.op_type + "_" + md5sum - op_node["bin_filename"] = bin_file - param_file = os.path.join(self.out_path, bin_file + "_param.json") - param_file = os.path.realpath(param_file) - with os.fdopen( - os.open(param_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - json.dump(param, fd, indent=" ") - self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) - - def _write_build_cmd( - self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str - ): - hard_soc = const_var.conv_soc_ver(self.soc) - if not hard_soc: - hard_soc = self.soc.capitalize() - name_com = [self.op_type, self.op_file, str(index)] - compile_file = os.path.join(self.out_path, "-".join(name_com) + ".sh") - compile_file = os.path.realpath(compile_file) - - bin_cmd_str = "res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ - --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic " - - build_cmd_var = "#!/bin/bash\n" - build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' - plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") - plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") - if plog_level is None: - build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR - if plog_stdout is None: - build_cmd_var += const_var.SET_PLOG_STDOUT - build_cmd_var += const_var.SRC_ENV - if hard_soc == "Ascend610Lite": - build_cmd_var += f"export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n" - build_cmd_var += bin_cmd_str.format( - fun=self.op_intf, - soc=hard_soc, - param=param_file, - impl="high_performance,optional", - ) - enable_tiling_keys = False - if self.tiling_keys: - tiling_keys_list = sorted(list(self.tiling_keys)) - tiling_key_str = ",".join([str(_key) for _key in tiling_keys_list]) - build_cmd_var += f' --tiling_key="{tiling_key_str}"' - enable_tiling_keys = True - - if self.op_debug_config: - op_debug_str = ",".join([str(_key) for _key in list(self.op_debug_config)]) - build_cmd_var += f" --op_debug_config={op_debug_str}" - - build_cmd_var += ")\n" - build_cmd_var += "\n" - if enable_tiling_keys is False: - build_cmd_var += 'echo "${res}"\n' - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".json") - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".o") - else: - build_cmd_var += "if [ $? -eq 1 ]; then\n" - build_cmd_var += ' if echo "${res}" | \ -grep -q "None of the given tiling keys are in the supported list"; then\n' - build_cmd_var += ' echo "${res}"\n' - build_cmd_var += " else\n" - build_cmd_var += ' echo "${res}"\n' - build_cmd_var += " exit 1\n" - build_cmd_var += " fi\n" - build_cmd_var += "else\n" - build_cmd_var += 'echo "${res}"\n' - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".json") - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".o") - build_cmd_var += "fi\n" - build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' - - with os.fdopen( - os.open(compile_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - fd.write(build_cmd_var) - - -def get_tiling_keys(tiling_keys: str) -> Set: - all_tiling_keys = set() - if not tiling_keys: - return all_tiling_keys - - tiling_key_list = tiling_keys.split(";") - for tiling_key_value in tiling_key_list: - pattern = r"(? int(end): - continue - for i in range(int(start), int(end) + 1): - all_tiling_keys.add(i) - elif tiling_key_value.isdigit(): - all_tiling_keys.add(int(tiling_key_value)) - return all_tiling_keys - - -def trans_soc_verion(soc_ver: str): - low_soc_ver = soc_ver.lower() - if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: - return low_soc_ver - return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] - - -def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: - tiling_key_info = defaultdict(set) - op_debug_config = defaultdict(set) - if not opc_config_file: - return tiling_key_info, op_debug_config - - if not os.path.exists(opc_config_file): - return tiling_key_info, op_debug_config - - with open(opc_config_file, "r") as file: - contents = file.readlines() - - for _content in contents: - content = _content.strip() - opc_configs = content.split("@") - if len(opc_configs) < 3: - continue - - op_type = opc_configs[0] - if not op_type: - continue - - compute_unit = opc_configs[1] - if compute_unit: - compute_unit_list = compute_unit.split(";") - soc_lists = [] - for soc_ver in compute_unit_list: - short_soc_ver = trans_soc_verion(soc_ver) - soc_lists.append(short_soc_ver) - if soc not in soc_lists: - continue - - for options in opc_configs[2:]: - if "--tiling_key" in options: - format_tiling_keys = get_tiling_keys(options.split("=")[1]) - if format_tiling_keys: - tiling_key_info[op_type].update(format_tiling_keys) - if "--op_debug_config" in options: - format_debug_config = set(options.split("=")[1].split(";")) - if format_debug_config: - op_debug_config[op_type].update(format_debug_config) - - return tiling_key_info, op_debug_config - - -def gen_bin_param_file( - cfgfile: str, out_dir: str, soc: str, opc_config_file: str = "", ops: list = None -): - if not os.path.exists(cfgfile): - print( - f"INFO: {cfgfile} does not exists in this project, skip generating compile commands." - ) - return - - op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) - tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) - auto_gen_path_dir = os.path.dirname(cfgfile) - all_soc_key = "ALL" - for op_desc in op_descs: - op_desc.set_soc_version(soc) - op_desc.set_out_path(out_dir) - if op_desc.op_type in op_debug_config: - op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) - if all_soc_key in op_debug_config: - op_desc.set_op_debug_config(op_debug_config[all_soc_key]) - if op_desc.op_type in tiling_key_info: - op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) - if all_soc_key in tiling_key_info: - op_desc.set_tiling_key(tiling_key_info[all_soc_key]) - op_desc.gen_input_json(auto_gen_path_dir) - - -def parse_args(argv): - """Command line parameter parsing""" - parser = argparse.ArgumentParser() - parser.add_argument("argv", nargs="+") - parser.add_argument("--opc-config-file", nargs="?", const="", default="") - return parser.parse_args(argv) - - -if __name__ == "__main__": - args = parse_args(sys.argv) - if len(args.argv) <= 3: - raise RuntimeError("arguments must greater than 3") - gen_bin_param_file( - args.argv[1], args.argv[2], args.argv[3], opc_config_file=args.opc_config_file - ) diff --git a/csrc/deepep/ops/cmake/util/ascendc_compile_kernel.py b/csrc/deepep/ops/cmake/util/ascendc_compile_kernel.py deleted file mode 100755 index c4c815765..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_compile_kernel.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import os -import shutil -import subprocess -import sys -import time - -import ascendc_bin_param_build -import ascendc_impl_build -import ascendc_op_info -import const_var - - -class CompileKernel: - def __init__(self: any, args: any): - self.op_type = args.op_name - self.op_cpp_file = os.path.realpath(args.src_file) - self.op_soc_ver = args.compute_unit - self.compile_options = args.compile_options - self.op_debug_config = args.debug_config - self.op_cfg_ini = os.path.realpath(args.config_ini) - self.op_tiling = os.path.realpath(args.tiling_lib) - self.op_output = os.path.realpath(args.output_path) - self.op_impl_py = None - self.compile_sh = [] - self.working_dir = os.path.join( - os.getcwd(), - self.op_type + "_" + self.op_soc_ver, - ) - self.build_opp_path = os.path.join(self.working_dir, "customize") - os.makedirs(self.working_dir) - os.makedirs(self.op_output, exist_ok=True) - if args.dynamic_dir is not None and args.dynamic_dir != "": - self.dynamic_dir = os.path.realpath(args.dynamic_dir) - else: - self.dynamic_dir = None - if args.json_file is not None and args.json_file != "": - self.json_file = args.json_file - else: - self.json_file = None - - def clean(self: any): - if "dump_cce" not in self.op_debug_config: - shutil.rmtree(self.working_dir) - return - - def ascendc_gen_impl(self: any): - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = "" - rep_cfg[const_var.REPLAY_ITERATE] = "" - cfg_dir = {} - cfg_dir[const_var.CFG_IMPL_DIR] = os.path.dirname(self.op_cpp_file) - cfg_dir[const_var.CFG_OUT_DIR] = os.path.join(self.working_dir, "dynamic") - os.makedirs(os.path.join(self.working_dir, "dynamic"), exist_ok=True) - cfg_dir[const_var.AUTO_GEN_DIR] = os.path.dirname(self.op_cfg_ini) - ascendc_impl_build.write_scripts( - self.op_cfg_ini, rep_cfg, cfg_dir, [self.op_type], self.compile_options - ) - py_files = glob.glob(os.path.join(self.working_dir, "dynamic", "*.py")) - if py_files is None or len(py_files) != 1: - self.clean() - raise RuntimeError("compile py file {} generated error!".format(py_files)) - self.op_impl_py = os.path.join( - self.working_dir, "dynamic", self.op_type + ".py" - ) - if self.dynamic_dir is not None: - shutil.copy(py_files[0], self.dynamic_dir) - os.rename(py_files[0], self.op_impl_py) - if not os.path.exists(self.op_impl_py): - self.clean() - raise RuntimeError( - "compile py file {} not generated!".format(self.op_impl_py) - ) - - def ascendc_gen_param(self: any): - bin_param_path = os.path.join(self.working_dir, "bin_param") - os.makedirs(bin_param_path) - base_dir = os.path.dirname(self.op_cfg_ini) - opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") - ascendc_bin_param_build.gen_bin_param_file( - self.op_cfg_ini, - bin_param_path, - self.op_soc_ver, - opc_config_file, - [self.op_type], - ) - tiling_key_info, op_debug_config = ascendc_bin_param_build.parse_op_debug_confg( - opc_config_file, self.op_type - ) - if self.op_type in op_debug_config: - self.op_debug_config = op_debug_config[self.op_type] - if "ALL" in op_debug_config: - self.op_debug_config = op_debug_config["ALL"] - bin_param_files = glob.glob(os.path.join(bin_param_path, "*.json")) - if bin_param_files is None or len(bin_param_files) <= 0: - self.clean() - raise RuntimeError("compile binary param json file not generated!") - self.compile_sh = glob.glob(os.path.join(bin_param_path, "*.sh")) - if self.compile_sh is None or len(self.compile_sh) != len(bin_param_files): - self.clean() - raise RuntimeError("compile binary shell file not generated!") - - def ascendc_put_tiling(self: any): - tiling_path = os.path.join( - self.build_opp_path, "op_impl", "ai_core", "tbe", "op_tiling" - ) - os.makedirs(tiling_path) - tiling_so = os.path.join(tiling_path, "liboptiling.so") - os.symlink(self.op_tiling, tiling_so) - if not os.path.exists(tiling_so): - self.clean() - raise RuntimeError("prepare tiling lib {} link failed!".format(tiling_so)) - - def ascendc_put_json(self: any): - if self.json_file is not None: - json_file_dir = os.path.join( - self.build_opp_path, - "op_impl", - "ai_core", - "tbe", - "config", - self.op_soc_ver, - ) - os.makedirs(json_file_dir) - shutil.copy(self.json_file, json_file_dir) - build_json_file = os.path.join( - json_file_dir, "aic-{}-ops-info.json".format(self.op_soc_ver) - ) - if not os.path.exists(build_json_file): - self.clean() - raise RuntimeError( - "prepare json file aic-{}-ops-info.json failed!".format( - self.op_soc_ver - ) - ) - - def ascendc_build(self: any): - op_info = ascendc_op_info.OpInfo(self.op_type, self.op_cfg_ini) - op_file = op_info.get_op_file() - op_bin_dir = os.path.join(self.op_output, self.op_soc_ver, op_file) - os.makedirs(op_bin_dir, exist_ok=True) - all_tar = [] - sub_cmd = [] - index = 0 - for sh in self.compile_sh: - tar = op_file + str(index) - build_path = os.path.join(self.working_dir, "kernel_" + str(index)) - os.makedirs(build_path) - all_tar.append(tar) - sub_cmd.append(tar + ":") - sub_cmd.append( - "\tcd {} && bash {} --kernel-src=$(CPP) $(PY) $(OUT) $(MAKE)".format( - build_path, sh - ) - ) - index += 1 - mkfile = os.path.join(self.working_dir, op_file + ".make") - with os.fdopen(os.open(mkfile, const_var.WFLAGS, const_var.WMODES), "w") as fd: - sub_cmd.insert(0, "all: " + " ".join(all_tar)) - fd.write("\n".join(sub_cmd)) - - if os.getenv("TILINGKEY_PAR_COMPILE") is None: - cmd_str = ( - "export HI_PYTHON=python3 && export ASCEND_CUSTOM_OPP_PATH={} && export TILINGKEY_PAR_COMPILE=1" - "&& make -f {} PY={} OUT={} CPP={}" - ) - else: - cmd_str = "export HI_PYTHON=python3 && export ASCEND_CUSTOM_OPP_PATH={} && make -f {} PY={} OUT={} CPP={}" - - if ( - os.system( - cmd_str.format( - self.build_opp_path, - mkfile, - self.op_impl_py, - op_bin_dir, - self.op_cpp_file, - ) - ) - != 0 - ): - raise RuntimeError( - "Kernel Compilation Error: OpType {} Kernel File {}!".format( - self.op_type, self.op_cpp_file - ) - ) - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-n", "--op-name", nargs="?", help="Op name(Camel string) to compile." - ) - parser.add_argument("-s", "--src-file", nargs="?", help="Op kernel source file.") - - parser.add_argument("-u", "--compute-unit", nargs="?", help="Compute unit.") - parser.add_argument( - "-c", "--compile-options", nargs="?", help="Compile options of compiler." - ) - parser.add_argument( - "-d", - "--debug-config", - nargs="?", - help="Debug config of op, ref opc op-debug-config.", - ) - parser.add_argument("-i", "--config-ini", nargs="?", help="Op config ini file.") - parser.add_argument( - "-t", "--tiling-lib", nargs="?", help="Tiling shared library file." - ) - - parser.add_argument( - "-o", "--output-path", nargs="?", help="Output path of compile result." - ) - parser.add_argument( - "-dy", - "--dynamic-dir", - nargs="?", - default=None, - help="dynamic path of source compile.", - ) - parser.add_argument( - "-eb", - "--enable-binary", - nargs="?", - default=None, - help="whether binary compile is enabled.", - ) - parser.add_argument( - "-j", - "--json-file", - nargs="?", - default=None, - help="aic--ops-info.json file path.", - ) - # $(MAKE) is necessary for parallel compiling - parser.add_argument( - "-b", "--build-tool", nargs="?", default=None, help="build tool must be make." - ) - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - kernel_builder = CompileKernel(args) - kernel_builder.clean() - if args.enable_binary == "False": - kernel_builder.ascendc_gen_impl() - kernel_builder.clean() - else: - kernel_builder.ascendc_gen_impl() - kernel_builder.ascendc_gen_param() - kernel_builder.ascendc_put_json() - kernel_builder.ascendc_put_tiling() - kernel_builder.ascendc_build() - kernel_builder.clean() diff --git a/csrc/deepep/ops/cmake/util/ascendc_gen_options.py b/csrc/deepep/ops/cmake/util/ascendc_gen_options.py deleted file mode 100755 index 637ae9d79..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_gen_options.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -import json -import os -import re -import stat -import sys - -import const_var - - -def write_options_to_file( - file_name: str, options_str: str, op_type: str, compute_unit: str, split_char: str -): - flags = os.O_WRONLY | os.O_CREAT - modes = stat.S_IWUSR | stat.S_IRUSR - try: - with os.fdopen(os.open(file_name, flags, modes), "a") as fd: - fd.write( - op_type + split_char + compute_unit + split_char + options_str + "\n" - ) - except Exception as err: - print("write compile options config file failed") - raise (err) - - -def gen_compile_options( - compile_options_file: str, op_type: str, compute_unit: str, compile_options: list -): - base_dir = os.path.dirname(compile_options_file) - opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") - compile_opt = [] - opc_debug_config = [] - opc_tiling_keys = "" - for opts in compile_options: - if "oom" in opts: - if opts == "--oom": - opc_debug_config.append("oom") - else: - raise RuntimeError(f"Unknown oom option format {opts}") - elif "--save-temp-files" in opts: - opc_debug_config.append("dump_cce") - elif "--tiling_key" in opts: - keys = opts.strip().split("=")[1].split(",") - keys_str = ";".join([key for key in keys]) - opc_tiling_keys = keys_str - else: - compile_opt.append(opts) - if len(compile_opt) > 0: - options_str = ";".join([opt for opt in compile_opt]) - write_options_to_file( - compile_options_file, options_str, op_type, compute_unit, "," - ) - opc_config_str = "" - if opc_debug_config: - opc_config_str = "--op_debug_config=" + ";".join( - [opt for opt in opc_debug_config] - ) - if len(opc_tiling_keys) > 0: - if opc_config_str != "": - opc_config_str += "@" - opc_config_str += "--tiling_key=" + opc_tiling_keys - - if opc_config_str != "": - write_options_to_file( - opc_config_file, opc_config_str, op_type, compute_unit, "@" - ) - - -if __name__ == "__main__": - if len(sys.argv) < 4: - raise RuntimeError("arguments must greater than 4") - compute_soc = "" - comp_options = [] - for i in range(len(sys.argv) - 3): - if sys.argv[i + 3].upper().startswith("ASCEND"): - compute_soc += sys.argv[i + 3] + ";" - else: - comp_options.append(sys.argv[i + 3]) - if compute_soc != "": - compute_soc = compute_soc[0:-1] - gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) diff --git a/csrc/deepep/ops/cmake/util/ascendc_get_op_name.py b/csrc/deepep/ops/cmake/util/ascendc_get_op_name.py deleted file mode 100755 index 5da592b3b..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_get_op_name.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import configparser - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--ini-file", help="op info ini.") - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - op_config = configparser.ConfigParser() - op_config.read(args.ini_file) - for section in op_config.sections(): - print(section, end="-") - print(op_config.get(section, "opFile.value"), end="\n") diff --git a/csrc/deepep/ops/cmake/util/ascendc_impl_build.py b/csrc/deepep/ops/cmake/util/ascendc_impl_build.py deleted file mode 100755 index 7486ad138..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_impl_build.py +++ /dev/null @@ -1,748 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import datetime -import glob -import os -import re -import sys -from typing import List - -import const_var -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -IMPL_HEAD = '''#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. -""" - -import os, sys -import ctypes -import json -import shutil -from tbe.common.platform import get_soc_spec -from tbe.common.utils import para_check -from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo -from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel -from tbe.common.buildcfg import get_default_build_config -from impl.util.platform_adapter import tbe_register -from tbe.common.buildcfg import get_current_build_config -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], - "float16": ["DT_FLOAT16", "half"], - "int8": ["DT_INT8", "int8_t"], - "int16": ["DT_INT16", "int16_t"], - "int32": ["DT_INT32", "int32_t"], - "int64": ["DT_INT64", "int64_t"], - "uint1": ["DT_UINT1", "uint8_t"], - "uint8": ["DT_UINT8", "uint8_t"], - "uint16": ["DT_UINT16", "uint16_t"], - "uint32": ["DT_UINT32", "uint32_t"], - "uint64": ["DT_UINT64", "uint64_t"], - "bool": ["DT_BOOL", "bool"], - "double": ["DT_DOUBLE", "double"], - "dual": ["DT_DUAL", "unknown"], - "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], - "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], - "string": ["DT_STRING", "unknown"], - "complex32": ["DT_COMPLEX32", "unknown"], - "complex64": ["DT_COMPLEX64", "unknown"], - "complex128": ["DT_COMPLEX128", "unknown"], - "qint8": ["DT_QINT8", "unknown"], - "qint16": ["DT_QINT16", "unknown"], - "qint32": ["DT_QINT32", "unknown"], - "quint8": ["DT_QUINT8", "unknown"], - "quint16": ["DT_QUINT16", "unknown"], - "resource": ["DT_RESOURCE", "unknown"], - "string_ref": ["DT_STRING_REF", "unknown"], - "int4": ["DT_INT4", "int4b_t"], - "bfloat16": ["DT_BF16", "bfloat16_t"]}} - -def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): - options = [] - x_fmt = x.get("format") - x_dtype = x.get("dtype") - x_n_in_kernel = x_n + '_REF' if is_ref else x_n - options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) - options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) - options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) - return options - -def get_dtype_fmt_options(__inputs__, __outputs__): - options = [] - input_names = {} - output_names = {} - unique_param_name_set = set() - for idx, x in enumerate(__inputs__): - if x is None: - continue - x_n = input_names[idx].upper() - unique_param_name_set.add(x_n) - options += add_dtype_fmt_option_single(x, x_n) - - for idx, x in enumerate(__outputs__): - if x is None: - continue - x_n = output_names[idx].upper() - if x_n in unique_param_name_set: - options += add_dtype_fmt_option_single(x, x_n, True) - else: - options += add_dtype_fmt_option_single(x, x_n) - return options - -def load_dso(so_path): - try: - ctypes.CDLL(so_path) - except OSError as error : - CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) - raise RuntimeError("cannot open %s" %(so_path)) - else: - msg = "load so succ " + so_path - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - -def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): - compile_options = [] - if shortsoc in compile_option_list: - compile_options.extend(compile_option_list[shortsoc]) - if '__ALLSOC__' in compile_option_list: - compile_options.extend(compile_option_list['__ALLSOC__']) - return compile_options - -def get_kernel_source(src_file, dir_snake, dir_ex): - src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) - if os.path.exists(src_ex): - return src_ex - src = os.environ.get('BUILD_KERNEL_SRC') - if src and os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, src_file) - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) - if os.path.exists(src): - return src - return src_ex - -''' - -IMPL_API = """ -@tbe_register.register_operator("{}", trans_bool_to_s8=False) -@para_check.check_op_params({}) -def {}({}, kernel_name="{}", impl_mode=""): -{} - if get_current_build_config("enable_op_prebuild"): - return - __inputs__, __outputs__, __attrs__ = _build_args({}) - options = get_dtype_fmt_options(__inputs__, __outputs__) - options += ["-x", "cce"] - bisheng = os.environ.get('BISHENG_REAL_PATH') - if bisheng is None: - bisheng = shutil.which("bisheng") - if bisheng != None: - bisheng_path = os.path.dirname(bisheng) - tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) - else: - tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") - options.append("-I" + tikcpp_path) - options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) - options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) - if impl_mode == "high_performance": - options.append("-DHIGH_PERFORMANCE=1") - elif impl_mode == "high_precision": - options.append("-DHIGH_PRECISION=1") - if get_current_build_config("enable_deterministic_mode") == 1: - options.append("-DDETERMINISTIC_MODE=1") - else: - options.append("-DDETERMINISTIC_MODE=0") - - custom_compile_options = {}, - custom_all_compile_options = {}, - soc_version = get_soc_spec("SOC_VERSION") - soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() - custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) - custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) - options += custom_all_compile_options_soc - options += custom_compile_options_soc - - origin_func_name = "{}" - ascendc_src_dir_ex = "{}" - ascendc_src_dir = "{}" - ascendc_src_file = "{}" - src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) -""" - -REPLAY_OP_API = """ - msg = "start replay Ascend C Operator {}, kernel name is {}" - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" - tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version - msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" - replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" - if PYF_PATH.endswith("dynamic"): - op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") - else: - op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") - replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") - load_dso(codegen_so_path) - load_dso(replaystub_so_path) - load_dso(replayapi_so_path) - op_type = "{}" - entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) - res, msg = replay_op(op_info, entry_obj, code_channel, src, options) - if not res: - print("call replay op failed for %s and get into call compile op" %(msg)) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -""" - -COMPILE_OP_API = """ - msg = "start compile Ascend C Operator {}, kernel name is " + kernel_name - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - op_type = "{}" - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ - param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ - output_shape_depend_on_compute = {}) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -""" - -COMPILE_OP_API_BUILT_IN = """ - msg = "start compile Ascend C Operator {}, kernel name is " + kernel_name - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - op_type = "{}" - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ - param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ - output_shape_depend_on_compute = {}) - - op_compile_option = '{}' - opp_path = os.environ.get('ASCEND_OPP_PATH') - dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) - if opp_path and os.path.exists(dat_path): - # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op - abs_rel_kernel_src_path = "{}" - extend_options = {{}} - extend_options['opp_kernel_hidden_dat_path'] = dat_path - compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ - extend_options) - else: - raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) -""" - -SUP_API = """ -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - ret_dict = json.loads(ret_str) - err_code = ret_dict.get("ret_code") - sup = "Unknown" - reason = "Unknown reason" - if err_code is not None: - if err_code == 0: - sup = "True" - reason = "" - elif err_code == 1: - sup = "False" - reason = ret_dict.get("reason") - else: - sup = "Unknown" - reason = ret_dict.get("reason") - return sup, reason -""" -CAP_API = """ -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - return result.decode("utf-8") -""" -GLZ_API = """ -@tbe_register.register_param_generalization("{}") -def {}_generalization({}, generalize_config=None): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) - return [json.loads(ret_str)] -""" - -ATTR_DEFAULT = { - "bool": "False", - "int": "0", - "float": "0.0", - "list_int": "[]", - "list_float": "[]", - "list_bool": "[]", - "list_list_int": "[[]]", - "str": "", -} - - -def optype_snake(origin_str): - temp_str = origin_str[0].lower() + origin_str[1:] - new_str = re.sub(r"([A-Z])", r"_\1", temp_str).lower() - return new_str - - -def optype_snake_ex(s): - snake_case = "" - for i, c in enumerate(s): - if i == 0: - snake_case += c.lower() - elif c.isupper(): - if s[i - 1] != "_": - if not s[i - 1].isupper(): - snake_case += "_" - elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): - snake_case += "_" - snake_case += c.lower() - else: - snake_case += c - return snake_case - - -class AdpBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - self.argsdefv = [] - self.op_compile_option: str = "{}" - super().__init__(op_type) - - def write_adapt( - self: any, impl_path, path: str, op_compile_option_all: list = None - ): - self._build_paradefault() - if os.environ.get("BUILD_BUILTIN_OPP") != "1" and impl_path != "": - src_file = os.path.join(impl_path, self.op_file + ".cpp") - if not os.path.exists(src_file): - print( - f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check." - ) - return - out_path = os.path.abspath(path) - if self.dynamic_shape and not out_path.endswith("dynamic"): - out_path = os.path.join(path, "dynamic") - os.makedirs(out_path, exist_ok=True) - adpfile = os.path.join(out_path, self.op_file + ".py") - self._gen_op_compile_option(op_compile_option_all) - with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), "w") as fd: - self._write_head(fd) - self._write_argparse(fd) - self._write_impl(fd, impl_path) - if self.op_chk_support: - self._write_cap("check_supported", fd) - self._write_cap("get_op_support_info", fd) - if self.op_fmt_sel: - self._write_cap("op_select_format", fd) - self._write_cap("get_op_specific_info", fd) - if self.op_range_limit == "limited" or self.op_range_limit == "dynamic": - self._write_glz(fd) - - def _gen_op_compile_option(self: any, op_compile_option_all: list = None): - if op_compile_option_all is not None: - if self.op_type in op_compile_option_all: - self.op_compile_option = op_compile_option_all[self.op_type] - elif "__all__" in op_compile_option_all: - self.op_compile_option = op_compile_option_all["__all__"] - - def _ip_argpack(self: any, default: bool = True) -> list: - args = [] - for i in range(len(self.input_name)): - arg = self.input_name[i] - if default and self.argsdefv[i] is not None: - arg += "=" + self.argsdefv[i] - args.append(arg) - return args - - def _op_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) - for i in range(len(self.output_name)): - arg = self.output_name[i] - if default and self.argsdefv[i + argidx] is not None: - arg += "=" + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _attr_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) + len(self.output_name) - for i in range(len(self.attr_list)): - att = self.attr_list[i] - arg = att - if default and self.argsdefv[i + argidx] is not None: - if self.attr_val.get(att).get("type") == "str": - arg += '="' + self.argsdefv[i + argidx] + '"' - elif self.attr_val.get(att).get("type") == "bool": - arg += "=" + self.argsdefv[i + argidx].capitalize() - else: - arg += "=" + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _build_paralist(self: any, default: bool = True) -> str: - args = [] - args.extend(self._ip_argpack(default)) - args.extend(self._op_argpack(default)) - args.extend(self._attr_argpack(default)) - return ", ".join(args) - - def _io_parachk(self: any, types: list, type_name: str) -> list: - chk = [] - for iot in types: - if iot == "optional": - ptype = "OPTION" - else: - ptype = iot.upper() - chk.append("para_check.{}_{}".format(ptype, type_name)) - return chk - - def _attr_parachk(self: any) -> list: - chk = [] - for att in self.attr_list: - att_type = self.attr_val.get(att).get("type").upper() - chk.append("para_check.{}_ATTR_{}".format("OPTION", att_type)) - return chk - - def _build_parachk(self: any) -> str: - chk = [] - chk.extend(self._io_parachk(self.input_type, "INPUT")) - chk.extend(self._io_parachk(self.output_type, "OUTPUT")) - chk.extend(self._attr_parachk()) - chk.append("para_check.KERNEL_NAME") - return ", ".join(chk) - - def _build_virtual(self: any) -> str: - virt_exp = [] - for index in range(len(self.input_name)): - if self.input_virt.get(index) is None: - continue - val = [] - val.append('"param_name":"{}"'.format(self.input_name[index])) - val.append('"index":{}'.format(index)) - val.append('"dtype":"{}"'.format(self.input_dtype[index].split(",")[0])) - val.append('"format":"{}"'.format(self.input_fmt[index].split(",")[0])) - val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(",")[0])) - val.append('"paramType":"optional"') - val.append('"shape":[1]') - val.append('"ori_shape":[1]') - virt_exp.append( - " " + self.input_name[index] + " = {" + ",".join(val) + "}" - ) - if len(virt_exp) > 0: - return "\n".join(virt_exp) - else: - return " # do ascendc build step" - - def _build_mc2_ctx(self: any): - if len(self.mc2_ctx) != 0: - return '["' + '", "'.join(self.mc2_ctx) + '"]' - return "[]" - - def _build_paradefault(self: any): - optional = False - argtypes = [] - argtypes.extend(self.input_type) - argtypes.extend(self.output_type) - in_idx = 0 - for atype in argtypes: - if atype == "optional": - optional = True - if optional: - self.argsdefv.append("None") - else: - self.argsdefv.append(None) - in_idx += 1 - for attr in self.attr_list: - atype = self.attr_val.get(attr).get("paramType") - if atype == "optional": - optional = True - attrval = self.attr_val.get(attr).get("defaultValue") - if attrval is not None: - optional = True - if type == "bool": - attrval = attrval.capitalize() - elif type == "str": - attrval = '"' + attrval + '"' - self.argsdefv.append(attrval) - continue - if optional: - self.argsdefv.append( - ATTR_DEFAULT.get(self.attr_val.get(attr).get("type")) - ) - else: - self.argsdefv.append(None) - - def _write_head(self: any, fd: object): - now = datetime.datetime.now() - curr_year = now.year - former_year = curr_year - 1 - fd.write( - IMPL_HEAD.format( - former_year, curr_year, self.input_ori_name, self.output_ori_name - ) - ) - - def _write_argparse(self: any, fd: object): - args = self._build_paralist(False) - fd.write("def _build_args({}):\n".format(args)) - fd.write(" __inputs__ = []\n") - fd.write(" for arg in [{}]:\n".format(", ".join(self.input_name))) - fd.write(" if arg != None:\n") - fd.write(" if isinstance(arg, (list, tuple)):\n") - fd.write(" if len(arg) == 0:\n") - fd.write(" continue\n") - fd.write(" __inputs__.append(arg[0])\n") - fd.write(" else:\n") - fd.write(" __inputs__.append(arg)\n") - fd.write(" else:\n") - fd.write(" __inputs__.append(arg)\n") - fd.write(" __outputs__ = []\n") - fd.write(" for arg in [{}]:\n".format(", ".join(self.output_name))) - fd.write(" if arg != None:\n") - fd.write(" if isinstance(arg, (list, tuple)):\n") - fd.write(" if len(arg) == 0:\n") - fd.write(" continue\n") - fd.write(" __outputs__.append(arg[0])\n") - fd.write(" else:\n") - fd.write(" __outputs__.append(arg)\n") - fd.write(" else:\n") - fd.write(" __outputs__.append(arg)\n") - fd.write(" __attrs__ = []\n") - for attr in self.attr_list: - fd.write(" if {} != None:\n".format(attr)) - fd.write(" attr = {}\n") - fd.write(' attr["name"] = "{}"\n'.format(attr)) - fd.write( - ' attr["dtype"] = "{}"\n'.format( - self.attr_val.get(attr).get("type") - ) - ) - fd.write(' attr["value"] = {}\n'.format(attr)) - fd.write(" __attrs__.append(attr)\n") - fd.write(" return __inputs__, __outputs__, __attrs__\n") - - def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): - src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) - if os.path.exists(src_ex): - return src_ex - src = os.environ.get("BUILD_KERNEL_SRC") - if src and os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, dir_snake, src_file) - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, src_file) - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) - if os.path.exists(src): - return src - return src_ex - - def _write_impl(self: any, fd: object, impl_path: str = ""): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - pchk = self._build_parachk() - if len(self.kern_name) > 0: - kern_name = self.kern_name - else: - kern_name = self.op_intf - src = self.op_file + ".cpp" - virt_exprs = self._build_virtual() - fd.write( - IMPL_API.format( - self.op_type, - pchk, - self.op_intf, - argsdef, - kern_name, - virt_exprs, - argsval, - self.custom_compile_options, - self.custom_all_compile_options, - self.op_intf, - optype_snake_ex(self.op_type), - optype_snake(self.op_type), - src, - ) - ) - if self.op_replay_flag: - fd.write( - REPLAY_OP_API.format( - self.op_type, - kern_name, - self.op_file, - self.op_type, - self.op_file, - self.param_type_dynamic, - self.op_compile_option, - ) - ) - else: - if os.environ.get("BUILD_BUILTIN_OPP") == "1": - relative_kernel_src_path = os.path.realpath( - self._get_kernel_source( - impl_path, - src, - optype_snake(self.op_type), - optype_snake_ex(self.op_type), - ) - ) - # to match src path in .dat file system, turn relative path into absolute path - abs_rel_kernel_src_path = os.path.join( - "/", os.path.relpath(relative_kernel_src_path, impl_path) - ) - - # compiling hidden src file requires src path before packaging .dat file, - # hard code such src path to .py - fd.write( - COMPILE_OP_API_BUILT_IN.format( - self.op_type, - self.op_type, - ", ".join(self.input_name), - ", ".join(self.output_name), - self.param_type_dynamic, - self._build_mc2_ctx(), - self.input_type + self.output_type, - self.output_init_value, - self.output_shape_depend_on_compute, - self.op_compile_option, - abs_rel_kernel_src_path, - ) - ) - else: - fd.write( - COMPILE_OP_API.format( - self.op_type, - self.op_type, - ", ".join(self.input_name), - ", ".join(self.output_name), - self.param_type_dynamic, - self._build_mc2_ctx(), - self.input_type + self.output_type, - self.output_init_value, - self.output_shape_depend_on_compute, - self.op_compile_option, - ) - ) - - def _write_cap(self: any, cap_name: str, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - if cap_name == "check_supported": - fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - else: - fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - - def _write_glz(self: any, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - fd.write( - GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type) - ) - - -def write_scripts( - cfgfile: str, - cfgs: dict, - dirs: dict, - ops: list = None, - op_compile_option: list = None, -): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(";") - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(";") - file_map = {} - op_descs = opdesc_parser.get_op_desc( - cfgfile, - batch_lists, - iterator_lists, - AdpBuilder, - ops, - dirs.get(const_var.AUTO_GEN_DIR), - ) - for op_desc in op_descs: - op_desc.write_adapt( - dirs.get(const_var.CFG_IMPL_DIR), - dirs.get(const_var.CFG_OUT_DIR), - op_compile_option, - ) - file_map[op_desc.op_type] = op_desc.op_file - return file_map - - -class OpFileNotExistsError(Exception): - """File does not exist error.""" - - def __str__(self) -> str: - return ( - f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" - ) - - -def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: - """Get all ops info files.""" - ops_info_files = [] - for _dir in opsinfo_dir: - ops_info_files.extend(glob.glob(f"{_dir}/aic-*-ops-info.ini")) - return sorted(ops_info_files) - - -def parse_args(argv): - """Command line parameter parsing""" - parser = argparse.ArgumentParser() - parser.add_argument("argv", nargs="+") - parser.add_argument("--opsinfo-dir", nargs="*", default=None) - return parser.parse_args(argv) - - -if __name__ == "__main__": - args = parse_args(sys.argv) - - if len(args.argv) <= 6: - raise RuntimeError("arguments must greater equal than 6") - - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] - - cfg_dir = {} - cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] - cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] - cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] - - ops_infos = [] - if args.opsinfo_dir: - ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) - if not ops_infos: - raise OpFileNotExistsError(args.opsinfo_dir) - else: - ops_infos.append(args.argv[1]) - - for ops_info in ops_infos: - write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/csrc/deepep/ops/cmake/util/ascendc_op_info.py b/csrc/deepep/ops/cmake/util/ascendc_op_info.py deleted file mode 100755 index a75404285..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_op_info.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import sys - -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class OpInfo: - def __init__(self: any, op_type: str, cfg_file: str): - op_descs = opdesc_parser.get_op_desc( - cfg_file, [], [], opdesc_parser.OpDesc, [op_type] - ) - if op_descs is None or len(op_descs) != 1: - raise RuntimeError("cannot get op info of {}".format(op_type)) - self.op_desc = op_descs[0] - - def get_op_file(self: any): - return self.op_desc.op_file - - def get_op_intf(self: any): - return self.op_desc.op_intf - - def get_inputs_name(self: any): - return self.op_desc.input_ori_name - - def get_outputs_name(self: any): - return self.op_desc.output_ori_name - - -if __name__ == "__main__": - if len(sys.argv) <= 2: - raise RuntimeError("arguments must greater than 2") - op_info = OpInfo(sys.argv[1], sys.argv[2]) - print(op_info.get_op_file()) - print(op_info.get_op_intf()) - print(op_info.get_inputs_name()) - print(op_info.get_outputs_name()) diff --git a/csrc/deepep/ops/cmake/util/ascendc_ops_config.py b/csrc/deepep/ops/cmake/util/ascendc_ops_config.py deleted file mode 100755 index 8c5dd2763..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_ops_config.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import json -import os -import sys - -import const_var - -BINARY_INFO_CONFIG_JSON = "binary_info_config.json" - - -def load_json(json_file: str): - with open(json_file, encoding="utf-8") as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, "**/*.{}".format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return sorted(all_suffix_files) - - -def add_dict_key(dict_to_add, key, value): - if value is None: - return - dict_to_add[key] = value - - -def correct_format_mode(format_mode): - if format_mode == "FormatDefault": - return "nd_agnostic" - if format_mode == "FormatAgnostic": - return "static_nd_agnostic" - if format_mode == "FormatFixed": - return "normal" - return format_mode - - -def get_input_or_output_config(in_or_out): - param_dict = {} - name = in_or_out.get("name") - index = in_or_out.get("index") - param_type = in_or_out.get("paramType") - - format_match_mode = in_or_out.get("format_match_mode") - format_mode = correct_format_mode(format_match_mode) - - dtype_mode = in_or_out.get("dtype_match_mode") - if dtype_mode == "DtypeByte": - dtype_mode = "bit" - - add_dict_key(param_dict, "name", name) - add_dict_key(param_dict, "index", index) - add_dict_key(param_dict, "paramType", param_type) - add_dict_key(param_dict, "dtypeMode", dtype_mode) - add_dict_key(param_dict, "formatMode", format_mode) - return param_dict - - -def get_inputs_or_outputs_config(inputs_or_outputs): - if inputs_or_outputs is None: - return None - inputs_or_outputs_list = [] - - for in_or_out in inputs_or_outputs: - if isinstance(in_or_out, dict): - dict_param_config = get_input_or_output_config(in_or_out) - inputs_or_outputs_list.append(dict_param_config) - elif isinstance(in_or_out, list): - param_info = in_or_out[0] - list_param_config = get_input_or_output_config(param_info) - tmp_list = [list_param_config] - inputs_or_outputs_list.append(tmp_list) - return inputs_or_outputs_list - - -def gen_attrs_config(attrs): - attrs_list = [] - for attr in attrs: - attrs_dict = {} - name = attr.get("name") - mode = attr.get("mode") - add_dict_key(attrs_dict, "name", name) - add_dict_key(attrs_dict, "mode", mode) - attrs_list.append(attrs_dict) - return attrs_list - - -def get_params_config(support_info): - params_dict = {} - - inputs = support_info.get("inputs") - inputs_list = get_inputs_or_outputs_config(inputs) - params_dict["inputs"] = inputs_list - - outputs = support_info.get("outputs") - outputs_list = get_inputs_or_outputs_config(outputs) - params_dict["outputs"] = outputs_list - - attrs = support_info.get("attrs") - if attrs is not None: - attrs_list = gen_attrs_config(attrs) - params_dict["attrs"] = attrs_list - - return params_dict - - -def add_simplified_config( - op_type, support_info, core_type, task_ration, objfile, config -): - simplified_key = support_info.get("simplifiedKey") - - json_path = objfile.split(".")[0] + ".json" - - simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) - op_cfg = simple_cfg.get(op_type) - if not op_cfg: - op_cfg = {"dynamicRankSupport": True} - - simplified_key_mode = support_info.get("simplifiedKeyMode") - add_dict_key(op_cfg, "simplifiedKeyMode", simplified_key_mode) - - optional_input_mode = support_info.get("optionalInputMode") - optional_output_mode = support_info.get("optionalOutputMode") - add_dict_key(op_cfg, "optionalInputMode", optional_input_mode) - if optional_output_mode is not None: - add_dict_key(op_cfg, "optionalOutputMode", optional_output_mode) - - params_info = get_params_config(support_info) - op_cfg["params"] = params_info - op_cfg["binaryList"] = [] - simple_cfg[op_type] = op_cfg - - bin_list = op_cfg.get("binaryList") - if core_type == 0 and task_ration == "tilingKey": - bin_list.append( - { - "coreType": core_type, - "simplifiedKey": simplified_key, - "multiKernelType": 1, - "binPath": objfile, - "jsonPath": json_path, - } - ) - else: - bin_list.append( - { - "coreType": core_type, - "simplifiedKey": simplified_key, - "binPath": objfile, - "jsonPath": json_path, - } - ) - - -def add_op_config(op_file, bin_info, config): - op_cfg = config.get(op_file) - if not op_cfg: - op_cfg = {"binList": []} - config[op_file] = op_cfg - op_cfg.get("binList").append(bin_info) - - -def gen_ops_config(json_file, soc, config): - core_type_map = { - "MIX": 0, - "AiCore": 1, - "VectorCore": 2, - "MIX_AICORE": 3, - "MIX_VECTOR_CORE": 4, - "MIX_AIV": 4, - } - contents = load_json(json_file) - if ("binFileName" not in contents) or ("supportInfo" not in contents): - return - json_base_name = os.path.basename(json_file) - op_dir = os.path.basename(os.path.dirname(json_file)) - - support_info = contents.get("supportInfo") - bin_name = contents.get("binFileName") - bin_suffix = contents.get("binFileSuffix") - core_type = contents.get("coreType") - task_ration = contents.get("taskRation") - core_type = core_type_map.get(core_type, -1) - if core_type == -1 and soc != "ascend310b": - raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") - - bin_file_name = bin_name + bin_suffix - op_type = bin_name.split("_")[0] - op_file = op_dir + ".json" - bin_info = {} - - add_dict_key(bin_info, "implMode", support_info.get("implMode")) - add_dict_key(bin_info, "int64Mode", support_info.get("int64Mode")) - add_dict_key(bin_info, "simplifiedKeyMode", support_info.get("simplifiedKeyMode")) - - simplified_key = support_info.get("simplifiedKey") - if simplified_key is not None: - bin_info["simplifiedKey"] = simplified_key - obj_file = os.path.join(soc, op_dir, bin_file_name) - add_simplified_config( - op_type, support_info, core_type, task_ration, obj_file, config - ) - - add_dict_key(bin_info, "dynamicParamMode", support_info.get("dynamicParamMode")) - bin_info["staticKey"] = support_info.get("staticKey") - bin_info["inputs"] = support_info.get("inputs") - bin_info["outputs"] = support_info.get("outputs") - if support_info.get("attrs"): - bin_info["attrs"] = support_info.get("attrs") - - add_dict_key(bin_info, "opMode", support_info.get("opMode")) - add_dict_key(bin_info, "optionalInputMode", support_info.get("optionalInputMode")) - add_dict_key(bin_info, "deterministic", support_info.get("deterministic")) - if support_info.get("optionalOutputMode") is not None: - add_dict_key( - bin_info, "optionalOutputMode", support_info.get("optionalOutputMode") - ) - - bin_info["binInfo"] = {"jsonFilePath": os.path.join(soc, op_dir, json_base_name)} - add_op_config(op_file, bin_info, config) - - -def check_single_op_is_void(root_dir): - for root, dirs, _ in os.walk(root_dir): - for sub_dir in dirs: - dir_path = os.path.join(root, sub_dir) - if len(os.listdir(dir_path)) == 0: - print(f"[ERROR] op {sub_dir}: not any obj compile success") - sys.exit(1) - - -def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): - suffix = "json" - config = {BINARY_INFO_CONFIG_JSON: {}} - check_single_op_is_void(root_dir) - all_json_files = get_specified_suffix_file(root_dir, suffix) - - for _json in all_json_files: - gen_ops_config(_json, soc, config) - file_path = soc + _json.split(soc)[1] - with open(_json, "r+") as f: - data = json.load(f) - data["filePath"] = file_path - f.seek(0) - json.dump(data, f, indent=" ") - f.truncate() - - for cfg_key in config.keys(): - if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: - continue - cfg_file = os.path.join(out_dir, cfg_key) - with os.fdopen( - os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - json.dump(config.get(cfg_key), fd, indent=" ") - - -# Parse multiple soc_versions ops in single path. -def gen_all_soc_config(all_path): - soc_roots = glob.glob(os.path.join(all_path, "ascend*")) - - for soc_root in soc_roots: - soc = os.path.basename(soc_root) - gen_all_config(soc_root, soc, soc_root, True) - cfg_files = glob.glob(os.path.join(soc_root, "*.json")) - cfg_path = os.path.join(all_path, "config", soc) - os.makedirs(cfg_path, exist_ok=True) - for cfg_file in cfg_files: - new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) - os.rename(cfg_file, new_file) - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-p", - "--path", - nargs="?", - required=True, - help="Parse the path of the json file.", - ) - - parser.add_argument( - "-s", "--soc", nargs="?", required=True, help="Parse the soc_version of ops." - ) - - parser.add_argument("-o", "--out", nargs="?", help="Output directory.") - - parser.add_argument( - "--skip-binary-info-config", - action="store_true", - help="binary_info_config.json file is not parsed.", - ) - - return parser.parse_args() - - -def main(): - args = args_prase() - if args.out is None: - out_dir = args.path - else: - out_dir = args.out - - gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) - - -if __name__ == "__main__": - main() diff --git a/csrc/deepep/ops/cmake/util/ascendc_pack_kernel.py b/csrc/deepep/ops/cmake/util/ascendc_pack_kernel.py deleted file mode 100755 index 430c1ba8e..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_pack_kernel.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import json -import math -import os -import subprocess -import sys - -import ascendc_ops_config -import const_var -from tbe.tikcpp.log_utils import AscendCLogLevel, LogUtil - - -class PackKernel: - def __init__(self: any, args: any): - self.in_path = os.path.realpath(args.input_path) - self.out_path = os.path.realpath(args.output_path) - self.is_lib = args.enable_library - self.platform = args.platform - self.op_info = {} - self.file_info = {} - try: - os.makedirs(self.out_path, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {self.out_path} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def load_json(self: any, json_file: str): - with open(json_file, encoding="utf-8") as file: - json_content = json.load(file) - return json_content - - def get_symbol(self: any, name: str): - name = name.replace("/", "_") - return name.replace(".", "_") - - def ascendc_gen_object(self: any, in_file: str, soc: str): - sym = self.get_symbol("_binary_" + in_file) - out_file = os.path.join(self.out_path, sym + ".o") - # ascend610lite only support aarch64 - if soc == "ascend610lite": - try: - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - except Exception as e: - LogUtil.print_compile_log( - "", - " ascend610lite execute objcopy fail!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - uname = os.popen("uname -m").read().strip() - if self.platform is not None: - target_platform = self.platform - else: - target_platform = uname - try: - if target_platform == "x86_64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-x86-64", - "--binary-architecture", - "i386", - in_file, - out_file, - ] - ) - elif target_platform == "aarch64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - else: - subprocess.run(["echo", "unsported environment!"]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"{target_platform} execute objcopy error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - - def ascendc_get_config(self: any): - os.chdir(self.in_path) - soc_vers = os.listdir("config") - for soc in soc_vers: - bin_infos = glob.glob(os.path.join("config", soc, "*.json")) - cfgs = {} - for bin_info in bin_infos: - if bin_info.find("binary_info_config.json") > 0: - continue - jobj = self.load_json(bin_info) - for bin_cfg in jobj.get("binList"): - js_cfg = bin_cfg.get("binInfo").get("jsonFilePath") - op_type = os.path.basename(js_cfg).split("_")[0] - if cfgs.get(op_type) is None: - op_obj = {} - op_obj["obj"] = [] - op_obj["cfg"] = bin_info - cfgs[op_type] = op_obj - op_obj = cfgs.get(op_type) - op_obj.get("obj").append(js_cfg[:-5]) - self.file_info[soc] = cfgs - - def ascendc_pack_kernel(self: any): - for soc in self.file_info.keys(): - os.chdir(self.in_path) - op_cfgs = self.file_info.get(soc) - for op_type in op_cfgs.keys(): - op_obj = op_cfgs.get(op_type) - if self.op_info.get(op_type) is None: - op_info = {} - op_info["op_fun"] = ["nullptr", "nullptr"] - op_info["op_bin"] = {} - op_info["op_rkb"] = [] - self.op_info[op_type] = op_info - op_info = self.op_info.get(op_type) - op_bin = op_info.get("op_bin") - if op_bin.get(soc) is None: - op_bin[soc] = [] - op_bin[soc].append(self.ascendc_gen_object(op_obj["cfg"], soc)) - op_soc = op_bin.get(soc) - for objs in op_obj["obj"]: - op_soc.append(self.ascendc_gen_object(objs + ".json", soc)) - op_soc.append(self.ascendc_gen_object(objs + ".o", soc)) - - def ascendc_gen_header(self: any): - for op_type in self.op_info.keys(): - op_obj = self.op_info.get(op_type) - macro_op = ( - "#define {}_OP_RESOURCES std::make_tuple, \\\n" - " std::map>>, \\\n" - " std::vector>>({{{}}}, \\\n".format( - op_type, ", ".join(op_obj.get("op_fun")) - ) - ) - op_bin = op_obj.get("op_bin") - socs_res = [] - op_syms = [] - for soc in op_bin.keys(): - soc_res = '{{ "{}", {{'.format(soc) - soc_syms = op_bin.get(soc) - soc_pairs = [] - for pair_addr in soc_syms: - pair_addr1 = ["&" + s for s in pair_addr] - op_syms += pair_addr - soc_pairs.append( - " {{ {} }} ".format(", \\\n ".join(pair_addr1)) - ) - soc_res += ", \\\n ".join(soc_pairs) - soc_res += " } }" - socs_res.append(soc_res) - macro_op += " {{ {} }}, \\\n".format(", \\\n ".join(socs_res)) - macro_op += " {{ {} }})\n\n".format(", ".join(op_obj.get("op_rkb"))) - macro_str = '#define {}_RESOURCES {{{{"{}", {}}}}}'.format( - op_type, op_type, "{}_OP_RESOURCES".format(op_type) - ) - var_str = ( - "extern gert::OpImplRegisterV2 op_impl_register_optiling_{};\n".format( - op_type - ) - ) - if len(op_syms) > 0: - var_str += ( - "extern uint8_t " + ";\nextern uint8_t ".join(op_syms) + ";\n" - ) - head_file = os.path.join(self.out_path, "{}_op_resource.h".format(op_type)) - try: - with os.fdopen( - os.open(head_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write('#include "graph/ascend_string.h"\n') - fd.write('#include "register/op_impl_registry.h"\n\n') - fd.write(var_str) - fd.write("\n") - fd.write(macro_op) - fd.write(macro_str) - except Exception as e: - LogUtil.print_compile_log( - "", - f"{op_type}_op_resource.h create error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_gen_lib(self: any): - out_lib = os.path.join(self.out_path, "libkernels.a") - if os.path.exists(out_lib): - os.remove(out_lib) - objs = glob.glob(os.path.join(self.out_path, "*.o")) - start = 0 - batch_size = 100 - for _ in range(math.ceil(len(objs) / batch_size)): - sub_objs = objs[start : start + batch_size] - start += batch_size - try: - subprocess.run(["ar", "qc", out_lib] + sub_objs) - subprocess.run(["ranlib", out_lib]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"execute ar/ranlib command error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_gen_opsinfo(self: any): - ascendc_ops_config.gen_all_soc_config(self.in_path) - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", "--input-path", nargs="?", help="Input path of compile result." - ) - parser.add_argument( - "-o", "--output-path", nargs="?", help="Output path of compile result." - ) - parser.add_argument( - "-l", - "--enable-library", - nargs="?", - default=None, - help="Whether library is enabled.", - ) - parser.add_argument( - "-p", - "--platform", - nargs="?", - default=None, - help="target platform is x86_64 or aarch64.", - ) - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - kernel_packer = PackKernel(args) - if kernel_packer.is_lib is None: - kernel_packer.ascendc_gen_opsinfo() - kernel_packer.ascendc_get_config() - kernel_packer.ascendc_pack_kernel() - kernel_packer.ascendc_gen_header() - kernel_packer.ascendc_gen_lib() diff --git a/csrc/deepep/ops/cmake/util/ascendc_pack_opregistry.py b/csrc/deepep/ops/cmake/util/ascendc_pack_opregistry.py deleted file mode 100755 index 1acb7e45e..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_pack_opregistry.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import math -import os -import shutil -import subprocess -import sys - -import const_var -from tbe.tikcpp.log_utils import AscendCLogLevel, LogUtil - - -class PackKernel: - def __init__(self: any, args: any): - self.in_path = os.path.realpath(args.input_path) - self.copy_path = os.path.realpath(args.copy_path) - self.out_path = os.path.realpath(args.output_path) - self.op_soc_ver = args.compute_unit.split("-") - self.vendor_name = args.vendor_name - self.framework_type = args.framework_type - self.platform = args.platform - self.op_info = {} - self.file_info = {} - if os.path.exists(self.copy_path): - try: - shutil.rmtree(self.copy_path) - except OSError as e: - LogUtil.print_compile_log( - "", - f"remove {self.copy_path} error!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - if os.path.exists(self.out_path): - try: - shutil.rmtree(self.out_path) - except OSError as e: - LogUtil.print_compile_log( - "", - f"remove {self.out_path} error!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - try: - os.makedirs(self.copy_path, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {self.copy_path} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - try: - os.makedirs(self.out_path, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {self.out_path} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def get_symbol(self: any, name: str): - name = name.replace("/", "_") - name = name.replace("-", "_") - return name.replace(".", "_") - - def ascendc_gen_object(self: any, in_file: str, path: str): - sym = self.get_symbol("_binary_" + in_file) - out_file = os.path.join(self.out_path, sym + ".o") - # ascend610lite only support aarch64 - if path.find("ascend610lite") != -1: - try: - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - except Exception as e: - LogUtil.print_compile_log( - "", - " ascend610lite execute objcopy fail!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - - uname = os.popen("uname -m").read().strip() - if self.platform is not None: - target_platform = self.platform - else: - target_platform = uname - try: - if target_platform == "x86_64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-x86-64", - "--binary-architecture", - "i386", - in_file, - out_file, - ] - ) - elif target_platform == "aarch64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - else: - subprocess.run(["echo", "unsupported environment!"]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"{target_platform} execute objcopy error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - - def ascendc_get_config(self: any): - os.chdir(self.copy_path) - current_directory = os.getcwd() - catalog_file = os.listdir(current_directory) - for catalog in catalog_file: - if catalog == "op_impl" or catalog == "framework": - files_dict = {} - for root, _, files in os.walk(catalog): - for file in files: - if ( - file.endswith(".json") - or file.endswith(".so") - or file.endswith(".cpp") - or file.endswith(".py") - or file.endswith(".o") - ): - file_path = os.path.join(root, file) - file_name = os.path.basename(file_path) - files_dict[file_name] = file_path - self.file_info[catalog] = files_dict - - def ascendc_pack_kernel(self: any): - op_info = {} - for files in self.file_info.keys(): - os.chdir(self.copy_path) - op_cfgs = self.file_info.get(files) - for file_name in op_cfgs.keys(): - op_info[file_name] = [] - path, filename = os.path.split(op_cfgs[file_name]) - op_info[file_name].append(os.path.join(self.vendor_name, path)) - op_info[file_name].append( - self.ascendc_gen_object(op_cfgs[file_name], path) - ) - self.op_info = op_info - - def ascendc_gen_header(self: any): - socs_res = [] - var_str = "" - macro_op = ( - "std::vector> __ascendc_op_info = \n" - ) - for file_name in self.op_info.keys(): - file_addr = self.op_info.get(file_name) - soc_pairs = [] - op_syms = [] - soc_res = ' {{ "{}", '.format(file_name) - soc_res += '"{}", '.format(file_addr[0]) - for pair_addr in file_addr[1]: - op_syms.append(pair_addr) - pair_addr1 = "&" + pair_addr - soc_pairs.append(pair_addr1) - soc_res += "{}, {}".format(soc_pairs[0], soc_pairs[1]) - soc_res += "}, \n" - socs_res.append(soc_res) - if len(op_syms) > 0: - var_str += "".join( - ["extern uint8_t {};\n".format(sym) for sym in op_syms] - ) - macro_op += "{{\n{}}}; \n".format("".join(socs_res)) - head_file = os.path.join(self.out_path, "ge_table_op_resource.h") - try: - with os.fdopen( - os.open(head_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write('#include "graph/ascend_string.h"\n') - fd.write('#include "register/op_impl_registry.h"\n\n') - fd.write(var_str) - fd.write("\n") - fd.write("namespace AscendC {\n") - fd.write(macro_op) - fd.write("}\n") - except Exception as e: - LogUtil.print_compile_log( - "", - f"ge_table_op_resource.h create error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_gen_lib(self: any): - out_lib = os.path.join(self.out_path, "libopregistry.a") - if os.path.exists(out_lib): - os.remove(out_lib) - objs = glob.glob(os.path.join(self.out_path, "*.o")) - start = 0 - batch_size = 100 - for _ in range(math.ceil(len(objs) / batch_size)): - sub_objs = objs[start : start + batch_size] - start += batch_size - try: - subprocess.run(["ar", "qc", out_lib] + sub_objs) - subprocess.run(["ranlib", out_lib]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"execute ar/ranlib command error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_copy_dir(self: any, src_dir: str, target_dir: str): - file_list = os.listdir(src_dir) - for file_name in file_list: - source_file = os.path.join(src_dir, file_name) - target_file = os.path.join(target_dir, file_name) - if os.path.isdir(source_file): - try: - shutil.copytree(source_file, target_file) - except Exception as e: - LogUtil.print_compile_log( - "", - f"copy {source_file} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_copy_file(self: any, src_dir: str, target_dir: str): - file_list = os.listdir(src_dir) - for file_name in file_list: - source_file = os.path.join(src_dir, file_name) - if os.path.isfile(source_file): - try: - os.makedirs(target_dir, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {target_dir} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - try: - shutil.copy(source_file, target_dir) - except Exception as e: - LogUtil.print_compile_log( - "", - f"copy {source_file} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_copy_func(self: any): - os.chdir(self.in_path) - framework_catalog = os.listdir("framework") - for catalog_file in framework_catalog: - if ( - catalog_file == "tf_plugin" - or catalog_file == "caffe_plugin" - or catalog_file == "onnx_plugin" - ): - source_dir = "op_kernel/tbe/op_info_cfg/ai_core" - dst_dir = os.path.join(self.copy_path, "framework", self.framework_type) - self.ascendc_copy_file(source_dir, dst_dir) - source_dir = os.path.join("framework", catalog_file) - dst_dir = os.path.join(self.copy_path, "framework", self.framework_type) - self.ascendc_copy_file(source_dir, dst_dir) - source_dir = "op_kernel/tbe/op_info_cfg/ai_core" - dst_dir = os.path.join(self.copy_path, "op_impl/ai_core/tbe/config") - self.ascendc_copy_dir(source_dir, dst_dir) - source_dir = "op_kernel/binary/dynamic" - dst_dir = os.path.join( - self.copy_path, "op_impl/ai_core/tbe", self.vendor_name + "_impl", "dynamic" - ) - self.ascendc_copy_file(source_dir, dst_dir) - for compute_unit in self.op_soc_ver: - source_dir = os.path.join("op_kernel/binary", compute_unit) - dst_dir = os.path.join( - self.copy_path, "op_impl/ai_core/tbe/kernel", compute_unit - ) - self.ascendc_copy_dir(source_dir, dst_dir) - source_dir = "op_kernel/binary/config" - dst_dir = os.path.join(self.copy_path, "op_impl/ai_core/tbe/kernel/config") - self.ascendc_copy_dir(source_dir, dst_dir) - so_file = "op_impl/ai_core/tbe/op_master_device/lib/libcust_opmaster.so" - if os.path.exists(so_file): - dst_dir = os.path.join( - self.copy_path, "op_impl/ai_core/tbe/op_master_device/lib" - ) - os.makedirs(dst_dir, exist_ok=True) - shutil.copy(so_file, dst_dir) - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", "--input-path", nargs="?", help="Input path of compile result." - ) - parser.add_argument( - "-c", "--copy-path", nargs="?", help="Copy path of compile result." - ) - parser.add_argument( - "-o", "--output-path", nargs="?", help="Output path of compile result." - ) - parser.add_argument("-n", "--vendor-name", nargs="?", help="Vendor name.") - parser.add_argument("-u", "--compute-unit", nargs="?", help="Compute unit.") - parser.add_argument( - "-t", "--framework-type", nargs="?", help="Framework type, eg:tensorflow." - ) - parser.add_argument( - "-p", - "--platform", - nargs="?", - default=None, - help="target platform is x86_64 or aarch64.", - ) - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - kernel_packer = PackKernel(args) - kernel_packer.ascendc_copy_func() - kernel_packer.ascendc_get_config() - kernel_packer.ascendc_pack_kernel() - kernel_packer.ascendc_gen_header() - kernel_packer.ascendc_gen_lib() diff --git a/csrc/deepep/ops/cmake/util/ascendc_replay_build.py b/csrc/deepep/ops/cmake/util/ascendc_replay_build.py deleted file mode 100755 index e07545f5c..000000000 --- a/csrc/deepep/ops/cmake/util/ascendc_replay_build.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import sys - -import const_var -import opdesc_parser -import replay_codegen -from replay_codegen import ReplayCodeGenParams - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class ReplayBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - - def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): - if not self.op_replay_flag: - print("{} replay not enabled".format(self.op_type)) - return - argn = len(self.input_name) + len(self.output_name) + 1 - if self.op_replay_batch: - print("{} replay in batch mode".format(self.op_type)) - else: - print("{} replay in normal mode".format(self.op_type)) - if impl_path.endswith("op_kernel"): - implf = os.path.join(impl_path, self.op_file + ".cpp") - tiling_file = os.path.join( - impl_path, "../op_host", self.op_file + "_tiling.h" - ) - else: - if self.dynamic_shape: - dyn_path = "dynamic" - else: - dyn_path = "" - implf = os.path.join(impl_path, dyn_path, self.op_file + ".cpp") - tiling_file = os.path.join( - impl_path, "../../op_tiling", self.op_file + "_tiling.h" - ) - rep_conf = replay_codegen.ReplayCodeGen( - ReplayCodeGenParams( - self.op_type, - implf, - tiling_file, - self.op_file, - self.op_intf, - argn, - self.op_replay_batch, - self.max_block_dim, - self.max_shape_size, - ) - ) - rep_conf.set_batch(self.op_replay_batch) - rep_conf.set_outdir(out_path) - rep_conf.gen_replay(ops_product) - - -def gen_replay( - cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None -): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(";") - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(";") - op_descs = opdesc_parser.get_op_desc( - cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops - ) - for op_desc in op_descs: - op_desc.gen_replay_source( - dirs.get(const_var.CFG_IMPL_DIR), - dirs.get(const_var.CFG_OUT_DIR), - ops_product, - ) - - -if __name__ == "__main__": - if len(sys.argv) <= 6: - raise RuntimeError("arguments must greater than 6") - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] - rep_dir = {} - rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] - rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] - gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/csrc/deepep/ops/cmake/util/batch_replay_impl.temp b/csrc/deepep/ops/cmake/util/batch_replay_impl.temp deleted file mode 100644 index 0e8834664..000000000 --- a/csrc/deepep/ops/cmake/util/batch_replay_impl.temp +++ /dev/null @@ -1,117 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -#include - -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], - int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N]; - int len[KERNEL_N]; - block_idx = 0; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - //__OP_SET_KERNEL__ - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, true); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[i] = (char *)pos; - len[i] = CodeLen(); - pos += len[i]; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/csrc/deepep/ops/cmake/util/code_channel_infer.py b/csrc/deepep/ops/cmake/util/code_channel_infer.py deleted file mode 100755 index c9042d4d3..000000000 --- a/csrc/deepep/ops/cmake/util/code_channel_infer.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import collections -import copy -import ctypes -import os -import shutil -import stat -import subprocess - -"""CODE_* is used to cube/vector api is called in operator code -CODE_MIX means both cube and vector api is called -CODE_CUBE means only cube api is called -CODE_VEC means only vector api is called -""" -CODE_MIX = 0 -CODE_CUBE = 1 -CODE_VEC = 2 - - -def _is_v220(op_product: str): - """return if current soc version is V220 - - Returns: - res: True means V220 - """ - if op_product == "ascend910_93" or op_product == "ascend910b": - return True - return False - - -InfoCodeChanelParams = collections.namedtuple( - "InfoCodeChanelParams", - [ - "src_file", - "tiling_header", - "kernel_name", - "outdir", - "op_product", - "compile_options", - ], -) - - -def infer_code_channel(params: InfoCodeChanelParams): - """get code channel for v220, return CODE_MIX if soc version is not V220 - - Args: - src_file (str): AscendC operator code file - src_file (str): AscendC operator tiling header file - kernel_name (str): kernel function name - optype (str): operator type - compile_options (list): compile options for bisheng cmd - - Raises: - Exception: if not exist L1/L0/UB if code, it's not a aicore code - - Returns: - res (int): CODE_MIX/CODE_CUBE/CODE_VEC - """ - if not _is_v220(params.op_product): - return CODE_MIX - return CODE_VEC diff --git a/csrc/deepep/ops/cmake/util/const_var.py b/csrc/deepep/ops/cmake/util/const_var.py deleted file mode 100755 index bc8f33bf9..000000000 --- a/csrc/deepep/ops/cmake/util/const_var.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import stat - -REPLAY_BATCH = "batch" -REPLAY_ITERATE = "iterate" -CFG_IMPL_DIR = "impl_dir" -CFG_OUT_DIR = "out_dir" -AUTO_GEN_DIR = "auto_gen_dir" -WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC -WMODES = stat.S_IWUSR | stat.S_IRUSR -SOC_MAP_EXT = { - "ascend310p": "Ascend310P3", - "ascend310b": "Ascend310B1", - "ascend910": "Ascend910A", - "ascend910b": "Ascend910B1", - "ascend910_93": "Ascend910_9391", - "ascend610lite": "Ascend610Lite", -} -BIN_CMD = "opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ ---output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n" -SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" -SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" -SRC_ENV = """ -while true; do - case "$1" in - --kernel-src=*) - export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) - shift - ;; - -*) - shift - ;; - *) - break - ;; - esac -done -""" -CHK_CMD = """ -if ! test -f $2/{res_file} ; then - echo "$2/{res_file} not generated!" - exit 1 -fi -""" -ATTR_DEF_VAL = { - "str": "", - "int": 0, - "float": 0.0, - "bool": False, - "list_bool": [], - "list_int": [], - "list_float": [], - "list_list_int": [[]], -} - - -def conv_soc_ver(ver: str): - return SOC_MAP_EXT.get(ver) diff --git a/csrc/deepep/ops/cmake/util/gen_impl_and_mrege_json.sh b/csrc/deepep/ops/cmake/util/gen_impl_and_mrege_json.sh deleted file mode 100755 index 93d7ec845..000000000 --- a/csrc/deepep/ops/cmake/util/gen_impl_and_mrege_json.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No project path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -# copy aicpu kernel so operators -if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then - cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl - rm -rf ${project_path}/cpukernel/aicpu_kernel_lib -fi diff --git a/csrc/deepep/ops/cmake/util/gen_ops_filter.sh b/csrc/deepep/ops/cmake/util/gen_ops_filter.sh deleted file mode 100755 index b06a4e9fc..000000000 --- a/csrc/deepep/ops/cmake/util/gen_ops_filter.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -if [[ -z "$1" ]]; then - echo -e "[ERROR] No source dir provided" - exit 1 -fi - -if [[ -z "$2" ]]; then - echo -e "[ERROR] No destination dir provided" - exit 1 -fi - -src=$1 -dest_file=$2/npu_supported_ops.json - -if [ -f "$dest_file" ];then - chmod u+w $dest_file -fi - -echo $* - -add_ops() { - name=$1 - isHeavy=$2 - file=$3 - grep -w "\"$name\"" ${file} >/dev/null - if [ $? == 0 ];then - return - fi - echo " \"${name}\": {" >> ${file} - echo " \"isGray\": false," >> ${file} - echo " \"isHeavy\": ${isHeavy}" >> ${file} - echo " }," >> ${file} -} - -echo "{" > ${dest_file} -ini_files=$(find ${src} -name "*.ini") -for file in ${ini_files} ; do - name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') - grep 'heavyOp.flag' ${file} >/dev/null - if [ $? == 0 ];then - isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') - else - isHeavy="false" - fi - for op in ${name} ; do - add_ops ${op} "false" ${dest_file} - done -done -echo "}" >> ${dest_file} -file_count=$(cat ${dest_file} | wc -l) -line=$(($file_count-1)) -sed -i "${line}{s/,//g}" ${dest_file} - -chmod 640 "${dest_file}" -echo -e "[INFO] Succeed generated ${dest_file}" - -exit 0 diff --git a/csrc/deepep/ops/cmake/util/gen_version_info.sh b/csrc/deepep/ops/cmake/util/gen_version_info.sh deleted file mode 100755 index 8468f949a..000000000 --- a/csrc/deepep/ops/cmake/util/gen_version_info.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -ascend_install_dir=$1 -gen_file_dir=$2 - -# create version.info -compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') -echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info diff --git a/csrc/deepep/ops/cmake/util/insert_op_info.py b/csrc/deepep/ops/cmake/util/insert_op_info.py deleted file mode 100644 index ca7562e7a..000000000 --- a/csrc/deepep/ops/cmake/util/insert_op_info.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- - -import json -import os -import stat -import sys - -import const_var - -if __name__ == "__main__": - if len(sys.argv) != 3: - print(sys.argv) - print("argv error, inert_op_info.py your_op_file lib_op_file") - sys.exit(2) - - with open(sys.argv[1], "r") as load_f: - insert_operator = json.load(load_f) - - all_operators = {} - if os.path.exists(sys.argv[2]): - if os.path.getsize(sys.argv[2]) != 0: - with open(sys.argv[2], "r") as load_f: - all_operators = json.load(load_f) - - for k in insert_operator.keys(): - if k in all_operators.keys(): - print("replace op:[", k, "] success") - else: - print("insert op:[", k, "] success") - all_operators[k] = insert_operator[k] - - with os.fdopen( - os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), "w" - ) as json_file: - json_file.write(json.dumps(all_operators, indent=4)) diff --git a/csrc/deepep/ops/cmake/util/insert_simplified_keys.py b/csrc/deepep/ops/cmake/util/insert_simplified_keys.py deleted file mode 100755 index 599ebe974..000000000 --- a/csrc/deepep/ops/cmake/util/insert_simplified_keys.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import json -import os -import re -import sys - -import const_var - -DATA_TPYE_DICT = { - "float32": 0, - "float16": 1, - "int8": 2, - "int16": 6, - "uint16": 7, - "uint8": 4, - "int32": 3, - "int64": 9, - "uint32": 8, - "uint64": 10, - "bool": 12, - "double": 11, - "string": 13, - "dual": 14, - "dual": 15, - "complex64": 16, - "complex128": 17, - "qint8": 18, - "qint16": 19, - "qint32": 20, - "quint8": 21, - "quint16": 22, - "resource": 23, - "string": 24, - "dual": 25, - "variant": 26, - "bf16": 27, - "bfloat16": 27, - "undefined": 28, - "int4": 29, - "uint1": 30, - "int2": 31, -} - -FORMAT_DICT = { - "NCHW": 0, - "NHWC": 1, - "ND": 2, - "NC1HWC0": 3, - "FRACTAL_Z": 4, - "NC1C0HWPAD": 5, - "NHWC1C0": 6, - "FSR_NCHW": 7, - "FRACTAL_DECONV": 8, - "C1HWNC0": 9, - "FRACTAL_DECONV_TRANSPOSE": 10, - "FRACTAL_DECONV_SP_STRIDE_TRANS": 11, - "NC1HWC0_C04": 12, - "FRACTAL_Z_C04": 13, - "CHWN": 14, - "FRACTAL_DECONV_SP_STRIDE8_TRANS": 15, - "HWCN": 16, - "NC1KHKWHWC0": 17, - "BN_WEIGHT": 18, - "FILTER_HWCK": 19, - "HASHTABLE_LOOKUP_LOOKUPS": 20, - "HASHTABLE_LOOKUP_KEYS": 21, - "HASHTABLE_LOOKUP_VALUE": 22, - "HASHTABLE_LOOKUP_OUTPUT": 23, - "HASHTABLE_LOOKUP_HITS": 24, - "C1HWNCoC0": 25, - "MD": 26, - "NDHWC": 27, - "FRACTAL_ZZ": 28, - "FRACTAL_NZ": 29, - "NCDHW": 30, - "DHWCN": 31, - "NDC1HWC0": 32, - "FRACTAL_Z_3D": 33, - "CN": 34, - "NC": 35, - "DHWNC": 36, - "FRACTAL_Z_3D_TRANSPOSE": 37, - "FRACTAL_ZN_LSTM": 38, - "FRACTAL_Z_G": 39, - "RESERVED": 40, - "ALL": 41, - "NULL": 42, - "ND_RNN_BIAS": 43, - "FRACTAL_ZN_RNN": 44, - "NYUV": 45, - "NYUV_A": 46, -} - - -def load_json(json_file: str): - with open(json_file, encoding="utf-8") as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, "**/*.{}".format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return all_suffix_files - - -def get_deterministic_value(support_info): - deterministic_key = "deterministic" - if deterministic_key not in support_info: - return 0 - deterministic_value = support_info.get(deterministic_key) - if deterministic_value == "true": - return 1 - else: - return 0 - - -def get_precision_value(support_info): - precision_key = "implMode" - precision_value = support_info.get(precision_key) - if precision_value == "high_performance": - _value = 1 - elif precision_value == "high_precision": - _value = 2 - else: - _value = 0 - return _value - - -def get_overflow_value(support_info): - return 0 - - -def get_parameters(info): - if info: - if "dtype" in info: - data_type = info["dtype"] - data_type_value = DATA_TPYE_DICT.get(data_type) - else: - data_type_value = 0 - if "format" in info: - _format = info["format"] - _format_value = FORMAT_DICT.get(_format) - else: - _format_value = 0 - else: - data_type_value = 0 - _format_value = 0 - return str(data_type_value), str(_format_value) - - -def get_dynamic_parameters(info): - # 动态输入时只需获取第一个参数 - return get_parameters(info[0]) - - -def get_all_parameters(support_info, _type): - result_list = list() - info_lists = support_info.get(_type) - if info_lists: - for _info in info_lists: - # 输入为列表时是动态输入 - if isinstance(_info, (list, tuple)): - data_type_value, _format_value = get_dynamic_parameters(_info) - else: - data_type_value, _format_value = get_parameters(_info) - result_list.append("{},{}".format(data_type_value, _format_value)) - return result_list - - -def get_all_input_parameters(support_info): - result = get_all_parameters(support_info, "inputs") - return "/".join(result) - - -def insert_content_into_file(input_file, content): - with open(input_file, "r+") as file: - lines = file.readlines() - for index, line in enumerate(lines): - match_result = re.search(r'"staticKey":', line) - if match_result: - count = len(line) - len(line.lstrip()) - new_content = "{}{}".format(" " * count, content) - # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 - lines.insert(index, new_content) - break - file.seek(0) - file.write("".join(lines)) - - -def insert_simplified_keys(json_file): - contents = load_json(json_file) - # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 - if ("binFileName" not in contents) or ("supportInfo" not in contents): - return - support_info = contents.get("supportInfo") - bin_file_name = contents.get("binFileName") - # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 - if "simplifiedKey" in support_info: - return - op_type = bin_file_name.split("_")[0] - deterministic = str(get_deterministic_value(support_info)) - precision = str(get_precision_value(support_info)) - overflow = str(get_overflow_value(support_info)) - input_parameters = get_all_input_parameters(support_info) - key = "{}/d={},p={},o={}/{}/".format( - op_type, deterministic, precision, overflow, input_parameters - ) - result = '"simplifiedKey": "' + key + '",\n' - insert_content_into_file(json_file, result) - - -def insert_all_simplified_keys(root_dir): - suffix = "json" - all_json_files = get_specified_suffix_file(root_dir, suffix) - for _json in all_json_files: - insert_simplified_keys(_json) - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-p", - "--path", - nargs="?", - required=True, - help="Parse the path of the json file.", - ) - return parser.parse_args() - - -def main(): - args = args_prase() - insert_all_simplified_keys(args.path) - - -if __name__ == "__main__": - main() diff --git a/csrc/deepep/ops/cmake/util/kernel_entry.py b/csrc/deepep/ops/cmake/util/kernel_entry.py deleted file mode 100755 index 255266fd9..000000000 --- a/csrc/deepep/ops/cmake/util/kernel_entry.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - - -def gen_fun_def(title, kernel, argn, arg_type, arg_name): - entry = [] - entry.append(title) - entry.append(kernel) - entry.append("(") - args = [] - for i in range(0, argn): - args.append(arg_type + " " + arg_name + str(i)) - entry.append(", ".join(args)) - entry.append(")") - return " ".join(entry) - - -def gen_batch_kernel_body(fname, argn, arg_name): - body = [] - body.append("{") - fun = [] - fun.append(fname) - fun.append("(") - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(", ".join(args)) - fun.append(");") - body.append(" ".join(fun)) - body.append("}") - return "\n".join(body) - - -def gen_mc_kernel_body(kn, argn, arg_name, blknum): - body = [] - body.append("{") - body.append(" switch(block_idx) {") - for blk in range(0, blknum): - fun = [] - fun.append("{}_blk{:02d}".format(kn, blk)) - fun.append("(") - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(", ".join(args)) - fun.append(")") - body.append(" case {}: {}; break;".format(blk, " ".join(fun))) - body.append(" default: break;") - body.append(" }") - body.append("}") - return "\n".join(body) - - -def gen_proc_body(argn, arg_name): - body = [] - body.append("{") - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - body.append("uint64_t __x = (uint64_t)" + " + (uint64_t)".join(args) + ";") - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append("}") - return "\n".join(body) - - -def batch_code_gen(kn, argn, argt): - codes = [] - kernel_name = kn - proc_name = kernel_name + "_percore" - arg_num = int(argn) - data_type = argt - arg_type = "__gm__ " + data_type + "* __restrict__" - arg_name = "arg" - kernel_title = 'extern "C" __global__ __aicore__ void' - proc_title = 'extern "C" __attribute__((noinline)) __aicore__ void' - codes.append("#ifndef __aicore__") - codes.append("#define __aicore__ [aicore]") - codes.append("#endif") - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ";") - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return "\n".join(codes) + "\n" - - -def mc_code_gen(kn, argn, argt, blknum): - codes = [] - kernel_name = kn - core_num = int(blknum) - arg_num = int(argn) - data_type = argt - arg_type = "__gm__ " + data_type + "* __restrict__" - arg_name = "arg" - kernel_title = 'extern "C" __global__ __aicore__ void' - proc_title = 'extern "C" __attribute__((noinline)) __aicore__ void' - codes.append("#ifndef __aicore__") - codes.append("#define __aicore__ [aicore]") - codes.append("#endif") - for i in range(0, core_num): - proc_name = "{}_blk{:02d}".format(kernel_name, i) - codes.append( - gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ";" - ) - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) - for i in range(0, core_num): - proc_name = "{}_blk{:02d}".format(kernel_name, i) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return "\n".join(codes) + "\n" diff --git a/csrc/deepep/ops/cmake/util/kernel_impl.temp b/csrc/deepep/ops/cmake/util/kernel_impl.temp deleted file mode 100644 index 5079a1043..000000000 --- a/csrc/deepep/ops/cmake/util/kernel_impl.temp +++ /dev/null @@ -1,10 +0,0 @@ -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#define __ASCENDC_REPLAY_CODE__ -#include "__CCE_FILE__" diff --git a/csrc/deepep/ops/cmake/util/makeself/COPYING b/csrc/deepep/ops/cmake/util/makeself/COPYING deleted file mode 100644 index d159169d1..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/csrc/deepep/ops/cmake/util/makeself/README.md b/csrc/deepep/ops/cmake/util/makeself/README.md deleted file mode 100644 index 9d3d4b86f..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/README.md +++ /dev/null @@ -1,246 +0,0 @@ -[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) -![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) - -# makeself - Make self-extractable archives on Unix - -[makeself.sh][1] is a small shell script that generates a self-extractable -compressed tar archive from a directory. The resulting file appears as a shell script -(many of those have a **.run** suffix), and can be launched as is. The archive -will then uncompress itself to a temporary directory and an optional arbitrary -command will be executed (for example an installation script). This is pretty -similar to archives generated with WinZip Self-Extractor in the Windows world. -Makeself archives also include checksums for integrity self-validation (CRC -and/or MD5/SHA256 checksums). - -The makeself.sh script itself is used only to create the archives from a -directory of files. The resultant archive is actually a compressed (using -gzip, bzip2, or compress) TAR archive, with a small shell script stub at the -beginning. This small stub performs all the steps of extracting the files, -running the embedded command, and removing the temporary files when done. -All the user has to do to install the software contained in such an -archive is to "run" the archive, i.e **sh nice-software.run**. I recommend -using the ".run" (which was introduced by some Makeself archives released by -Loki Software) or ".sh" suffix for such archives not to confuse the users, -so that they will know they are actually shell scripts (with quite a lot of binary data -attached to them though!). - -I am trying to keep the code of this script as portable as possible, i.e it is -not relying on any bash-specific features and only calls commands that are -installed on any functioning UNIX-compatible system. This script as well as -the archives it generates should run on any Unix flavor, with any compatible -Bourne shell, provided of course that the compression programs are available. - -As of version 2.1, Makeself has been rewritten and tested on the following -platforms : - - * Linux (all distributions) - * Sun Solaris (8 and above) - * HP-UX (tested on 11.0 and 11i on HPPA RISC) - * SCO OpenUnix and OpenServer - * IBM AIX 5.1L - * macOS (Darwin) - * SGI IRIX 6.5 - * FreeBSD - * UnicOS / Cray - * Cygwin (Windows) - -If you successfully run Makeself and/or archives created with it on another -system, then please [let me know][2]! - -Examples of publicly available archives made using makeself are : - - * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; - * All game patches released by [Loki Software][4] for the Linux version of popular games ; - * The [nVidia drivers][5] for Linux - * The installer for the Linux version of [Google Earth][6] - * The [VirtualBox][7] installers for Linux - * The [Makeself][1] distribution itself ;-) - * and countless others... - -**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : - -`AddType application/x-makeself .run` - -**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : - -`export _POSIX2_VERSION=199209` - -## Usage - -The syntax of makeself is the following: - -``` -makeself.sh [args] archive_dir file_name label startup_script [script_args] -``` - - * _args_ are optional options for Makeself. The available ones are : - - * **`--version`** : Prints the version number on stdout, then exits immediately - * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) - * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. - * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. - * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. - * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. - * **`--pigz`** : Use pigz for compression. - * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). - * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. - * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. - * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. - * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. - * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) - * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. - * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). - * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. - * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. - * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. - * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. - * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. - * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. - * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. - * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. - * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. - * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. - * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. - * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). - * **`--tar-extra opt`** : Append more options to the tar command line. - - For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` - - * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. - * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. - * **`--license`** : Append a license file. - * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. - * **`--help-header file`** : Add a header to the archive's `--help` output. - * `archive_dir` is the name of the directory that contains the files to be archived - * `file_name` is the name of the archive to be created - * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. - * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. - -Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named -**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : - -`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -` - -Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : - -`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` - -Archives generated with Makeself can be passed the following arguments: - - * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. - * **`--verbose`** : Will prompt the user before executing the embedded command - * **`--target dir`** : Allows to extract the archive in an arbitrary place. - * **`--nox11`** : Do not spawn a X11 terminal. - * **`--confirm`** : Prompt the user for confirmation before running the embedded command. - * **`--info`** : Print out general information about the archive (does not extract). - * **`--lsm`** : Print out the LSM entry, if it is present. - * **`--list`** : List the files in the archive. - * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. - * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. - * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. - * **`--noexec`** : Do not run the embedded script after extraction. - * **`--noexec-cleanup`** : Do not run the embedded cleanup script. - * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. - * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. - -Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. - -## Startup Script - -The startup script must be a regular Shell script. - -Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. - -`my-self-extracting-script.sh --fooBarFileParameter foo.bar` - -## Building and Testing - -Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. - -* To make a release: `make` -* To run all tests: `make test` - -## Maven Usage - -Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. - -## License - -Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. - -## Contributing - -I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: - - * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. - * Please explain clearly what the purpose of the patch is, and how you achieved it. - -## Download - -Get the latest official distribution [here][9] (version 2.4.2). - -The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. - -## Version history - - * **v1.0:** Initial public release - * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan - * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. - * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. - * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. - * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. - * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. - * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. - * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. - * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. - * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. - * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. - * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. - * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). - * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) - * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. - * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overridden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. - * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. - * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. - * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. - * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. - * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. - * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) - -## Links - - * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. - * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] - -## Contact - -This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. - -Contributions were included from John C. Quillan, Bjarni R. Einarsson, -Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot -your name, don't hesitate to contact me. - -This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. - -* * * - -[Stephane Peter][2] - - [1]: http://makeself.io/ - [2]: mailto:megastep@megastep.org - [3]: http://www.idsoftware.com/ - [4]: http://www.lokigames.com/products/myth2/updates.php3 - [5]: http://www.nvidia.com/ - [6]: http://earth.google.com/ - [7]: http://www.virtualbox.org/ - [8]: http://www.gnu.org/copyleft/gpl.html - [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run - [10]: https://github.com/megastep/makeself - [11]: https://github.com/megastep/loki_setup/ - [12]: http://www.unrealtournament2003.com/ - [13]: http://www.icculus.org/ - [14]: http://bre.klaki.net/programs/setup.sh/ - [15]: https://stephanepeter.com/ diff --git a/csrc/deepep/ops/cmake/util/makeself/VERSION b/csrc/deepep/ops/cmake/util/makeself/VERSION deleted file mode 100644 index 59aa62c1f..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/VERSION +++ /dev/null @@ -1 +0,0 @@ -2.4.5 diff --git a/csrc/deepep/ops/cmake/util/makeself/make-release.sh b/csrc/deepep/ops/cmake/util/makeself/make-release.sh deleted file mode 100755 index 65d698f25..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/make-release.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# -# Create a distributable archive of the current version of Makeself - -VER=`cat VERSION` -mkdir -p /tmp/makeself-$VER release -cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ -./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" diff --git a/csrc/deepep/ops/cmake/util/makeself/makeself-header.sh b/csrc/deepep/ops/cmake/util/makeself/makeself-header.sh deleted file mode 100644 index 23ffc483e..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/makeself-header.sh +++ /dev/null @@ -1,660 +0,0 @@ -cat << EOF > "$archname" -#!/bin/bash -# This script was generated using Makeself $MS_VERSION -# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) -# 2022.3.19-Modified the MS_Help function and some options -# Huawei Technologies Co., Ltd. - -ORIG_UMASK=\`umask\` - -CRCsum="$CRCsum" -MD5="$MD5sum" -SHA="$SHAsum" -SIGNATURE="$Signature" -TMPROOT=\${TMPDIR:="\$HOME"} -if ! test -d "\$TMPROOT"; then - TMPROOT="\$PWD" -fi -export TMPDIR="\$TMPROOT" -USER_PWD="\$PWD" -if ! test -d "\$USER_PWD"; then - exit 1 -fi -export USER_PWD -ARCHIVE_DIR=\`dirname "\$0"\` -export ARCHIVE_DIR - -name_of_file="\$0 " -pwd_of_file="\$PWD" -label="$LABEL" -script="$SCRIPT" -scriptargs="$SCRIPTARGS" -cleanup_script="${CLEANUP_SCRIPT}" -licensetxt="$LICENSE" -helpheader='$HELPHEADER' -targetdir="$archdirname" -filesizes="$filesizes" -totalsize="$totalsize" -keep="$KEEP" -nooverwrite="$NOOVERWRITE" -quiet="n" -accept="n" -nodiskspace="n" -export_conf="$EXPORT_CONF" -decrypt_cmd="$DECRYPT_CMD" -skip="$SKIP" - -print_cmd_arg="" -if type printf > /dev/null; then - print_cmd="printf" -elif test -x /usr/ucb/echo; then - print_cmd="/usr/ucb/echo" -else - print_cmd="echo" -fi - -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:\$PATH - export PATH -fi - -if test -d /usr/sfw/bin; then - PATH=\$PATH:/usr/sfw/bin - export PATH -fi - -unset CDPATH - -MS_Printf() -{ - \$print_cmd \$print_cmd_arg "\$1" -} - -MS_PrintLicense() -{ - PAGER=\${PAGER:=more} - if test x"\$licensetxt" != x; then - PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` - if test -x "\$PAGER_PATH"; then - echo "\$licensetxt" | \$PAGER - else - echo "\$licensetxt" - fi - if test x"\$accept" != xy; then - while true - do - MS_Printf "Please type y to accept, n otherwise: " - read yn - if test x"\$yn" = xn; then - keep=n - eval \$finish; exit 1 - break; - elif test x"\$yn" = xy; then - break; - fi - done - fi - fi -} - -MS_diskspace() -{ - ( - df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' - ) -} - -MS_dd() -{ - blocks=\`expr \$3 / 1024\` - bytes=\`expr \$3 % 1024\` - # Test for ibs, obs and conv feature - if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then - dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ - { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ - test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null - else - dd if="\$1" bs=\$2 skip=1 2> /dev/null - fi -} - -MS_dd_Progress() -{ - if test x"\$noprogress" = xy; then - MS_dd "\$@" - return \$? - fi - file="\$1" - offset=\$2 - length=\$3 - pos=0 - bsize=4194304 - while test \$bsize -gt \$length; do - bsize=\`expr \$bsize / 4\` - done - blocks=\`expr \$length / \$bsize\` - bytes=\`expr \$length % \$bsize\` - ( - dd ibs=\$offset skip=1 2>/dev/null - pos=\`expr \$pos \+ \$bsize\` - MS_Printf " 0%% " 1>&2 - if test \$blocks -gt 0; then - while test \$pos -le \$length; do - dd bs=\$bsize count=1 2>/dev/null - pcent=\`expr \$length / 100\` - pcent=\`expr \$pos / \$pcent\` - if test \$pcent -lt 100; then - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - if test \$pcent -lt 10; then - MS_Printf " \$pcent%% " 1>&2 - else - MS_Printf " \$pcent%% " 1>&2 - fi - fi - pos=\`expr \$pos \+ \$bsize\` - done - fi - if test \$bytes -gt 0; then - dd bs=\$bytes count=1 2>/dev/null - fi - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - MS_Printf " 100%% " 1>&2 - ) < "\$file" -} - -MS_Help() -{ - cat << EOH >&2 -Usage: \$0 [options] -Options: - --help | -h Print this message - --info Print embedded info : title, default target directory, embedded script ... - --list Print the list of files in the archive - --check Checks integrity and version dependency of the archive - --quiet Quiet install mode, skip human-computer interactions - --nox11 Do not spawn an xterm - --noexec Do not run embedded script - --extract= Extract directly to a target directory (absolute or relative) - Usually used with --noexec to just extract files without running - --tar arg1 [arg2 ...] Access the contents of the archive through the tar command -\${helpheader} -EOH -} - -MS_Verify_Sig() -{ - GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - temp_sig=\`mktemp -t XXXXX\` - echo \$SIGNATURE | base64 --decode > "\$temp_sig" - gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` - gpg_res=\$? - rm -f "\$temp_sig" - if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then - if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then - test x"\$quiet" = xn && echo "GPG signature is good" >&2 - else - echo "GPG Signature key does not match" >&2 - exit 2 - fi - else - test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 - exit 2 - fi -} - -MS_Check() -{ - OLD_PATH="\$PATH" - PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` - PATH="\$OLD_PATH" - - SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` - test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` - - if test x"\$quiet" = xn; then - MS_Printf "Verifying archive integrity..." - fi - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - fsize=\`cat "\$1" | wc -c | tr -d " "\` - if test \$totalsize -ne \`expr \$fsize - \$offset\`; then - echo " Unexpected archive size." >&2 - exit 2 - fi - verb=\$2 - i=1 - for s in \$filesizes - do - crc=\`echo \$CRCsum | cut -d" " -f\$i\` - if test -x "\$SHA_PATH"; then - if test x"\`basename \$SHA_PATH\`" = xshasum; then - SHA_ARG="-a 256" - fi - sha=\`echo \$SHA | cut -d" " -f\$i\` - if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 - else - shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; - if test x"\$shasum" != x"\$sha"; then - echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " SHA256 checksums are OK." >&2 - fi - crc="0000000000"; - fi - fi - if test -x "\$MD5_PATH"; then - if test x"\`basename \$MD5_PATH\`" = xdigest; then - MD5_ARG="-a md5" - fi - md5=\`echo \$MD5 | cut -d" " -f\$i\` - if test x"\$md5" = x00000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 - else - md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; - if test x"\$md5sum" != x"\$md5"; then - echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " MD5 checksums are OK." >&2 - fi - crc="0000000000"; verb=n - fi - fi - if test x"\$crc" = x0000000000; then - test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 - else - sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` - if test x"\$sum1" != x"\$crc"; then - echo "Error in checksums: \$sum1 is different from \$crc" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " CRC checksums are OK." >&2 - fi - fi - i=\`expr \$i + 1\` - offset=\`expr \$offset + \$s\` - done - if test x"\$quiet" = xn; then - echo " All good." - fi -} - -MS_Decompress() -{ - if test x"\$decrypt_cmd" != x""; then - { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" - else - eval "$GUNZIP_CMD" - fi - - if test \$? -ne 0; then - echo " ... Decompression failed." >&2 - fi -} - -UnTAR() -{ - if test x"\$quiet" = xn; then - tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } - else - tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } - fi -} - -MS_exec_cleanup() { - if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then - cleanup=n - cd "\$tmpdir" - eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" - fi -} - -MS_cleanup() -{ - echo 'Signal caught, cleaning up' >&2 - MS_exec_cleanup - cd "\$TMPROOT" - rm -rf "\$tmpdir" - eval \$finish; exit 15 -} - -Script_Args_Check() -{ - script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) - arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) - - for arg in \${script_supported_args}; - do - if test x"\$arg_to_test" = x"\$arg" ;then - return - fi - done - - MS_Help - exit 1 -} - -finish=true -xterm_loop= -noprogress=$NOPROGRESS -nox11=$NOX11 -copy=$COPY -ownership=$OWNERSHIP -verbose=n -cleanup=y -cleanupargs= -sig_key= - -initargs="\$@" - -while [ -n "\$*" ] -do - case "\$1" in - -h | --help) - MS_Help - exit 0 - ;; - -q | --quiet) - quiet=y - noprogress=y - shift - ;; - --info) - echo Identification: "\$label" - echo Target directory: "\$targetdir" - echo Uncompressed size: $USIZE KB - echo Compression: $COMPRESS - if test x"$ENCRYPT" != x""; then - echo Encryption: $ENCRYPT - fi - echo Date of packaging: $DATE - echo Built with Makeself version $MS_VERSION - echo Build command was: "$MS_COMMAND" - if test x"\$script" != x; then - echo Script run after extraction: - echo " " \$script \$scriptargs - fi - if test x"$copy" = xcopy; then - echo "Archive will copy itself to a temporary location" - fi - if test x"$NEED_ROOT" = xy; then - echo "Root permissions required for extraction" - fi - if test x"$KEEP" = xy; then - echo "directory \$targetdir is permanent" - else - echo "\$targetdir will be removed after extraction" - fi - exit 0 - ;; - --list) - echo Target directory: \$targetdir - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --tar) - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - arg1="\$2" - shift 2 || { MS_Help; exit 1; } - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --check) - MS_Check "\$0" y - scriptargs="\$scriptargs \$1" - shift - ;; - --noexec) - script="" - cleanup_script="" - shift - ;; - --extract=*) - keep=y - targetdir=\`echo \$1 | cut -d"=" -f2 \` - if ! shift; then MS_Help; exit 1; fi - ;; - --nox11) - nox11=y - shift - ;; - --xwin) - if test "$NOWAIT" = n; then - finish="echo Press Return to close this window...; read junk" - fi - xterm_loop=1 - shift - ;; - --phase2) - copy=phase2 - shift - ;; - --repack | --repack-path=*) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - if [[ ! "\$1" =~ ^-.* ]]; then - scriptargs="\$scriptargs '\$1'" - shift - fi - ;; - *) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - ;; - esac -done - -quiet_para="" -if test x"\$quiet" = xy; then - quiet_para="--quiet " -fi -scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" - -if test x"\$quiet" = xy -a x"\$verbose" = xy; then - echo Cannot be verbose and quiet at the same time. >&2 - exit 1 -fi - -if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then - echo "Administrative privileges required for this archive (use su or sudo)" >&2 - exit 1 -fi - -if test x"\$copy" \!= xphase2; then - MS_PrintLicense -fi - -case "\$copy" in -copy) - tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ - mkdir "\$tmpdir" || { - echo "Could not create temporary directory \$tmpdir" >&2 - exit 1 - } - SCRIPT_COPY="\$tmpdir/makeself" - echo "Copying to a temporary location..." >&2 - cp "\$0" "\$SCRIPT_COPY" - chmod +x "\$SCRIPT_COPY" - cd "\$TMPROOT" - exec "\$SCRIPT_COPY" --phase2 -- \$initargs - ;; -phase2) - finish="\$finish ; rm -rf \`dirname \$0\`" - ;; -esac - -if test x"\$nox11" = xn; then - if tty -s; then # Do we have a terminal? - : - else - if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? - if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable - GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" - for a in \$GUESS_XTERMS; do - if type \$a >/dev/null 2>&1; then - XTERM=\$a - break - fi - done - chmod a+x \$0 || echo Please add execution rights on \$0 - if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! - exec \$XTERM -e "\$0 --xwin \$initargs" - else - exec \$XTERM -e "./\$0 --xwin \$initargs" - fi - fi - fi - fi -fi - -if test x"\$targetdir" = x.; then - tmpdir="." -else - if test x"\$keep" = xy; then - if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then - echo "Target directory \$targetdir already exists, aborting." >&2 - exit 1 - fi - if test x"\$quiet" = xn; then - echo "Creating directory \$targetdir" >&2 - fi - tmpdir="\$targetdir" - dashp="-p" - else - tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" - dashp="" - fi - mkdir \$dashp "\$tmpdir" || { - echo 'Cannot create target directory' \$tmpdir >&2 - echo 'You should try option --extract=' >&2 - eval \$finish - exit 1 - } -fi - -location="\`pwd\`" -if test x"\$SETUP_NOCHECK" != x1; then - MS_Check "\$0" -fi -offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - -if test x"\$verbose" = xy; then - MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " - read yn - if test x"\$yn" = xn; then - eval \$finish; exit 1 - fi -fi - -if test x"\$quiet" = xn; then - # Decrypting with openssl will ask for password, - # the prompt needs to start on new line - if test x"$ENCRYPT" = x"openssl"; then - echo "Decrypting and uncompressing \$label..." - else - MS_Printf "Uncompressing \$label" - fi -fi -res=3 -if test x"\$keep" = xn; then - trap MS_cleanup 1 2 3 15 -fi - -if test x"\$nodiskspace" = xn; then - leftspace=\`MS_diskspace "\$tmpdir"\` - if test -n "\$leftspace"; then - if test "\$leftspace" -lt $USIZE; then - echo - echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 - if test x"\$keep" = xn; then - echo "Consider setting TMPDIR to a directory with more free space." - fi - eval \$finish; exit 1 - fi - fi -fi - -for s in \$filesizes -do - if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then - if test x"\$ownership" = xy; then - (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) - fi - else - echo >&2 - echo "Unable to decompress \$0" >&2 - eval \$finish; exit 1 - fi - offset=\`expr \$offset + \$s\` -done -if test x"\$quiet" = xn; then - echo -fi - -cd "\$tmpdir" -res=0 -if test x"\$script" != x; then - if test x"\$export_conf" = x"y"; then - MS_BUNDLE="\$0" - MS_LABEL="\$label" - MS_SCRIPT="\$script" - MS_SCRIPTARGS="\$scriptargs" - MS_ARCHDIRNAME="\$archdirname" - MS_KEEP="\$KEEP" - MS_NOOVERWRITE="\$NOOVERWRITE" - MS_COMPRESS="\$COMPRESS" - MS_CLEANUP="\$cleanup" - export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS - export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS - fi - - if test x"\$verbose" = x"y"; then - yn="x" - while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN - do - MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " - read yn - if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; - elif test x"\$yn" = xn -o x"\$yn" = xN; then - echo "Unable to decompress \$script ,because of aborting! ";res=\$? - else - echo "Input value is unacceptable,please try again." - fi - done - else - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? - fi - if test "\$res" -ne 0; then - test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 - fi -fi - -MS_exec_cleanup - -if test x"\$keep" = xn; then - cd "\$TMPROOT" - rm -rf "\$tmpdir" -fi -eval \$finish; exit \$res -EOF diff --git a/csrc/deepep/ops/cmake/util/makeself/makeself.1 b/csrc/deepep/ops/cmake/util/makeself/makeself.1 deleted file mode 100644 index 81bf6e4ff..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/makeself.1 +++ /dev/null @@ -1,110 +0,0 @@ -.TH "MAKESELF" "1" "2.4.5" -.SH "NAME" -makeself \- An utility to generate self-extractable archives. -.SH "SYNTAX" -.B makeself [\fIoptions\fP] archive_dir file_name label -.B [\fIstartup_script\fP] [\fIargs\fP] -.SH "DESCRIPTION" -This program is a free (GPL) utility designed to create self-extractable -archives from a directory. -.SH "OPTIONS" -The following options are supported. -.TP 15 -.B -v, --version -Prints out the makeself version number and exits. -.TP -.B -h, --help -Print out help information. -.TP -.B --tar-quietly -Suppress verbose output from the tar command -.TP -.B --quiet -Do not print any messages other than errors -.TP -.B --gzip -Compress using gzip (default if detected). -.TP -.B --bzip2 -Compress using bzip2. -.TP -.B --pbzip2 -Compress using pbzip2. -.TP -.B --xz -Compress using xz. -.TP -.B --lzo -Compress using lzop. -.TP -.B --lz4 -Compress using lz4. -.TP -.B --compress -Compress using the UNIX 'compress' command. -.TP -.B --nocomp -Do not compress the data. -.TP -.B --complevel lvl -Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 -.TP -.B --notemp -The archive will create archive_dir in the current directory and -uncompress in ./archive_dir. -.TP -.B --copy -Upon extraction, the archive will first copy itself to a temporary directory. -.TP -.B --append -Append more files to an existing makeself archive. The label and startup scripts will then be ignored. -.TP -.B --current -Files will be extracted to the current directory. Both --current and --target dir imply --notemp. -.TP -.B --target dir -Extract directly to a target directory. Directory path can be either absolute or relative. -.TP -.B --header file -Specify location of the header script. -.TP -.B --cleanup file -Specify a cleanup script that executes on interrupt and when finished successfully. -.TP -.B --follow -Follow the symlinks in the archive. -.TP -.B --noprogress -Do not show the progress during the decompression. -.TP -.B --nox11 -Disable automatic spawn of an xterm if running in X11. -.TP -.B --nowait -Do not wait for user input after executing embedded program from an xterm. -.TP -.B --nomd5 -Do not create a MD5 checksum for the archive. -.TP -.B --nocrc -Do not create a CRC32 checksum for the archive. -.TP -.B --lsm file -LSM file describing the package. -.B --packaging-date date -Use provided string as the packaging date instead of the current date. -.SH "EXAMPLES" -Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, -and he wants to generate a self-extracting package named mysoft.sh, which will launch -the "setup" script initially stored in /home/joe/mysoft: -.TP -makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -.TP -Here is also how I created the makeself.run archive which contains the Makeself distribution: -.TP -makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" -.SH "AUTHORS" -Makeself has been written by Stéphane Peter . -.BR -This man page was originally written by Bartosz Fenski for the -Debian GNU/Linux distribution (but it may be used by others). diff --git a/csrc/deepep/ops/cmake/util/makeself/makeself.lsm b/csrc/deepep/ops/cmake/util/makeself/makeself.lsm deleted file mode 100644 index 802cada3a..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/makeself.lsm +++ /dev/null @@ -1,16 +0,0 @@ -Begin3 -Title: makeself.sh -Version: 2.4.5 -Description: makeself.sh is a shell script that generates a self-extractable - tar.gz archive from a directory. The resulting file appears as a shell - script, and can be launched as is. The archive will then uncompress - itself to a temporary directory and an arbitrary command will be - executed (for example an installation script). This is pretty similar - to archives generated with WinZip Self-Extractor in the Windows world. -Keywords: Installation archive tar winzip -Author: Stephane Peter (megastep@megastep.org) -Maintained-by: Stephane Peter (megastep@megastep.org) -Original-site: https://makeself.io/ -Platform: Unix -Copying-policy: GPL -End diff --git a/csrc/deepep/ops/cmake/util/makeself/makeself.sh b/csrc/deepep/ops/cmake/util/makeself/makeself.sh deleted file mode 100755 index 60ced4a8a..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/makeself.sh +++ /dev/null @@ -1,822 +0,0 @@ -#!/bin/sh -# -# Makeself version 2.4.x -# by Stephane Peter -# -# Utility to create self-extracting tar.gz archives. -# The resulting archive is a file holding the tar.gz archive with -# a small Shell script stub that uncompresses the archive to a temporary -# directory and then executes a given script from within that directory. -# -# Makeself home page: https://makeself.io/ -# -# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. -# -# Version history : -# - 1.0 : Initial public release -# - 1.1 : The archive can be passed parameters that will be passed on to -# the embedded script, thanks to John C. Quillan -# - 1.2 : Package distribution, bzip2 compression, more command line options, -# support for non-temporary archives. Ideas thanks to Francois Petitjean -# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: -# Support for no compression (--nocomp), script is no longer mandatory, -# automatic launch in an xterm, optional verbose output, and -target -# archive option to indicate where to extract the files. -# - 1.4 : Improved UNIX compatibility (Francois Petitjean) -# Automatic integrity checking, support of LSM files (Francois Petitjean) -# - 1.5 : Many bugfixes. Optionally disable xterm spawning. -# - 1.5.1 : More bugfixes, added archive options -list and -check. -# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big -# archives (Quake III demo) -# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. -# More verbosity in xterms and check for embedded command's return value. -# Bugfix for Debian 2.0 systems that have a different "print" command. -# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. -# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to -# bypass checksum verification of archives. -# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) -# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. -# - 2.0.1 : Added --copy -# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. -# Added --nochown for archives -# Stopped doing redundant checksums when not necessary -# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command -# Cleaned up the code to handle error codes from compress. Simplified the extraction code. -# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. -# - 2.1.3 : Bug fixes with command line when spawning terminals. -# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. -# Added --noexec to prevent execution of embedded scripts. -# Added --nomd5 and --nocrc to avoid creating checksums in archives. -# Added command used to create the archive in --info output. -# Run the embedded script through eval. -# - 2.1.4 : Fixed --info output. -# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) -# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) -# Avoid some race conditions (Ludwig Nussel) -# Unset the $CDPATH variable to avoid problems if it is set. (Debian) -# Better handling of dot files in the archive directory. -# - 2.1.5 : Made the md5sum detection consistent with the header code. -# Check for the presence of the archive directory -# Added --encrypt for symmetric encryption through gpg (Eric Windisch) -# Added support for the digest command on Solaris 10 for MD5 checksums -# Check for available disk space before extracting to the target directory (Andreas Schweitzer) -# Allow extraction to run asynchronously (patch by Peter Hatch) -# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) -# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spinning cursor (Guy Baconniere) -# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) -# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) -# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. -# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) -# - 2.4.0 : Optional support for SHA256 checksums in archives. -# - 2.4.2 : Add support for threads for several compressors. (M. Limber) -# Added zstd support. -# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. -# - 2.4.5 : Added --tar-format to override ustar tar archive format -# -# (C) 1998-2021 by Stephane Peter -# -# This software is released under the terms of the GNU GPL version 2 and above -# Please read the license at http://www.gnu.org/copyleft/gpl.html -# Self-extracting archives created with this script are explicitly NOT released under the term of the GPL -# - -MS_VERSION=2.4.5 -MS_COMMAND="$0" -unset CDPATH - -for f in ${1+"$@"}; do - MS_COMMAND="$MS_COMMAND \\\\ - \\\"$f\\\"" -done - -# For Solaris systems -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:$PATH - export PATH -fi - -# Procedures - -MS_Usage() -{ - echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" - echo "args can be one or more of the following :" - echo " --version | -v : Print out Makeself version number and exit" - echo " --help | -h : Print out this help message" - echo " --tar-quietly : Suppress verbose output from the tar command" - echo " --quiet | -q : Do not print any messages other than errors." - echo " --gzip : Compress using gzip (default if detected)" - echo " --pigz : Compress with pigz" - echo " --zstd : Compress with zstd" - echo " --bzip2 : Compress using bzip2 instead of gzip" - echo " --pbzip2 : Compress using pbzip2 instead of gzip" - echo " --xz : Compress using xz instead of gzip" - echo " --lzo : Compress using lzop instead of gzip" - echo " --lz4 : Compress using lz4 instead of gzip" - echo " --compress : Compress using the UNIX 'compress' command" - echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" - echo " --threads thds : Number of threads to be used by compressors that support parallelization." - echo " Omit to use compressor's default. Most useful (and required) for opting" - echo " into xz's threading, usually with '--threads=0' for all available cores." - echo " pbzip2 and pigz are parallel by default, and setting this value allows" - echo " limiting the number of threads they use." - echo " --base64 : Instead of compressing, encode the data using base64" - echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" - echo " --gpg-asymmetric-encrypt-sign" - echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" - echo " --gpg-extra opt : Append more options to the gpg command line" - echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" - echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" - echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" - echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." - echo " If this option is not supplied, the user will be asked to enter" - echo " encryption password on the current terminal." - echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." - echo " --nochown : Do not give the target folder to the current user (default)" - echo " --chown : Give the target folder to the current user recursively" - echo " --nocomp : Do not compress the data" - echo " --notemp : The archive will create archive_dir in the" - echo " current directory and uncompress in ./archive_dir" - echo " --needroot : Check that the root user is extracting the archive before proceeding" - echo " --copy : Upon extraction, the archive will first copy itself to" - echo " a temporary directory" - echo " --append : Append more files to an existing Makeself archive" - echo " The label and startup scripts will then be ignored" - echo " --target dir : Extract directly to a target directory" - echo " directory path can be either absolute or relative" - echo " --nooverwrite : Do not extract the archive if the specified target directory exists" - echo " --current : Files will be extracted to the current directory" - echo " Both --current and --target imply --notemp" - echo " --tar-format opt : Specify a tar archive format (default is ustar)" - echo " --tar-extra opt : Append more options to the tar command line" - echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" - echo " --nomd5 : Don't calculate an MD5 for archive" - echo " --nocrc : Don't calculate a CRC for archive" - echo " --sha256 : Compute a SHA256 checksum for the archive" - echo " --header file : Specify location of the header script" - echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." - echo " --follow : Follow the symlinks in the archive" - echo " --noprogress : Do not show the progress during the decompression" - echo " --nox11 : Disable automatic spawn of a xterm" - echo " --nowait : Do not wait for user input after executing embedded" - echo " program from an xterm" - echo " --sign passphrase : Signature private key to sign the package with" - echo " --lsm file : LSM file describing the package" - echo " --license file : Append a license file" - echo " --help-header file : Add a header to the archive's --help output" - echo " --packaging-date date" - echo " : Use provided string as the packaging date" - echo " instead of the current date." - echo - echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." - echo " --export-conf : Export configuration variables to startup_script" - echo - echo "Do not forget to give a fully qualified startup script name" - echo "(i.e. with a ./ prefix if inside the archive)." - exit 1 -} - -# Default settings -if type gzip >/dev/null 2>&1; then - COMPRESS=gzip -elif type compress >/dev/null 2>&1; then - COMPRESS=compress -else - echo "ERROR: missing commands: gzip, compress" >&2 - MS_Usage -fi -ENCRYPT=n -PASSWD="" -PASSWD_SRC="" -OPENSSL_NO_MD=n -COMPRESS_LEVEL=9 -DEFAULT_THREADS=123456 # Sentinel value -THREADS=$DEFAULT_THREADS -KEEP=n -CURRENT=n -NOX11=n -NOWAIT=n -APPEND=n -TAR_QUIETLY=n -KEEP_UMASK=n -QUIET=n -NOPROGRESS=n -COPY=none -NEED_ROOT=n -TAR_ARGS=rvf -TAR_FORMAT=ustar -TAR_EXTRA="" -GPG_EXTRA="" -DU_ARGS=-ks -HEADER=`dirname "$0"`/makeself-header.sh -SIGNATURE="" -TARGETDIR="" -NOOVERWRITE=n -DATE=`LC_ALL=C date` -EXPORT_CONF=n -SHA256=n -OWNERSHIP=n -SIGN=n -GPG_PASSPHRASE="" - -# LSM file stuff -LSM_CMD="echo No LSM. >> \"\$archname\"" - -while true -do - case "$1" in - --version | -v) - echo Makeself version $MS_VERSION - exit 0 - ;; - --pbzip2) - COMPRESS=pbzip2 - shift - ;; - --bzip2) - COMPRESS=bzip2 - shift - ;; - --gzip) - COMPRESS=gzip - shift - ;; - --pigz) - COMPRESS=pigz - shift - ;; - --zstd) - COMPRESS=zstd - shift - ;; - --xz) - COMPRESS=xz - shift - ;; - --lzo) - COMPRESS=lzo - shift - ;; - --lz4) - COMPRESS=lz4 - shift - ;; - --compress) - COMPRESS=compress - shift - ;; - --base64) - COMPRESS=base64 - shift - ;; - --gpg-encrypt) - COMPRESS=gpg - shift - ;; - --gpg-asymmetric-encrypt-sign) - COMPRESS=gpg-asymmetric - shift - ;; - --gpg-extra) - GPG_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-encrypt) - ENCRYPT=openssl - shift - ;; - --ssl-passwd) - PASSWD=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-pass-src) - PASSWD_SRC=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-no-md) - OPENSSL_NO_MD=y - shift - ;; - --nocomp) - COMPRESS=none - shift - ;; - --complevel) - COMPRESS_LEVEL="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --threads) - THREADS="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nochown) - OWNERSHIP=n - shift - ;; - --chown) - OWNERSHIP=y - shift - ;; - --notemp) - KEEP=y - shift - ;; - --copy) - COPY=copy - shift - ;; - --current) - CURRENT=y - KEEP=y - shift - ;; - --tar-format) - TAR_FORMAT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --tar-extra) - TAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --untar-extra) - UNTAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --target) - TARGETDIR="$2" - KEEP=y - shift 2 || { MS_Usage; exit 1; } - ;; - --sign) - SIGN=y - GPG_PASSPHRASE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nooverwrite) - NOOVERWRITE=y - shift - ;; - --needroot) - NEED_ROOT=y - shift - ;; - --header) - HEADER="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --cleanup) - CLEANUP_SCRIPT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --license) - # We need to escape all characters having a special meaning in double quotes - LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") - shift 2 || { MS_Usage; exit 1; } - ;; - --follow) - TAR_ARGS=rvhf - DU_ARGS=-ksL - shift - ;; - --noprogress) - NOPROGRESS=y - shift - ;; - --nox11) - NOX11=y - shift - ;; - --nowait) - NOWAIT=y - shift - ;; - --nomd5) - NOMD5=y - shift - ;; - --sha256) - SHA256=y - shift - ;; - --nocrc) - NOCRC=y - shift - ;; - --append) - APPEND=y - shift - ;; - --lsm) - LSM_CMD="cat \"$2\" >> \"\$archname\"" - shift 2 || { MS_Usage; exit 1; } - ;; - --packaging-date) - DATE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --help-header) - HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` - shift 2 || { MS_Usage; exit 1; } - [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER -" - ;; - --tar-quietly) - TAR_QUIETLY=y - shift - ;; - --keep-umask) - KEEP_UMASK=y - shift - ;; - --export-conf) - EXPORT_CONF=y - shift - ;; - -q | --quiet) - QUIET=y - shift - ;; - -h | --help) - MS_Usage - ;; - -*) - echo Unrecognized flag : "$1" - MS_Usage - ;; - *) - break - ;; - esac -done - -if test $# -lt 1; then - MS_Usage -else - if test -d "$1"; then - archdir="$1" - else - echo "Directory $1 does not exist." >&2 - exit 1 - fi -fi -archname="$2" - -if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then - if test "$TAR_ARGS" = "rvf"; then - TAR_ARGS="rf" - elif test "$TAR_ARGS" = "rvhf"; then - TAR_ARGS="rhf" - fi -fi - -if test "$APPEND" = y; then - if test $# -lt 2; then - MS_Usage - fi - - # Gather the info from the original archive - OLDENV=`sh "$archname" --dumpconf` - if test $? -ne 0; then - echo "Unable to update archive: $archname" >&2 - exit 1 - else - eval "$OLDENV" - OLDSKIP=`expr $SKIP + 1` - fi -else - if test "$KEEP" = n -a $# = 3; then - echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 - echo >&2 - MS_Usage - fi - # We don't want to create an absolute directory unless a target directory is defined - if test "$CURRENT" = y; then - archdirname="." - elif test x"$TARGETDIR" != x; then - archdirname="$TARGETDIR" - else - archdirname=`basename "$1"` - fi - - if test $# -lt 3; then - MS_Usage - fi - - LABEL="$3" - SCRIPT="$4" - test "x$SCRIPT" = x || shift 1 - shift 3 - SCRIPTARGS="$*" -fi - -if test "$KEEP" = n -a "$CURRENT" = y; then - echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 - exit 1 -fi - -case $COMPRESS in -gzip) - GZIP_CMD="gzip -c$COMPRESS_LEVEL" - GUNZIP_CMD="gzip -cd" - ;; -pigz) - GZIP_CMD="pigz -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --processes $THREADS" - fi - GUNZIP_CMD="gzip -cd" - ;; -zstd) - GZIP_CMD="zstd -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="zstd -cd" - ;; -pbzip2) - GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD -p$THREADS" - fi - GUNZIP_CMD="bzip2 -d" - ;; -bzip2) - GZIP_CMD="bzip2 -$COMPRESS_LEVEL" - GUNZIP_CMD="bzip2 -d" - ;; -xz) - GZIP_CMD="xz -c$COMPRESS_LEVEL" - # Must opt-in by specifying a value since not all versions of xz support threads - if test $THREADS -ne $DEFAULT_THREADS; then - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="xz -d" - ;; -lzo) - GZIP_CMD="lzop -c$COMPRESS_LEVEL" - GUNZIP_CMD="lzop -d" - ;; -lz4) - GZIP_CMD="lz4 -c$COMPRESS_LEVEL" - GUNZIP_CMD="lz4 -d" - ;; -base64) - GZIP_CMD="base64" - GUNZIP_CMD="base64 --decode -i -" - ;; -gpg) - GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" - GUNZIP_CMD="gpg -d" - ENCRYPT="gpg" - ;; -gpg-asymmetric) - GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" - GUNZIP_CMD="gpg --yes -d" - ENCRYPT="gpg" - ;; -compress) - GZIP_CMD="compress -fc" - GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" - ;; -none) - GZIP_CMD="cat" - GUNZIP_CMD="cat" - ;; -esac - -if test x"$ENCRYPT" = x"openssl"; then - if test x"$APPEND" = x"y"; then - echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 - fi - - ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" - DECRYPT_CMD="openssl enc -aes-256-cbc -d" - - if test x"$OPENSSL_NO_MD" != x"y"; then - ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" - DECRYPT_CMD="$DECRYPT_CMD -md sha256" - fi - - if test -n "$PASSWD_SRC"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" - elif test -n "$PASSWD"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" - fi -fi - -tmpfile="${TMPDIR:-/tmp}/mkself$$" - -if test -f "$HEADER"; then - oldarchname="$archname" - archname="$tmpfile" - # Generate a fake header to count its lines - SKIP=0 - . "$HEADER" - SKIP=`cat "$tmpfile" |wc -l` - # Get rid of any spaces - SKIP=`expr $SKIP` - rm -f "$tmpfile" - if test "$QUIET" = "n"; then - echo "Header is $SKIP lines long" >&2 - fi - archname="$oldarchname" -else - echo "Unable to open header file: $HEADER" >&2 - exit 1 -fi - -if test "$QUIET" = "n"; then - echo -fi - -if test "$APPEND" = n; then - if test -f "$archname"; then - echo "WARNING: Overwriting existing file: $archname" >&2 - fi -fi - -USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` - -if test "." = "$archdirname"; then - if test "$KEEP" = n; then - archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" - fi -fi - -test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } -if test "$QUIET" = "n"; then - echo "About to compress $USIZE KB of data..." - echo "Adding files to archive named \"$archname\"..." -fi - -# See if we have GNU tar -TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` -test -x "$TAR" || TAR=tar - -tmparch="${TMPDIR:-/tmp}/mkself$$.tar" -( - if test "$APPEND" = "y"; then - tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" - fi - cd "$archdir" - # "Determining if a directory is empty" - # https://www.etalabs.net/sh_tricks.html - find . \ - \( \ - ! -type d \ - -o \ - \( -links 2 -exec sh -c ' - is_empty () ( - cd "$1" - set -- .[!.]* ; test -f "$1" && return 1 - set -- ..?* ; test -f "$1" && return 1 - set -- * ; test -f "$1" && return 1 - return 0 - ) - is_empty "$0"' {} \; \ - \) \ - \) -print \ - | LC_ALL=C sort \ - | sed 's/./\\&/g' \ - | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" -) || { - echo "ERROR: failed to create temporary archive: $tmparch" - rm -f "$tmparch" "$tmpfile" - exit 1 -} - -USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` - -eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { - echo "ERROR: failed to create temporary file: $tmpfile" - rm -f "$tmparch" "$tmpfile" - exit 1 -} -rm -f "$tmparch" - -if test x"$ENCRYPT" = x"openssl"; then - echo "About to encrypt archive \"$archname\"..." - { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ - { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } -fi - -fsize=`cat "$tmpfile" | wc -c | tr -d " "` - -# Compute the checksums - -shasum=0000000000000000000000000000000000000000000000000000000000000000 -md5sum=00000000000000000000000000000000 -crcsum=0000000000 - -if test "$NOCRC" = y; then - if test "$QUIET" = "n"; then - echo "skipping crc at user request" - fi -else - crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` - if test "$QUIET" = "n"; then - echo "CRC: $crcsum" - fi -fi - -if test "$SHA256" = y; then - SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` - if test -x "$SHA_PATH"; then - shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` - else - SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` - shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` - fi - if test "$QUIET" = "n"; then - if test -x "$SHA_PATH"; then - echo "SHA256: $shasum" - else - echo "SHA256: none, SHA command not found" - fi - fi -fi -if test "$NOMD5" = y; then - if test "$QUIET" = "n"; then - echo "Skipping md5sum at user request" - fi -else - # Try to locate a MD5 binary - OLD_PATH=$PATH - PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` - PATH=$OLD_PATH - if test -x "$MD5_PATH"; then - if test `basename ${MD5_PATH}`x = digestx; then - MD5_ARG="-a md5" - fi - md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` - if test "$QUIET" = "n"; then - echo "MD5: $md5sum" - fi - else - if test "$QUIET" = "n"; then - echo "MD5: none, MD5 command not found" - fi - fi -fi -if test "$SIGN" = y; then - GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` - if test -x "$GPG_PATH"; then - SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` - if test "$QUIET" = "n"; then - echo "Signature: $SIGNATURE" - fi - else - echo "Missing gpg command" >&2 - fi -fi - -totalsize=0 -for size in $fsize; -do - totalsize=`expr $totalsize + $size` -done - -if test "$APPEND" = y; then - mv "$archname" "$archname".bak || exit - - # Prepare entry for new archive - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - # Generate the header - . "$HEADER" - # Append the new data - cat "$tmpfile" >> "$archname" - - chmod +x "$archname" - rm -f "$archname".bak - if test "$QUIET" = "n"; then - echo "Self-extractable archive \"$archname\" successfully updated." - fi -else - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - - # Generate the header - . "$HEADER" - - # Append the compressed tar data after the stub - if test "$QUIET" = "n"; then - echo - fi - cat "$tmpfile" >> "$archname" - chmod +x "$archname" - if test "$QUIET" = "n"; then - echo Self-extractable archive \"$archname\" successfully created. - fi -fi -rm -f "$tmpfile" diff --git a/csrc/deepep/ops/cmake/util/makeself/run-tests.sh b/csrc/deepep/ops/cmake/util/makeself/run-tests.sh deleted file mode 100755 index 31ee16511..000000000 --- a/csrc/deepep/ops/cmake/util/makeself/run-tests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# Run every available test - Bash needed -cd test -for test in *test; -do - echo "Running test $test ..." - bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } -done diff --git a/csrc/deepep/ops/cmake/util/merge_aicpu_info_json.sh b/csrc/deepep/ops/cmake/util/merge_aicpu_info_json.sh deleted file mode 100755 index 970a44bf1..000000000 --- a/csrc/deepep/ops/cmake/util/merge_aicpu_info_json.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -echo $@ -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No project path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -if [[ ! -d "$ASCEND_OPP_PATH" ]]; then - echo "[ERROR] No opp install path is provided" - exit 1 -fi -custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json - -if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then - cp -f $custom_exist_info_json $temp_info_json - chmod +w $temp_info_json - python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} - cp -f $temp_info_json $custom_new_info_json - rm -f $temp_info_json -fi diff --git a/csrc/deepep/ops/cmake/util/opdesc_parser.py b/csrc/deepep/ops/cmake/util/opdesc_parser.py deleted file mode 100755 index 7b789567a..000000000 --- a/csrc/deepep/ops/cmake/util/opdesc_parser.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import sys - -OP_ALL = "__ALLOP__" -SOC_ALL = "__ALLSOC__" -SOC_TO_SHORT_SOC_MAP = { - "ascend910a": "ascend910", - "ascend910proa": "ascend910", - "ascend910b": "ascend910", - "ascend910prob": "ascend910", - "ascend910premiuma": "ascend910", - "ascend910b1": "ascend910b", - "ascend910b2": "ascend910b", - "ascend910b2c": "ascend910b", - "ascend910b3": "ascend910b", - "ascend910b4": "ascend910b", - "ascend910b4-1": "ascend910b", - "ascend910_9391": "ascend910_93", - "ascend910_9381": "ascend910_93", - "ascend910_9372": "ascend910_93", - "ascend910_9392": "ascend910_93", - "ascend910_9382": "ascend910_93", - "ascend910_9361": "ascend910_93", - "ascend310p1": "ascend310p", - "ascend310p3": "ascend310p", - "ascend310p3vir01": "ascend310p", - "ascend310p3vir02": "ascend310p", - "ascend310p3vir04": "ascend310p", - "ascend310p3vir08": "ascend310p", - "ascend310b1": "ascend310b", - "bs9sx1aa": "bs9sx1a", - "ascend610lite": "ascend610lite", -} -CONFLICT_KEYWORDS = { - "and", - "as", - "assert", - "break", - "class", - "continue", - "def", - "del", - "elif", - "else", - "except", - "finally", - "for", - "from", - "global", - "if", - "import", - "in", - "is", - "lambda", - "not", - "or", - "pass", - "raise", - "return", - "try", - "while", - "with", - "yield", - "False", - "None", - "True", - "nonlocal", - "arg", - "__inputs__", - "__outputs__", - "options", - "bisheng", - "bisheng_path", - "tikcpp_path", - "impl_mode", - "custom_compile_options", - "custom_all_compile_options", - "soc_version", - "soc_short", - "custom_compile_options_soc", - "custom_all_compile_options_soc", - "origin_func_name", - "ascendc_src_dir_ex", - "ascendc_src_dir", - "ascendc_src_file", - "src", - "op_type", - "code_channel", - "op_info", - "compile_op", - "get_code_channel", - "result", - "__attrs__", - "isinstance", - "attr", - "get_current_build_config", - "_build_args", - "get_dtype_fmt_options", - "shutil", - "os", - "get_kernel_source", -} - - -class OpDesc: - def __init__(self: any, op_type: str): - self.op_type = op_type - self.attr_list = [] - self.attr_val = {} - self.input_name = [] - self.input_ori_name = [] - self.input_type = [] - self.input_dtype = [] - self.input_dtype_for_bin_list = [] - self.input_dtype_for_bin = {} - self.input_fmt = [] - self.input_fmt_for_bin_list = [] - self.input_fmt_for_bin = {} - self.input_virt = {} - self.output_name = [] - self.output_ori_name = [] - self.output_type = [] - self.output_dtype = [] - self.output_dtype_for_bin_list = [] - self.output_dtype_for_bin = {} - self.output_fmt = [] - self.output_fmt_for_bin_list = [] - self.output_fmt_for_bin = {} - self.output_init_value = [] - self.output_shape_depend_on_compute = [] - self.op_fmt_sel = False - self.op_chk_support = False - self.op_intf = "" - self.kern_name = "" - self.op_file = "" - self.op_replay_flag = False - self.op_replay_batch = False - self.input_idx = -1 - self.output_idx = -1 - self.max_block_dim = 32 - self.max_shape_size = 268435456 - self.dynamic_shape = False - self.op_range_limit = "" - self.custom_compile_options = {} - self.custom_all_compile_options = {} - self.param_type_dynamic = False - self.mc2_ctx = [] - self.bin_cprs_list = [] - self.bin_cprs_head = [] - self.bin_save_list = [] - - @staticmethod - def _parse_digit(conf: str) -> int: - return int(conf.split("=")[1]) - - @staticmethod - def _parse_flag(conf: str) -> bool: - if "true" == conf.split("=")[1]: - return True - return False - - @staticmethod - def _parse_str(conf: str) -> str: - return conf.split("=")[1] - - @staticmethod - def _parse_list(conf: str) -> list: - return conf.split("=")[1].split(",") - - def parse_input(self: any, conf: str): - if conf.startswith("input{}.name".format(int(self.input_idx) + 1)): - self.input_idx += 1 - self.input_ori_name.append(self._parse_str(conf)) - self.input_name.append(self.input_ori_name[-1] + "_in__") - elif conf.startswith("input{}.paramType".format(int(self.input_idx))): - param_type = self._parse_str(conf) - self.input_type.append(param_type) - if param_type == "dynamic": - self.param_type_dynamic = True - elif conf.startswith("input{}.dtype".format(int(self.input_idx))): - self.input_dtype.append(self._parse_str(conf)) - elif conf.startswith("input{}.for_bin_dtype".format(int(self.input_idx))): - self.input_dtype_for_bin.update({self.input_idx: self._parse_str(conf)}) - elif conf.startswith("input{}.format".format(int(self.input_idx))): - self.input_fmt.append(self._parse_str(conf)) - elif conf.startswith("input{}.for_bin_format".format(int(self.input_idx))): - self.input_fmt_for_bin.update({self.input_idx: self._parse_str(conf)}) - elif conf.startswith("input{}.virtual".format(int(self.input_idx))): - self.input_virt[self.input_idx] = self._parse_str(conf) - elif conf.startswith("input{}.initValue".format(int(self.input_idx))): - raise Exception( - f"[ERROR]: Op: {{'{self.op_type}'}} input {self.input_ori_name[int(self.input_idx)]}\ - has InitValue, which is not support!" - ) - else: - return - - def parse_output(self: any, conf: str): - if conf.startswith("output{}.name".format(int(self.output_idx) + 1)): - self.output_idx += 1 - self.output_ori_name.append(self._parse_str(conf)) - self.output_name.append(self.output_ori_name[-1] + "_out_") - self.output_init_value.append(None) - elif conf.startswith("output{}.paramType".format(int(self.output_idx))): - param_type = self._parse_str(conf) - self.output_type.append(param_type) - if param_type == "dynamic": - self.param_type_dynamic = True - elif conf.startswith("output{}.dtype".format(int(self.output_idx))): - self.output_dtype.append(self._parse_str(conf)) - elif conf.startswith("output{}.for_bin_dtype".format(int(self.output_idx))): - self.output_dtype_for_bin.update({self.output_idx: self._parse_str(conf)}) - elif conf.startswith("output{}.format".format(int(self.output_idx))): - self.output_fmt.append(self._parse_str(conf)) - elif conf.startswith("output{}.for_bin_format".format(int(self.output_idx))): - self.output_fmt_for_bin.update({self.output_idx: self._parse_str(conf)}) - elif conf.startswith("output{}.initValue".format(int(self.output_idx))): - self.output_init_value[int(self.output_idx)] = self._parse_str(conf) - elif conf.startswith( - "output{}.outputShapeDependOnCompute=true".format(int(self.output_idx)) - ): - self.output_shape_depend_on_compute.append(int(self.output_idx)) - else: - return - - def parse_op_format(self: any, conf: str): - self.op_fmt_sel = self._parse_flag(conf) - - def parse_check_support(self: any, conf: str): - self.op_chk_support = self._parse_flag(conf) - - def parse_range_limit(self: any, conf: str): - self.op_range_limit = self._parse_str(conf) - - def parse_kern_name(self: any, conf: str): - self.kern_name = self._parse_str(conf) - - def parse_op_intf(self: any, conf: str): - self.op_intf = self._parse_str(conf) - - def parse_op_file(self: any, conf: str): - self.op_file = self._parse_str(conf) - - def parse_dynamic_shape(self: any, conf: str): - self.dynamic_shape = self._parse_flag(conf) - - def parse_attr_list(self: any, conf: str): - self.attr_list = self._parse_list(conf) - intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS - if intersection_element: - raise Exception( - f"[ERROR]: The attribute name: {intersection_element} in op: {{'{self.op_type}'}} \ -conflicts with the built-in variable name. Use a complex name or prefix the operator name." - ) - - def parse_mc2_ctx(self: any, conf: str): - self.mc2_ctx = self._parse_list(conf) - - @staticmethod - def _camel_to_snake(camel_case_str: str): - snake_case_str = "" - for i, c in enumerate(camel_case_str): - if i == 0: - snake_case_str += c.lower() - elif c.isupper(): - snake_case_str += "_" + c.lower() - else: - snake_case_str += c - return snake_case_str - - def parse_attr_val(self: any, conf: str): - for attr in self.attr_list: - if self.attr_val.get(attr) is None: - self.attr_val[attr] = {} - if conf.startswith("attr_{}.type".format(attr)): - self.attr_val.get(attr)["type"] = self._camel_to_snake( - self._parse_str(conf) - ) - elif conf.startswith("attr_{}.paramType".format(attr)): - self.attr_val.get(attr)["paramType"] = self._parse_str(conf) - elif conf.startswith("attr_{}.defaultValue".format(attr)): - self.attr_val.get(attr)["defaultValue"] = self._parse_str(conf) - - def parse_replay_val(self: any, batch_list: list, iterator_list: list): - if self.op_type in batch_list: - self.op_replay_flag = True - self.op_replay_batch = True - elif self.op_type in iterator_list: - self.op_replay_flag = True - self.op_replay_batch = False - - -def _is_op_type_in_opdesc(op_descs: list, op_type: str): - for op in op_descs: - if op_type == op.op_type: - return True - return False - - -def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): - for op in op_descs: - op.custom_all_compile_options = soc_ver_compile_options - - -def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): - for op in op_descs: - if op.op_type != op_type: - continue - op.custom_compile_options.update(soc_ver_compile_options) - - -def _trans_soc_ver_to_short(soc_ver: str): - low_soc_ver = soc_ver.lower() - if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: - print( - f"WARNING: caution: {soc_ver} will trans into ascend910, if not your intention," - f"use ascend910b1~4 instead" - ) - return SOC_TO_SHORT_SOC_MAP[low_soc_ver] - - -def _get_op_custom_options(op_descs: list, auto_gen_dir: str): - if auto_gen_dir is None: - return {} - file = os.path.join(auto_gen_dir, "custom_compile_options.ini") - if not os.path.exists(file): - print(f"WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini") - return {} - with open(file, "r") as fd: - lines = fd.readlines() - for line in lines: - param_list = str.split(line.rstrip("\n"), ",") - if len(param_list) != 3: - raise Exception( - f"ERROR: custom compile option {param_list} len is not 3" - ) - op_type = param_list[0] - if op_type.upper() == "ALL": - op_type = OP_ALL - if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: - continue - soc_ver_compile_options = {} - soc_ver = param_list[1] - options_str = param_list[2] - options = str.split(options_str, ";") - if soc_ver == "": - soc_ver_compile_options[SOC_ALL] = options - else: - soc_ver_list = str.split(soc_ver, ";") - for ver in soc_ver_list: - short_ver = _trans_soc_ver_to_short(ver) - soc_ver_compile_options[short_ver] = options - if op_type == OP_ALL: - _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) - else: - _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) - - -def get_op_desc( - file: str, - batch_list: list, - iterator_list: list, - builder: any, - op_type: list, - auto_gen_dir: str = None, -) -> list: - op_descs = [] - op_match = False - with open(file, "r") as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if line.startswith("["): - name = line[1:-1] - if op_type is None or name in op_type: - op_match = True - op_desc = builder(name) - op_desc.parse_replay_val(batch_list, iterator_list) - op_descs.append(op_desc) - else: - op_match = False - if op_type is not None and len(op_descs) == len(op_type): - break - continue - if not op_match: - continue - if line.startswith("input"): - op_desc.parse_input(line) - elif line.startswith("output"): - op_desc.parse_output(line) - elif line.startswith("dynamicFormat.flag"): - op_desc.parse_op_format(line) - elif line.startswith("needCheckSupport.flag"): - op_desc.parse_check_support(line) - elif line.startswith("rangeLimit.value"): - op_desc.parse_range_limit(line) - elif line.startswith("opInterface.value"): - op_desc.parse_op_intf(line) - elif line.startswith("kernel.name"): - op_desc.parse_kern_name(line) - elif line.startswith("opFile.value"): - op_desc.parse_op_file(line) - elif line.startswith("dynamicShapeSupport.flag"): - op_desc.parse_dynamic_shape(line) - elif line.startswith("mc2.ctx"): - op_desc.parse_mc2_ctx(line) - elif line.startswith("attr.list"): - op_desc.parse_attr_list(line) - elif line.startswith("attr_"): - op_desc.parse_attr_val(line) - _get_op_custom_options(op_descs, auto_gen_dir) - return op_descs diff --git a/csrc/deepep/ops/cmake/util/parse_ini_to_json.py b/csrc/deepep/ops/cmake/util/parse_ini_to_json.py deleted file mode 100755 index 928acae6f..000000000 --- a/csrc/deepep/ops/cmake/util/parse_ini_to_json.py +++ /dev/null @@ -1,449 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -""" -parser ini to json -""" - -import json -import os -import stat -import sys - -ATTR_TYPE_LIST = [ - "int", - "float", - "bool", - "str", - "listInt", - "listFloat", - "listBool", - "listStr", - "listListInt", - "type", - "listType", - "tensor", - "listTensor", -] -ATTR_PARAMTYPE_LIST = ["optional", "required"] -BOOL_FLAG_KEY = [ - "dynamicFormat", - "dynamicShapeSupport", - "dynamicRankSupport", - "precision_reduce", - "heavyOp", - "needCheckSupport", - "enableVectorCore", -] -BOOL_LIST = ["true", "false"] -DTYPE_LIST = [ - "float16", - "float", - "float32", - "int8", - "int16", - "int32", - "uint8", - "uint16", - "uint32", - "bool", - "int64", - "uint64", - "qint8", - "qint16", - "qint32", - "quint8", - "quint16", - "double", - "complex64", - "complex128", - "string", - "resource", - "dual", - "dual_sub_int8", - "dual_sub_uint8", - "string_ref", - "int4", - "bfloat16", - "uint1", -] -FORMAT_LIST = [ - "NCHW", - "NHWC", - "ND", - "NC1HWC0", - "FRACTAL_Z", - "NC1C0HWPAD", - "NHWC1C0", - "FSR_NCHW", - "FRACTAL_DECONV", - "C1HWNC0", - "FRACTAL_DECONV_TRANSPOSE", - "FRACTAL_DECONV_SP_STRIDE_TRANS", - "NC1HWC0_C04", - "FRACTAL_Z_C04", - "CHWN", - "FRACTAL_DECONV_SP_STRIDE8_TRANS", - "HWCN", - "NC1KHKWHWC0", - "BN_WEIGHT", - "FILTER_HWCK", - "HASHTABLE_LOOKUP_LOOKUPS", - "HASHTABLE_LOOKUP_KEYS", - "HASHTABLE_LOOKUP_VALUE", - "HASHTABLE_LOOKUP_OUTPUT", - "HASHTABLE_LOOKUP_HITS", - "C1HWNCoC0", - "MD", - "NDHWC", - "FRACTAL_ZZ", - "FRACTAL_NZ", - "NCDHW", - "DHWCN", - "NDC1HWC0", - "FRACTAL_Z_3D", - "CN", - "NC", - "DHWNC", - "FRACTAL_Z_3D_TRANSPOSE", - "FRACTAL_ZN_LSTM", - "FRACTAL_ZN_RNN", - "FRACTAL_Z_G", - "NULL", -] - - -def parse_ini_files(ini_files): - """ - parse ini files to json - Parameters: - ---------------- - ini_files:input file list - return:ops_info - ---------------- - """ - tbe_ops_info = {} - for ini_file in ini_files: - check_file_size(ini_file) - parse_ini_to_obj(ini_file, tbe_ops_info) - return tbe_ops_info - - -def check_file_size(input_file): - try: - file_size = os.path.getsize(input_file) - except OSError as os_error: - print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) - raise OSError from os_error - if file_size > 10 * 1024 * 1024: - print( - "[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait." - % input_file - ) - - -def parse_ini_to_obj(ini_file, tbe_ops_info): - """ - parse ini file to json obj - Parameters: - ---------------- - ini_file:ini file path - tbe_ops_info:ops_info - ---------------- - """ - with open(ini_file) as ini_file: - lines = ini_file.readlines() - op_dict = {} - op_name = "" - find_op_type = False - for line in lines: - line = line.rstrip() - if line == "": - continue - if line.startswith("["): - if line.endswith("]"): - op_name = line[1:-1] - op_dict = {} - tbe_ops_info[op_name] = op_dict - find_op_type = True - elif "=" in line: - key1 = line[: line.index("=")] - key2 = line[line.index("=") + 1 :] - key1_0, key1_1 = key1.split(".") - if key1_0 not in op_dict: - op_dict[key1_0] = {} - if key1_1 in op_dict.get(key1_0): - raise RuntimeError( - "Op:" + op_name + " " + key1_0 + " " + key1_1 + " is repeated!" - ) - dic_key = op_dict.get(key1_0) - dic_key[key1_1] = key2 - else: - continue - if not find_op_type: - raise RuntimeError("Not find OpType in .ini file.") - - -def check_output_exist(op_dict, is_valid): - """ - Function Description: - Check output is exist - Parameter: op_dict - Parameter: is_valid - """ - if "output0" in op_dict: - output0_dict = op_dict.get("output0") - if output0_dict.get("name", None) is None: - is_valid = False - print("output0.name is required in .ini file!") - else: - is_valid = False - print("output0 is required in .ini file!") - return is_valid - - -def check_attr_dict(attr_dict, is_valid, attr): - """ - Function Description: - Check attr_dict - Parameter: attr_dict - Parameter: is_valid - Parameter: attr - """ - attr_type = attr_dict.get("type") - value = attr_dict.get("value") - param_type = attr_dict.get("paramType") - if attr_type is None or value is None: - is_valid = False - print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) - if param_type and param_type not in ATTR_PARAMTYPE_LIST: - is_valid = False - print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) - if attr_type and attr_type not in ATTR_TYPE_LIST: - is_valid = False - print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) - return is_valid - - -def check_attr(op_dict, is_valid): - """ - Function Description: - Check attr - Parameter: op_dict - Parameter: is_valid - """ - if "attr" in op_dict: - attr_dict = op_dict.get("attr") - attr_list_str = attr_dict.get("list", None) - if attr_list_str is None: - is_valid = False - print("attr.list is required in .ini file!") - else: - attr_list = attr_list_str.split(",") - for attr_name in attr_list: - attr = "attr_" + attr_name.strip() - attr_dict = op_dict.get(attr) - if attr_dict: - is_valid = check_attr_dict(attr_dict, is_valid, attr) - else: - is_valid = False - print( - "%s is required in .ini file, when attr.list is %s!" - % (attr, attr_list_str) - ) - return is_valid - - -def check_bool_flag(op_dict, is_valid): - """ - Function Description: - check_bool_flag - Parameter: op_dict - Parameter: is_valid - """ - for key in BOOL_FLAG_KEY: - if key in op_dict: - op_bool_key = op_dict.get(key) - if op_bool_key.get("flag").strip() not in BOOL_LIST: - is_valid = False - print("{0}.flag only support {1}.".format(key, BOOL_LIST)) - return is_valid - - -def check_type_format(op_info, is_valid, op_info_key): - """ - Function Description: - Check type and format - Parameter: op_info - Parameter: is_valid - Parameter: op_info_key - """ - op_info_dtype_str = op_info.get("dtype") - op_info_dtype_num = 0 - op_info_format_num = 0 - if op_info_dtype_str: - op_info_dtype = op_info_dtype_str.split(",") - op_info_dtype_num = len(op_info_dtype) - for dtype in op_info_dtype: - if dtype.strip() not in DTYPE_LIST: - is_valid = False - print("{0}.dtype not support {1}.".format(op_info_key, dtype)) - op_info_format_str = op_info.get("format") - if op_info_format_str: - op_info_format = op_info_format_str.split(",") - op_info_format_num = len(op_info_format) - for op_format in op_info_format: - if op_format.strip() not in FORMAT_LIST: - is_valid = False - print("{0}.format not support {1}.".format(op_info_key, op_format)) - if op_info_dtype_num > 0 and op_info_format_num > 0: - if op_info_dtype_num != op_info_format_num: - is_valid = False - print( - "The number of {0}.dtype not match the number of {0}.format.".format( - op_info_key - ) - ) - return is_valid - - -def check_op_info(tbe_ops): - """ - Function Description: - Check info. - Parameter: tbe_ops - Return Value: is_valid - """ - print("\n\n==============check valid for ops info start==============") - required_op_input_info_keys = ["paramType", "name"] - required_op_output_info_keys = ["paramType", "name"] - param_type_valid_value = ["dynamic", "optional", "required"] - is_valid = True - for op_key in tbe_ops: - op_dict = tbe_ops[op_key] - for op_info_key in op_dict: - if op_info_key.startswith("input"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_input_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print( - "op: " - + op_key - + " " - + op_info_key - + " missing: " - + ",".join(missing_keys) - ) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print( - "op: " - + op_key - + " " - + op_info_key - + " paramType not valid, valid key:[dynamic, " - "optional, required]" - ) - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - if op_info_key.startswith("output"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_output_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print( - "op: " - + op_key - + " " - + op_info_key - + " missing: " - + ",".join(missing_keys) - ) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print( - "op: " - + op_key - + " " - + op_info_key - + " paramType not valid, valid key:[dynamic, " - "optional, required]" - ) - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - is_valid = check_attr(op_dict, is_valid) - is_valid = check_bool_flag(op_dict, is_valid) - print("==============check valid for ops info end================\n\n") - return is_valid - - -def write_json_file(tbe_ops_info, json_file_path): - """ - Save info to json file - Parameters: - ---------------- - tbe_ops_info: ops_info - json_file_path: json file path - ---------------- - """ - json_file_real_path = os.path.realpath(json_file_path) - wr_flag = os.O_WRONLY | os.O_CREAT - wr_mode = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), "w") as file_path: - # The owner have all rights£¬group only have read rights - os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + stat.S_IRUSR) - json.dump( - tbe_ops_info, file_path, sort_keys=True, indent=4, separators=(",", ":") - ) - print("Compile op info cfg successfully.") - - -def parse_ini_to_json(ini_file_paths, outfile_path): - """ - parse ini files to json file - Parameters: - ---------------- - ini_file_paths: list of ini file path - outfile_path: output file path - ---------------- - """ - tbe_ops_info = parse_ini_files(ini_file_paths) - if not check_op_info(tbe_ops_info): - print("Compile op info cfg failed.") - return False - write_json_file(tbe_ops_info, outfile_path) - return True - - -if __name__ == "__main__": - args = sys.argv - - OUTPUT_FILE_PATH = "tbe_ops_info.json" - ini_file_path_list = [] - parse_ini_list = [] - - for arg in args: - if arg.endswith("ini"): - ini_file_path_list.append(arg) - OUTPUT_FILE_PATH = arg.replace(".ini", ".json") - if arg.endswith("json"): - OUTPUT_FILE_PATH = arg - - if not ini_file_path_list: - ini_file_path_list.append("tbe_ops_info.ini") - - for ini_file in ini_file_path_list: - if os.path.exists(ini_file): - parse_ini_list.append(ini_file) - - if parse_ini_list: - if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): - sys.exit(1) - sys.exit(0) diff --git a/csrc/deepep/ops/cmake/util/preset_parse.py b/csrc/deepep/ops/cmake/util/preset_parse.py deleted file mode 100755 index 983f16e74..000000000 --- a/csrc/deepep/ops/cmake/util/preset_parse.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import json -import os -import sys - - -def read_json(file): - with open(file, "r") as fd: - config = json.load(fd) - return config - - -def get_config_opts(file): - config = read_json(file) - - src_dir = os.path.abspath(os.path.dirname(file)) - opts = "" - - for conf in config: - if conf == "configurePresets": - for node in config[conf]: - macros = node.get("cacheVariables") - if macros is not None: - for key in macros: - opts += "-D{}={} ".format(key, macros[key]["value"]) - - opts = opts.replace("${sourceDir}", src_dir) - print(opts) - - -if __name__ == "__main__": - get_config_opts(sys.argv[1]) diff --git a/csrc/deepep/ops/cmake/util/replay_codegen.py b/csrc/deepep/ops/cmake/util/replay_codegen.py deleted file mode 100755 index 6f896a098..000000000 --- a/csrc/deepep/ops/cmake/util/replay_codegen.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import collections -import os -import stat - -import code_channel_infer -import const_var -import kernel_entry as keb -from tiling_data_def_build import gen_tiling - -PYF_PATH = os.path.dirname(__file__) - -ReplayCodeGenParams = collections.namedtuple( - "ReplayCodeGenParams", - [ - "op_type", - "impl", - "tiling_file", - "kernel", - "entry", - "argn", - "op_replay_batch", - "max_block_dim", - "max_shape_size", - ], -) - - -class ReplayCodeGen: - def __init__(self, replayCodeGenParams): - self.op_type = replayCodeGenParams.op_type - self.impl = replayCodeGenParams.impl - self.tiling_file = replayCodeGenParams.tiling_file - self.tiling_data_file = "" - self.kernel = replayCodeGenParams.kernel - self.entry = replayCodeGenParams.entry - self.argn = replayCodeGenParams.argn - self.batch = False - self.outdir = "" - self.data_type = "uint8_t" - self.blknum = 32 - self.op_replay_batch = replayCodeGenParams.op_replay_batch - self.max_block_dim = replayCodeGenParams.max_block_dim - self.max_shape_size = replayCodeGenParams.max_shape_size - - def set_batch(self, is_batch): - self.batch = is_batch - - def set_outdir(self, outdir): - self.outdir = outdir - - def gen_replay(self, ops_product: str): - kerentry = os.path.join(self.outdir, self.kernel + "_entry.cce") - kerimpl = os.path.join(self.outdir, self.kernel + "_impl.cpp") - replayimpl = os.path.join(self.outdir, self.kernel + "_replay.cpp") - if self.batch: - reptmp = os.path.join(PYF_PATH, "batch_replay_impl.temp") - else: - reptmp = os.path.join(PYF_PATH, "replay_impl.temp") - kertmp = os.path.join(PYF_PATH, "kernel_impl.temp") - self._gen_kentry(kerentry) - self._gen_kimpl_code(kerimpl, kertmp) - self._gen_tiling_data_header() - self._gen_replay_code(replayimpl, reptmp, ops_product) - - def _gen_tiling_data_header(self): - self.tiling_data_file = os.path.join( - self.outdir, self.kernel + "_tiling_data.h" - ) - gen_tiling(self.tiling_file, self.tiling_data_file) - - def _gen_kimpl_code(self, src, tmpfile): - with open(tmpfile, "r") as fd: - temp = fd.read() - temp = temp.replace("__CCE_FILE__", self.impl) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), "w") as ofd: - ofd.write(temp) - - def _gen_replay_code(self, src, tmpfile, ops_product: str): - with open(tmpfile, "r") as fd: - temp = fd.read() - temp = temp.replace("__ARG_NUM__", str(self.argn)) - argdef = [] - kargs = [] - for i in range(0, self.argn): - argdef.append("{} *".format(self.data_type)) - kargs.append("({} *)GetArg({})".format(self.data_type, i)) - temp = temp.replace("__ARGS_DEF__", ", ".join(argdef)) - temp = temp.replace("__KERNEL_ARGS__", ", ".join(kargs)) - temp = temp.replace("__KERNEL_FUN__", self.entry) - core_type_infer = "core_type" - code_channel = code_channel_infer.infer_code_channel( - code_channel_infer.InfoCodeChanelParams( - self.impl, - self.tiling_data_file, - self.kernel, - self.outdir, - ops_product, - None, - ) - ) - if code_channel == code_channel_infer.CODE_VEC: - core_type_infer = "0" - elif code_channel == code_channel_infer.CODE_CUBE: - core_type_infer = "1" - temp = temp.replace("__CORE_TYPE__", core_type_infer) - # register function - temp = temp.replace("__OPS_PRODUCT__", ops_product) - temp = temp.replace("__OPTYPE__", self.op_type) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), "w") as ofd: - ofd.write(temp) - - def _gen_kentry(self, src): - kf = "" - pre_alloc_str = "A" * 256 - if self.batch: - kf += keb.batch_code_gen( - "K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), - self.argn, - self.data_type, - ) - else: - kf += keb.mc_code_gen( - "K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), - self.argn, - self.data_type, - self.blknum, - ) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), "w") as ofd: - ofd.write(kf) diff --git a/csrc/deepep/ops/cmake/util/replay_impl.temp b/csrc/deepep/ops/cmake/util/replay_impl.temp deleted file mode 100644 index 1d30dd865..000000000 --- a/csrc/deepep/ops/cmake/util/replay_impl.temp +++ /dev/null @@ -1,120 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], - int alen[], int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N * 32]; - int len[KERNEL_N * 32]; - int blknum[KERNEL_N]; - int max; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); - for (block_idx = 0; block_idx < block_num; block_idx++) { - //__OP_SET_KERNEL__ - int code_idx = i * block_num + block_idx; -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, false); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[code_idx] = (char *)pos; - len[code_idx] = CodeLen(); - pos += len[code_idx]; - printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); - } - blknum[i] = block_num; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/csrc/deepep/ops/cmake/util/tiling_data_def_build.py b/csrc/deepep/ops/cmake/util/tiling_data_def_build.py deleted file mode 100755 index 0576b202d..000000000 --- a/csrc/deepep/ops/cmake/util/tiling_data_def_build.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Function: -The replay function entry -""" - -import os -import re -import stat -import sys - -import const_var - - -def gen_tiling(tiling_header_file: str, tiling_file_out: str): - if not os.path.exists(tiling_header_file): - print("warning: no userdef tiling header file: ", tiling_header_file) - return - print("generate tiling def header file: ", tiling_file_out) - tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() - tiling_source = "#ifndef __{}_H__\n".format(tmp_name) - tiling_source += "#define __{}_H__\n\n".format(tmp_name) - tiling_source += "#include \n" - tiling_source += "#include \n\n" - tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' - end_source = "" - pattern = re.compile(r"[(](.*)[)]", re.S) - with open(tiling_header_file, "r") as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if line.startswith("BEGIN_TILING_DATA_DEF"): - tiling_source += "#pragma pack(1)\n" - tiling_source += "struct " - struct_def = re.findall(pattern, line)[0] - tiling_source += struct_def + " {\n" - elif line.startswith("TILING_DATA_FIELD_DEF_ARR"): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(",") - tiling_source += " {} {}[{}] = {{}};\n".format( - fds[0].strip(), fds[2].strip(), fds[1].strip() - ) - elif line.startswith("TILING_DATA_FIELD_DEF_STRUCT"): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(",") - tiling_source += " {} {};\n".format(fds[0].strip(), fds[1].strip()) - elif line.startswith("TILING_DATA_FIELD_DEF"): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(",") - tiling_source += " {} {} = 0;\n".format( - fds[0].strip(), fds[1].strip() - ) - elif line.startswith("END_TILING_DATA_DEF"): - tiling_source += "};\n" - tiling_source += "#pragma pack()\n\n" - tiling_source += "#ifdef __NPU_TILING__\n" - tiling_source += "inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n".format( - stru=struct_def - ) - tiling_source += "{\n" - tiling_source += " const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n" - tiling_source += " uint32_t *dst = (uint32_t *)const_data;\n" - tiling_source += " for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n".format( - struct_def - ) - tiling_source += "}\n" - tiling_source += "#else\n" - tiling_source += "inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n".format( - stru=struct_def - ) - tiling_source += "{\n" - tiling_source += " uint64_t *src = (uint64_t *)tiling;\n" - tiling_source += " uint64_t *dst = (uint64_t *)const_data;\n" - tiling_source += " for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n".format( - struct_def - ) - tiling_source += "}\n" - tiling_source += "#endif\n\n" - end_source = """ -#undef GET_TILING_DATA -#define GET_TILING_DATA(tiling_data, tiling_arg) \\ -{stru} tiling_data; \\ -Init{stru}(tiling_arg, &tiling_data)\n -""".format( - stru=struct_def - ) - tiling_source += end_source - tiling_source += "#endif" - with os.fdopen( - os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), "w" - ) as ofd: - ofd.write(tiling_source) - - -if __name__ == "__main__": - if len(sys.argv) <= 2: - raise RuntimeError("arguments must greater than 2") - gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/csrc/deepep/ops/cmake/func.cmake b/csrc/deepep/ops/func.cmake similarity index 100% rename from csrc/deepep/ops/cmake/func.cmake rename to csrc/deepep/ops/func.cmake diff --git a/csrc/deepep/ops2/CMakeLists.txt b/csrc/deepep/ops2/CMakeLists.txt index a27c0b46c..4bf4630cf 100644 --- a/csrc/deepep/ops2/CMakeLists.txt +++ b/csrc/deepep/ops2/CMakeLists.txt @@ -9,10 +9,34 @@ else() message(FATAL_ERROR "Unsupported host processor: ${CMAKE_SYSTEM_PROCESSOR}") endif() +set(CANN82_PATH "${ASCEND_CANN_PACKAGE_PATH}/include/experiment/platform/platform/platform_infos_def.h") +set(CANN83_PATH "${ASCEND_CANN_PACKAGE_PATH}/include/platform/platform_infos_def.h") + +if (EXISTS "${CANN83_PATH}") + set(CANN_VERSION_MACRO "USE_CANN83_PATH") + message(STATUS "Detected CANN 8.3+ include structure, using ${CANN83_PATH}") +elseif (EXISTS "${CANN82_PATH}") + set(CANN_VERSION_MACRO "USE_CANN82_PATH") + message(STATUS "Detected CANN 8.2 include structure, using ${CANN82_PATH}") +else() + message(FATAL_ERROR + "Cannot find 'platform_infos_def.h' in expected locations.\n" + "Please verify that CANN is correctly installed and that either:\n" + " ${CANN83_PATH}\n" + " or\n" + " ${CANN82_PATH}\n" + "exists and contains the header file." + ) +endif() + include(cmake/config.cmake) -include(cmake/func.cmake) +include(func.cmake) include(cmake/intf.cmake) +if (CANN_VERSION_MACRO) + add_compile_definitions(${CANN_VERSION_MACRO}) +endif() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) @@ -36,6 +60,35 @@ add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_ ) add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) +if (DEFINED CANN_VERSION_MACRO AND NOT "${CANN_VERSION_MACRO}" STREQUAL "") + set(CANN_VERSION_FLAG "-D${CANN_VERSION_MACRO}") + message(STATUS "opbuild: Detected CANN_VERSION_MACRO = ${CANN_VERSION_MACRO}") +else() + set(CANN_VERSION_FLAG "") + message(WARNING "opbuild: No CANN_VERSION_MACRO defined! Possible #error in .cc files.") +endif() + +execute_process( + COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 + ${OPBUILD_OPS_SRC} + -D_GLIBCXX_USE_CXX11_ABI=0 + ${CANN_VERSION_FLAG} + -I ${ASCEND_CANN_PACKAGE_PATH}/include + -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel + -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 + -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR +) +if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") +endif() + get_system_info(SYSTEM_INFO) # gen version.info diff --git a/csrc/deepep/ops2/cmake/config.cmake b/csrc/deepep/ops2/cmake/config.cmake deleted file mode 100644 index d990cd312..000000000 --- a/csrc/deepep/ops2/cmake/config.cmake +++ /dev/null @@ -1,42 +0,0 @@ - -set(CMAKE_CXX_FLAGS_DEBUG "") -set(CMAKE_CXX_FLAGS_RELEASE "") - -if (NOT DEFINED CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release CACHE STRING "") -endif() -if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/build_out" CACHE PATH "" FORCE) -endif() -if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) - set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") -endif() -if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) - set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") -endif() -if (NOT DEFINED ASCEND_COMPUTE_UNIT) - set(ASCEND_COMPUTE_UNIT ascend910_93 CACHE STRING "") -endif() -if (NOT DEFINED ENABLE_TEST) - set(ENABLE_TEST FALSE CACHE BOOL "") -endif() -if (NOT DEFINED ENABLE_CROSS_COMPILE) - set(ENABLE_CROSS_COMPILE FALSE CACHE BOOL "") -endif() -if (NOT DEFINED CMAKE_CROSS_PLATFORM_COMPILER) - set(CMAKE_CROSS_PLATFORM_COMPILER "/your/cross/compiler/path" CACHE PATH "") -endif() -if (NOT DEFINED CMAKE_CROSS_LIBRARY_PATH) - set(CMAKE_CROSS_LIBRARY_PATH "" CACHE PATH "") -endif() -if (NOT DEFINED ASCEND_PACK_SHARED_LIBRARY) - set(ASCEND_PACK_SHARED_LIBRARY False CACHE BOOL "") -endif() -set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) -set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) -set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) -file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) -set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") -set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") -file(WRITE ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} "") -file(WRITE ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} "") diff --git a/csrc/deepep/ops2/cmake/device_task.cmake b/csrc/deepep/ops2/cmake/device_task.cmake deleted file mode 100644 index 3b7c0a136..000000000 --- a/csrc/deepep/ops2/cmake/device_task.cmake +++ /dev/null @@ -1,48 +0,0 @@ -message(STATUS "TILING SINK TASK BEGIN") -message(STATUS "TARGET: ${TARGET}") -message(STATUS "OPTION: ${OPTION}") -message(STATUS "SRC: ${SRC}") -message(STATUS "VENDOR: ${VENDOR_NAME}") - -set(CMAKE_CXX_COMPILER ${ASCEND_CANN_PACKAGE_PATH}/toolkit/toolchain/hcc/bin/aarch64-target-linux-gnu-g++) -set(CMAKE_C_COMPILER ${ASCEND_CANN_PACKAGE_PATH}/toolkit/toolchain/hcc/bin/aarch64-target-linux-gnu-gcc) - -string(REPLACE " " ";" SRC "${SRC}") -add_library(${TARGET} ${OPTION} - ${SRC} -) -target_compile_definitions(${TARGET} PRIVATE - DEVICE_OP_TILING_LIB - _FORTIFY_SOURCE=2 - google=ascend_private -) -target_include_directories(${TARGET} PRIVATE - ${ASCEND_CANN_PACKAGE_PATH}/include -) -target_compile_options(${TARGET} PRIVATE - -fPIC - -fstack-protector-strong - -fstack-protector-all - -O2 - -std=c++11 - -fvisibility-inlines-hidden - -fvisibility=hidden -) -target_link_libraries(${TARGET} PRIVATE - -Wl,--whole-archive - device_register - c_sec - mmpa - tiling_api - platform_static - ascend_protobuf - exe_meta_device - -Wl,--no-whole-archive -) -target_link_directories(${TARGET} PRIVATE - ${ASCEND_CANN_PACKAGE_PATH}/lib64/device/lib64 - ${ASCEND_CANN_PACKAGE_PATH}/compiler/lib64 -) -set_target_properties(${TARGET} PROPERTIES - OUTPUT_NAME cust_opmaster -) diff --git a/csrc/deepep/ops2/cmake/intf.cmake b/csrc/deepep/ops2/cmake/intf.cmake deleted file mode 100644 index d2643bbc7..000000000 --- a/csrc/deepep/ops2/cmake/intf.cmake +++ /dev/null @@ -1,28 +0,0 @@ - -add_library(intf_pub INTERFACE) -target_compile_options(intf_pub INTERFACE - -fPIC - -fvisibility=hidden - -fvisibility-inlines-hidden - $<$:-O2> - $<$:-O0 -g> - $<$:-std=c++11> - $<$,$>:-ftrapv -fstack-check> - $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> - $,-fstack-protector-strong,-fstack-protector-all> -) -target_compile_definitions(intf_pub INTERFACE - _GLIBCXX_USE_CXX11_ABI=0 - $<$:_FORTIFY_SOURCE=2> -) -target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include - ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel -) -target_link_options(intf_pub INTERFACE - $<$,EXECUTABLE>:-pie> - $<$:-s> - -Wl,-z,relro - -Wl,-z,now - -Wl,-z,noexecstack -) -target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/csrc/deepep/ops2/cmake/makeself.cmake b/csrc/deepep/ops2/cmake/makeself.cmake deleted file mode 100644 index 82c8da7eb..000000000 --- a/csrc/deepep/ops2/cmake/makeself.cmake +++ /dev/null @@ -1,31 +0,0 @@ -execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh - --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh - --help-header ./help.info - --gzip --complevel 4 --nomd5 --sha256 - ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - RESULT_VARIABLE EXEC_RESULT - ERROR_VARIABLE EXEC_ERROR -) - -if (NOT "${EXEC_RESULT}x" STREQUAL "0x") - message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") -endif() - -execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ - COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - ) - -if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") - execute_process( - COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - ) - - execute_process( - COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ - COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - ) -endif() diff --git a/csrc/deepep/ops2/cmake/util/__init__.py b/csrc/deepep/ops2/cmake/util/__init__.py deleted file mode 100755 index 364083fab..000000000 --- a/csrc/deepep/ops2/cmake/util/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(PYF_PATH) diff --git a/csrc/deepep/ops2/cmake/util/ascendc_bin_param_build.py b/csrc/deepep/ops2/cmake/util/ascendc_bin_param_build.py deleted file mode 100755 index 388575cde..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_bin_param_build.py +++ /dev/null @@ -1,531 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import copy -import hashlib -import json -import os -import re -import sys -from collections import defaultdict -from typing import Dict, List, NamedTuple, Set, Tuple - -import const_var -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class ParamInfo(NamedTuple): - dtype_list: list - format_list: list - dtype_for_bin_list: dict - format_for_bin_list: dict - - -class BinParamBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - self.soc = "" - self.out_path = "" - self.tiling_keys = set() - self.op_debug_config = "" - - def set_soc_version(self: any, soc: str): - self.soc = soc - - def set_out_path(self: any, out_path: str): - self.out_path = out_path - - def set_tiling_key(self: any, tiling_key_info: Set): - if tiling_key_info: - self.tiling_keys.update(tiling_key_info) - - def set_op_debug_config(self: any, op_debug_config: str): - if op_debug_config: - self.op_debug_config = op_debug_config - - def get_full_list(self: any): - dtype_list = [] - for dtype_in in self.input_dtype: - dtype_list.append(dtype_in.split(",")) - for dtype_out in self.output_dtype: - dtype_list.append(dtype_out.split(",")) - - format_list = [] - for fmt_in in self.input_fmt: - format_list.append(fmt_in.split(",")) - for fmt_out in self.output_fmt: - format_list.append(fmt_out.split(",")) - - dtype_for_bin_list = [ - [] for _ in range(len(self.input_dtype) + len(self.output_dtype)) - ] - format_for_bin_list = copy.deepcopy(dtype_for_bin_list) - - for key, value in self.input_dtype_for_bin.items(): - dtype_for_bin_list[key] = value.split(",") - for key, value in self.output_dtype_for_bin.items(): - dtype_for_bin_list[key + len(self.input_dtype)] = value.split(",") - for key, value in self.input_fmt_for_bin.items(): - format_for_bin_list[key] = value.split(",") - for key, value in self.output_fmt_for_bin.items(): - format_for_bin_list[key + len(self.input_dtype)] = value.split(",") - - return ParamInfo( - dtype_list, format_list, dtype_for_bin_list, format_for_bin_list - ) - - def gen_bin_cprs_list(self: any, param_info: ParamInfo): - combine_dict = {} - origin_combine_dict = {} - for cob_idx in range(0, len(self.input_dtype[0].split(","))): - origin_combine = "" - combine = "" - for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): - if param_info.dtype_for_bin_list[param_idx]: - combine += param_info.dtype_for_bin_list[param_idx][cob_idx] - else: - combine += param_info.dtype_list[param_idx][cob_idx] - origin_combine += param_info.dtype_list[param_idx][cob_idx] - if param_info.format_for_bin_list[param_idx]: - combine += param_info.format_for_bin_list[param_idx][cob_idx] - else: - combine += param_info.format_list[param_idx][cob_idx] - origin_combine += param_info.format_list[param_idx][cob_idx] - if combine not in combine_dict: - combine_dict[combine] = [] - combine_dict[combine].append(cob_idx) - origin_combine_dict[origin_combine] = cob_idx - for key, value in combine_dict.items(): - if key not in origin_combine_dict: - print(f"WARNING: ForBinQuery {key} not in origin combine") - self.bin_save_list += value - continue - if len(value) == 1 and value[0] == origin_combine_dict[key]: - self.bin_save_list += value - continue - self.bin_cprs_head.append(origin_combine_dict[key]) - self.bin_cprs_list.append(value) - for index, sub_list in enumerate(self.bin_cprs_list): - if self.bin_cprs_head[index] not in self.bin_save_list: - continue - sub_list.append(self.bin_cprs_head[index]) - self.bin_save_list += self.bin_cprs_head - - def gen_for_bin_list(self: any, param_info: ParamInfo): - combine_size = len(self.input_dtype[0].split(",")) - input_size = len(self.input_dtype) - output_size = len(self.output_dtype) - - self.input_dtype_for_bin_list = [[] for _ in range(input_size)] - self.output_dtype_for_bin_list = [[] for _ in range(output_size)] - for i in range(0, input_size): - self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] - for i in range(0, output_size): - self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] - self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) - self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) - - for index, sub_list in enumerate(self.bin_cprs_list): - head_idx = self.bin_cprs_head[index] - for cmb_idx in sub_list: - for i in range(0, input_size): - self.input_dtype_for_bin_list[i][head_idx].append( - param_info.dtype_list[i][cmb_idx] - ) - self.input_fmt_for_bin_list[i][head_idx].append( - param_info.format_list[i][cmb_idx] - ) - for i in range(0, output_size): - self.output_dtype_for_bin_list[i][head_idx].append( - param_info.dtype_list[i + input_size][cmb_idx] - ) - self.output_fmt_for_bin_list[i][head_idx].append( - param_info.format_list[i + input_size][cmb_idx] - ) - - def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): - for i in range(0, input_size): - self.input_dtype_for_bin_list[i] = [ - element - for index, element in enumerate(self.input_dtype_for_bin_list[i]) - if index in self.bin_save_list - ] - self.input_fmt_for_bin_list[i] = [ - element - for index, element in enumerate(self.input_fmt_for_bin_list[i]) - if index in self.bin_save_list - ] - new_dtype_list = [ - element - for index, element in enumerate(dtype_list[i]) - if index in self.bin_save_list - ] - new_dtype_str = "" - for dtype in new_dtype_list: - new_dtype_str += f"{dtype}," - self.input_dtype[i] = new_dtype_str[:-1] - new_format_list = [ - element - for index, element in enumerate(format_list[i]) - if index in self.bin_save_list - ] - new_format_str = "" - for fmt in new_format_list: - new_format_str += f"{fmt}," - self.input_fmt[i] = new_format_str[:-1] - for i in range(0, output_size): - self.output_dtype_for_bin_list[i] = [ - element - for index, element in enumerate(self.output_dtype_for_bin_list[i]) - if index in self.bin_save_list - ] - self.output_fmt_for_bin_list[i] = [ - element - for index, element in enumerate(self.output_fmt_for_bin_list[i]) - if index in self.bin_save_list - ] - new_dtype_list = [ - element - for index, element in enumerate(dtype_list[i + input_size]) - if index in self.bin_save_list - ] - new_dtype_str = "" - for dtype in new_dtype_list: - new_dtype_str += f"{dtype}," - self.output_dtype[i] = new_dtype_str[:-1] - new_format_list = [ - element - for index, element in enumerate(format_list[i + input_size]) - if index in self.bin_save_list - ] - new_format_str = "" - for fmt in new_format_list: - new_format_str += f"{fmt}," - self.output_fmt[i] = new_format_str[:-1] - - def is_set_for_bin_query(self: any): - return any( - [ - self.input_dtype_for_bin, - self.output_dtype_for_bin, - self.input_fmt_for_bin, - self.output_fmt_for_bin, - ] - ) - - def for_bin_list_match(self: any): - if not self.is_set_for_bin_query(): - return - input_size = len(self.input_dtype) - output_size = len(self.output_dtype) - param_info = self.get_full_list() - self.gen_bin_cprs_list(param_info) - self.gen_for_bin_list(param_info) - if len(self.bin_save_list) == len(self.input_dtype[0].split(",")): - print( - f"WARNING: ForBinQuery can not compress number of bin file with this set, please check!!." - ) - return - self.rm_cprs_cmb( - param_info.dtype_list, param_info.format_list, input_size, output_size - ) - - def gen_input_json(self: any, auto_gen_path: str): - key_map = {} - self.for_bin_list_match() - count = len(self.input_dtype[0].split(",")) - required_parameters = set() - index_value = -1 - - for i in range(0, count): - inputs = [] - outputs = [] - attrs = [] - required_parameter = [] - op_node = {} - - for idx in range(0, len(self.input_name)): - idtypes = self.input_dtype[idx].split(",") - ifmts = self.input_fmt[idx].split(",") - itype = self.input_type[idx] - para = {} - para["name"] = self.input_name[idx][:-5] - para["index"] = idx - para["dtype"] = idtypes[i] - if ( - self.is_set_for_bin_query() - and self.input_dtype_for_bin_list[idx][i] - ): - para["dtypeForBinQuery"] = self.input_dtype_for_bin_list[idx][i] - para["format"] = ifmts[i] - if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: - para["formatForBinQuery"] = self.input_fmt_for_bin_list[idx][i] - para["paramType"] = itype - para["shape"] = [-2] - para["format_match_mode"] = "FormatAgnostic" - - input_parameter_key = (idtypes[i], ifmts[i]) - if itype == "dynamic": - inputs.append([para]) - required_parameter.append(input_parameter_key) - elif itype == "required": - inputs.append(para) - required_parameter.append(input_parameter_key) - else: - inputs.append(para) - - for idx in range(0, len(self.output_name)): - odtypes = self.output_dtype[idx].split(",") - ofmts = self.output_fmt[idx].split(",") - otype = self.output_type[idx] - para = {} - para["name"] = self.output_name[idx][:-5] - para["index"] = idx - para["dtype"] = odtypes[i] - if ( - self.is_set_for_bin_query() - and self.output_dtype_for_bin_list[idx][i] - ): - para["dtypeForBinQuery"] = self.output_dtype_for_bin_list[idx][i] - para["format"] = ofmts[i] - if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: - para["formatForBinQuery"] = self.output_fmt_for_bin_list[idx][i] - para["paramType"] = otype - para["shape"] = [-2] - para["format_match_mode"] = "FormatAgnostic" - output_parameter_key = (odtypes[i], ofmts[i]) - if otype == "dynamic": - outputs.append([para]) - required_parameter.append(output_parameter_key) - elif otype == "required": - outputs.append(para) - required_parameter.append(output_parameter_key) - else: - outputs.append(para) - - for attr in self.attr_list: - att = {} - att["name"] = attr - atype = self.attr_val.get(attr).get("type").lower() - att["dtype"] = atype - att["value"] = const_var.ATTR_DEF_VAL.get(atype) - attrs.append(att) - - required_parameter_tuple = tuple(required_parameter) - if required_parameter_tuple in required_parameters: - continue - else: - required_parameters.add(required_parameter_tuple) - index_value += 1 - - op_node["bin_filename"] = "" - op_node["inputs"] = inputs - op_node["outputs"] = outputs - if len(attrs) > 0: - op_node["attrs"] = attrs - - param = {} - param["op_type"] = self.op_type - param["op_list"] = [op_node] - objstr = json.dumps(param, indent=" ") - md5sum = hashlib.md5(objstr.encode("utf-8")).hexdigest() - while key_map.get(md5sum) is not None: - objstr += "1" - md5sum = hashlib.md5(objstr.encode("utf-8")).hexdigest() - key_map[md5sum] = md5sum - bin_file = self.op_type + "_" + md5sum - op_node["bin_filename"] = bin_file - param_file = os.path.join(self.out_path, bin_file + "_param.json") - param_file = os.path.realpath(param_file) - with os.fdopen( - os.open(param_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - json.dump(param, fd, indent=" ") - self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) - - def _write_build_cmd( - self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str - ): - hard_soc = const_var.conv_soc_ver(self.soc) - if not hard_soc: - hard_soc = self.soc.capitalize() - name_com = [self.op_type, self.op_file, str(index)] - compile_file = os.path.join(self.out_path, "-".join(name_com) + ".sh") - compile_file = os.path.realpath(compile_file) - - bin_cmd_str = "res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ - --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic " - - build_cmd_var = "#!/bin/bash\n" - build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' - plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") - plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") - if plog_level is None: - build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR - if plog_stdout is None: - build_cmd_var += const_var.SET_PLOG_STDOUT - build_cmd_var += const_var.SRC_ENV - if hard_soc == "Ascend610Lite": - build_cmd_var += f"export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n" - build_cmd_var += bin_cmd_str.format( - fun=self.op_intf, - soc=hard_soc, - param=param_file, - impl="high_performance,optional", - ) - enable_tiling_keys = False - if self.tiling_keys: - tiling_keys_list = sorted(list(self.tiling_keys)) - tiling_key_str = ",".join([str(_key) for _key in tiling_keys_list]) - build_cmd_var += f' --tiling_key="{tiling_key_str}"' - enable_tiling_keys = True - - if self.op_debug_config: - op_debug_str = ",".join([str(_key) for _key in list(self.op_debug_config)]) - build_cmd_var += f" --op_debug_config={op_debug_str}" - - build_cmd_var += ")\n" - build_cmd_var += "\n" - if enable_tiling_keys is False: - build_cmd_var += 'echo "${res}"\n' - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".json") - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".o") - else: - build_cmd_var += "if [ $? -eq 1 ]; then\n" - build_cmd_var += ' if echo "${res}" | \ -grep -q "None of the given tiling keys are in the supported list"; then\n' - build_cmd_var += ' echo "${res}"\n' - build_cmd_var += " else\n" - build_cmd_var += ' echo "${res}"\n' - build_cmd_var += " exit 1\n" - build_cmd_var += " fi\n" - build_cmd_var += "else\n" - build_cmd_var += 'echo "${res}"\n' - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".json") - build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + ".o") - build_cmd_var += "fi\n" - build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' - - with os.fdopen( - os.open(compile_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - fd.write(build_cmd_var) - - -def get_tiling_keys(tiling_keys: str) -> Set: - all_tiling_keys = set() - if not tiling_keys: - return all_tiling_keys - - tiling_key_list = tiling_keys.split(";") - for tiling_key_value in tiling_key_list: - pattern = r"(? int(end): - continue - for i in range(int(start), int(end) + 1): - all_tiling_keys.add(i) - elif tiling_key_value.isdigit(): - all_tiling_keys.add(int(tiling_key_value)) - return all_tiling_keys - - -def trans_soc_verion(soc_ver: str): - low_soc_ver = soc_ver.lower() - if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: - return low_soc_ver - return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] - - -def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: - tiling_key_info = defaultdict(set) - op_debug_config = defaultdict(set) - if not opc_config_file: - return tiling_key_info, op_debug_config - - if not os.path.exists(opc_config_file): - return tiling_key_info, op_debug_config - - with open(opc_config_file, "r") as file: - contents = file.readlines() - - for _content in contents: - content = _content.strip() - opc_configs = content.split("@") - if len(opc_configs) < 3: - continue - - op_type = opc_configs[0] - if not op_type: - continue - - compute_unit = opc_configs[1] - if compute_unit: - compute_unit_list = compute_unit.split(";") - soc_lists = [] - for soc_ver in compute_unit_list: - short_soc_ver = trans_soc_verion(soc_ver) - soc_lists.append(short_soc_ver) - if soc not in soc_lists: - continue - - for options in opc_configs[2:]: - if "--tiling_key" in options: - format_tiling_keys = get_tiling_keys(options.split("=")[1]) - if format_tiling_keys: - tiling_key_info[op_type].update(format_tiling_keys) - if "--op_debug_config" in options: - format_debug_config = set(options.split("=")[1].split(";")) - if format_debug_config: - op_debug_config[op_type].update(format_debug_config) - - return tiling_key_info, op_debug_config - - -def gen_bin_param_file( - cfgfile: str, out_dir: str, soc: str, opc_config_file: str = "", ops: list = None -): - if not os.path.exists(cfgfile): - print( - f"INFO: {cfgfile} does not exists in this project, skip generating compile commands." - ) - return - - op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) - tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) - auto_gen_path_dir = os.path.dirname(cfgfile) - all_soc_key = "ALL" - for op_desc in op_descs: - op_desc.set_soc_version(soc) - op_desc.set_out_path(out_dir) - if op_desc.op_type in op_debug_config: - op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) - if all_soc_key in op_debug_config: - op_desc.set_op_debug_config(op_debug_config[all_soc_key]) - if op_desc.op_type in tiling_key_info: - op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) - if all_soc_key in tiling_key_info: - op_desc.set_tiling_key(tiling_key_info[all_soc_key]) - op_desc.gen_input_json(auto_gen_path_dir) - - -def parse_args(argv): - """Command line parameter parsing""" - parser = argparse.ArgumentParser() - parser.add_argument("argv", nargs="+") - parser.add_argument("--opc-config-file", nargs="?", const="", default="") - return parser.parse_args(argv) - - -if __name__ == "__main__": - args = parse_args(sys.argv) - if len(args.argv) <= 3: - raise RuntimeError("arguments must greater than 3") - gen_bin_param_file( - args.argv[1], args.argv[2], args.argv[3], opc_config_file=args.opc_config_file - ) diff --git a/csrc/deepep/ops2/cmake/util/ascendc_compile_kernel.py b/csrc/deepep/ops2/cmake/util/ascendc_compile_kernel.py deleted file mode 100755 index c4c815765..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_compile_kernel.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import os -import shutil -import subprocess -import sys -import time - -import ascendc_bin_param_build -import ascendc_impl_build -import ascendc_op_info -import const_var - - -class CompileKernel: - def __init__(self: any, args: any): - self.op_type = args.op_name - self.op_cpp_file = os.path.realpath(args.src_file) - self.op_soc_ver = args.compute_unit - self.compile_options = args.compile_options - self.op_debug_config = args.debug_config - self.op_cfg_ini = os.path.realpath(args.config_ini) - self.op_tiling = os.path.realpath(args.tiling_lib) - self.op_output = os.path.realpath(args.output_path) - self.op_impl_py = None - self.compile_sh = [] - self.working_dir = os.path.join( - os.getcwd(), - self.op_type + "_" + self.op_soc_ver, - ) - self.build_opp_path = os.path.join(self.working_dir, "customize") - os.makedirs(self.working_dir) - os.makedirs(self.op_output, exist_ok=True) - if args.dynamic_dir is not None and args.dynamic_dir != "": - self.dynamic_dir = os.path.realpath(args.dynamic_dir) - else: - self.dynamic_dir = None - if args.json_file is not None and args.json_file != "": - self.json_file = args.json_file - else: - self.json_file = None - - def clean(self: any): - if "dump_cce" not in self.op_debug_config: - shutil.rmtree(self.working_dir) - return - - def ascendc_gen_impl(self: any): - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = "" - rep_cfg[const_var.REPLAY_ITERATE] = "" - cfg_dir = {} - cfg_dir[const_var.CFG_IMPL_DIR] = os.path.dirname(self.op_cpp_file) - cfg_dir[const_var.CFG_OUT_DIR] = os.path.join(self.working_dir, "dynamic") - os.makedirs(os.path.join(self.working_dir, "dynamic"), exist_ok=True) - cfg_dir[const_var.AUTO_GEN_DIR] = os.path.dirname(self.op_cfg_ini) - ascendc_impl_build.write_scripts( - self.op_cfg_ini, rep_cfg, cfg_dir, [self.op_type], self.compile_options - ) - py_files = glob.glob(os.path.join(self.working_dir, "dynamic", "*.py")) - if py_files is None or len(py_files) != 1: - self.clean() - raise RuntimeError("compile py file {} generated error!".format(py_files)) - self.op_impl_py = os.path.join( - self.working_dir, "dynamic", self.op_type + ".py" - ) - if self.dynamic_dir is not None: - shutil.copy(py_files[0], self.dynamic_dir) - os.rename(py_files[0], self.op_impl_py) - if not os.path.exists(self.op_impl_py): - self.clean() - raise RuntimeError( - "compile py file {} not generated!".format(self.op_impl_py) - ) - - def ascendc_gen_param(self: any): - bin_param_path = os.path.join(self.working_dir, "bin_param") - os.makedirs(bin_param_path) - base_dir = os.path.dirname(self.op_cfg_ini) - opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") - ascendc_bin_param_build.gen_bin_param_file( - self.op_cfg_ini, - bin_param_path, - self.op_soc_ver, - opc_config_file, - [self.op_type], - ) - tiling_key_info, op_debug_config = ascendc_bin_param_build.parse_op_debug_confg( - opc_config_file, self.op_type - ) - if self.op_type in op_debug_config: - self.op_debug_config = op_debug_config[self.op_type] - if "ALL" in op_debug_config: - self.op_debug_config = op_debug_config["ALL"] - bin_param_files = glob.glob(os.path.join(bin_param_path, "*.json")) - if bin_param_files is None or len(bin_param_files) <= 0: - self.clean() - raise RuntimeError("compile binary param json file not generated!") - self.compile_sh = glob.glob(os.path.join(bin_param_path, "*.sh")) - if self.compile_sh is None or len(self.compile_sh) != len(bin_param_files): - self.clean() - raise RuntimeError("compile binary shell file not generated!") - - def ascendc_put_tiling(self: any): - tiling_path = os.path.join( - self.build_opp_path, "op_impl", "ai_core", "tbe", "op_tiling" - ) - os.makedirs(tiling_path) - tiling_so = os.path.join(tiling_path, "liboptiling.so") - os.symlink(self.op_tiling, tiling_so) - if not os.path.exists(tiling_so): - self.clean() - raise RuntimeError("prepare tiling lib {} link failed!".format(tiling_so)) - - def ascendc_put_json(self: any): - if self.json_file is not None: - json_file_dir = os.path.join( - self.build_opp_path, - "op_impl", - "ai_core", - "tbe", - "config", - self.op_soc_ver, - ) - os.makedirs(json_file_dir) - shutil.copy(self.json_file, json_file_dir) - build_json_file = os.path.join( - json_file_dir, "aic-{}-ops-info.json".format(self.op_soc_ver) - ) - if not os.path.exists(build_json_file): - self.clean() - raise RuntimeError( - "prepare json file aic-{}-ops-info.json failed!".format( - self.op_soc_ver - ) - ) - - def ascendc_build(self: any): - op_info = ascendc_op_info.OpInfo(self.op_type, self.op_cfg_ini) - op_file = op_info.get_op_file() - op_bin_dir = os.path.join(self.op_output, self.op_soc_ver, op_file) - os.makedirs(op_bin_dir, exist_ok=True) - all_tar = [] - sub_cmd = [] - index = 0 - for sh in self.compile_sh: - tar = op_file + str(index) - build_path = os.path.join(self.working_dir, "kernel_" + str(index)) - os.makedirs(build_path) - all_tar.append(tar) - sub_cmd.append(tar + ":") - sub_cmd.append( - "\tcd {} && bash {} --kernel-src=$(CPP) $(PY) $(OUT) $(MAKE)".format( - build_path, sh - ) - ) - index += 1 - mkfile = os.path.join(self.working_dir, op_file + ".make") - with os.fdopen(os.open(mkfile, const_var.WFLAGS, const_var.WMODES), "w") as fd: - sub_cmd.insert(0, "all: " + " ".join(all_tar)) - fd.write("\n".join(sub_cmd)) - - if os.getenv("TILINGKEY_PAR_COMPILE") is None: - cmd_str = ( - "export HI_PYTHON=python3 && export ASCEND_CUSTOM_OPP_PATH={} && export TILINGKEY_PAR_COMPILE=1" - "&& make -f {} PY={} OUT={} CPP={}" - ) - else: - cmd_str = "export HI_PYTHON=python3 && export ASCEND_CUSTOM_OPP_PATH={} && make -f {} PY={} OUT={} CPP={}" - - if ( - os.system( - cmd_str.format( - self.build_opp_path, - mkfile, - self.op_impl_py, - op_bin_dir, - self.op_cpp_file, - ) - ) - != 0 - ): - raise RuntimeError( - "Kernel Compilation Error: OpType {} Kernel File {}!".format( - self.op_type, self.op_cpp_file - ) - ) - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-n", "--op-name", nargs="?", help="Op name(Camel string) to compile." - ) - parser.add_argument("-s", "--src-file", nargs="?", help="Op kernel source file.") - - parser.add_argument("-u", "--compute-unit", nargs="?", help="Compute unit.") - parser.add_argument( - "-c", "--compile-options", nargs="?", help="Compile options of compiler." - ) - parser.add_argument( - "-d", - "--debug-config", - nargs="?", - help="Debug config of op, ref opc op-debug-config.", - ) - parser.add_argument("-i", "--config-ini", nargs="?", help="Op config ini file.") - parser.add_argument( - "-t", "--tiling-lib", nargs="?", help="Tiling shared library file." - ) - - parser.add_argument( - "-o", "--output-path", nargs="?", help="Output path of compile result." - ) - parser.add_argument( - "-dy", - "--dynamic-dir", - nargs="?", - default=None, - help="dynamic path of source compile.", - ) - parser.add_argument( - "-eb", - "--enable-binary", - nargs="?", - default=None, - help="whether binary compile is enabled.", - ) - parser.add_argument( - "-j", - "--json-file", - nargs="?", - default=None, - help="aic--ops-info.json file path.", - ) - # $(MAKE) is necessary for parallel compiling - parser.add_argument( - "-b", "--build-tool", nargs="?", default=None, help="build tool must be make." - ) - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - kernel_builder = CompileKernel(args) - kernel_builder.clean() - if args.enable_binary == "False": - kernel_builder.ascendc_gen_impl() - kernel_builder.clean() - else: - kernel_builder.ascendc_gen_impl() - kernel_builder.ascendc_gen_param() - kernel_builder.ascendc_put_json() - kernel_builder.ascendc_put_tiling() - kernel_builder.ascendc_build() - kernel_builder.clean() diff --git a/csrc/deepep/ops2/cmake/util/ascendc_gen_options.py b/csrc/deepep/ops2/cmake/util/ascendc_gen_options.py deleted file mode 100755 index 637ae9d79..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_gen_options.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -import json -import os -import re -import stat -import sys - -import const_var - - -def write_options_to_file( - file_name: str, options_str: str, op_type: str, compute_unit: str, split_char: str -): - flags = os.O_WRONLY | os.O_CREAT - modes = stat.S_IWUSR | stat.S_IRUSR - try: - with os.fdopen(os.open(file_name, flags, modes), "a") as fd: - fd.write( - op_type + split_char + compute_unit + split_char + options_str + "\n" - ) - except Exception as err: - print("write compile options config file failed") - raise (err) - - -def gen_compile_options( - compile_options_file: str, op_type: str, compute_unit: str, compile_options: list -): - base_dir = os.path.dirname(compile_options_file) - opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") - compile_opt = [] - opc_debug_config = [] - opc_tiling_keys = "" - for opts in compile_options: - if "oom" in opts: - if opts == "--oom": - opc_debug_config.append("oom") - else: - raise RuntimeError(f"Unknown oom option format {opts}") - elif "--save-temp-files" in opts: - opc_debug_config.append("dump_cce") - elif "--tiling_key" in opts: - keys = opts.strip().split("=")[1].split(",") - keys_str = ";".join([key for key in keys]) - opc_tiling_keys = keys_str - else: - compile_opt.append(opts) - if len(compile_opt) > 0: - options_str = ";".join([opt for opt in compile_opt]) - write_options_to_file( - compile_options_file, options_str, op_type, compute_unit, "," - ) - opc_config_str = "" - if opc_debug_config: - opc_config_str = "--op_debug_config=" + ";".join( - [opt for opt in opc_debug_config] - ) - if len(opc_tiling_keys) > 0: - if opc_config_str != "": - opc_config_str += "@" - opc_config_str += "--tiling_key=" + opc_tiling_keys - - if opc_config_str != "": - write_options_to_file( - opc_config_file, opc_config_str, op_type, compute_unit, "@" - ) - - -if __name__ == "__main__": - if len(sys.argv) < 4: - raise RuntimeError("arguments must greater than 4") - compute_soc = "" - comp_options = [] - for i in range(len(sys.argv) - 3): - if sys.argv[i + 3].upper().startswith("ASCEND"): - compute_soc += sys.argv[i + 3] + ";" - else: - comp_options.append(sys.argv[i + 3]) - if compute_soc != "": - compute_soc = compute_soc[0:-1] - gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) diff --git a/csrc/deepep/ops2/cmake/util/ascendc_get_op_name.py b/csrc/deepep/ops2/cmake/util/ascendc_get_op_name.py deleted file mode 100755 index 5da592b3b..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_get_op_name.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import configparser - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--ini-file", help="op info ini.") - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - op_config = configparser.ConfigParser() - op_config.read(args.ini_file) - for section in op_config.sections(): - print(section, end="-") - print(op_config.get(section, "opFile.value"), end="\n") diff --git a/csrc/deepep/ops2/cmake/util/ascendc_impl_build.py b/csrc/deepep/ops2/cmake/util/ascendc_impl_build.py deleted file mode 100755 index 7486ad138..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_impl_build.py +++ /dev/null @@ -1,748 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import datetime -import glob -import os -import re -import sys -from typing import List - -import const_var -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -IMPL_HEAD = '''#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. -""" - -import os, sys -import ctypes -import json -import shutil -from tbe.common.platform import get_soc_spec -from tbe.common.utils import para_check -from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo -from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel -from tbe.common.buildcfg import get_default_build_config -from impl.util.platform_adapter import tbe_register -from tbe.common.buildcfg import get_current_build_config -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], - "float16": ["DT_FLOAT16", "half"], - "int8": ["DT_INT8", "int8_t"], - "int16": ["DT_INT16", "int16_t"], - "int32": ["DT_INT32", "int32_t"], - "int64": ["DT_INT64", "int64_t"], - "uint1": ["DT_UINT1", "uint8_t"], - "uint8": ["DT_UINT8", "uint8_t"], - "uint16": ["DT_UINT16", "uint16_t"], - "uint32": ["DT_UINT32", "uint32_t"], - "uint64": ["DT_UINT64", "uint64_t"], - "bool": ["DT_BOOL", "bool"], - "double": ["DT_DOUBLE", "double"], - "dual": ["DT_DUAL", "unknown"], - "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], - "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], - "string": ["DT_STRING", "unknown"], - "complex32": ["DT_COMPLEX32", "unknown"], - "complex64": ["DT_COMPLEX64", "unknown"], - "complex128": ["DT_COMPLEX128", "unknown"], - "qint8": ["DT_QINT8", "unknown"], - "qint16": ["DT_QINT16", "unknown"], - "qint32": ["DT_QINT32", "unknown"], - "quint8": ["DT_QUINT8", "unknown"], - "quint16": ["DT_QUINT16", "unknown"], - "resource": ["DT_RESOURCE", "unknown"], - "string_ref": ["DT_STRING_REF", "unknown"], - "int4": ["DT_INT4", "int4b_t"], - "bfloat16": ["DT_BF16", "bfloat16_t"]}} - -def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): - options = [] - x_fmt = x.get("format") - x_dtype = x.get("dtype") - x_n_in_kernel = x_n + '_REF' if is_ref else x_n - options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) - options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) - options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) - return options - -def get_dtype_fmt_options(__inputs__, __outputs__): - options = [] - input_names = {} - output_names = {} - unique_param_name_set = set() - for idx, x in enumerate(__inputs__): - if x is None: - continue - x_n = input_names[idx].upper() - unique_param_name_set.add(x_n) - options += add_dtype_fmt_option_single(x, x_n) - - for idx, x in enumerate(__outputs__): - if x is None: - continue - x_n = output_names[idx].upper() - if x_n in unique_param_name_set: - options += add_dtype_fmt_option_single(x, x_n, True) - else: - options += add_dtype_fmt_option_single(x, x_n) - return options - -def load_dso(so_path): - try: - ctypes.CDLL(so_path) - except OSError as error : - CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) - raise RuntimeError("cannot open %s" %(so_path)) - else: - msg = "load so succ " + so_path - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - -def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): - compile_options = [] - if shortsoc in compile_option_list: - compile_options.extend(compile_option_list[shortsoc]) - if '__ALLSOC__' in compile_option_list: - compile_options.extend(compile_option_list['__ALLSOC__']) - return compile_options - -def get_kernel_source(src_file, dir_snake, dir_ex): - src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) - if os.path.exists(src_ex): - return src_ex - src = os.environ.get('BUILD_KERNEL_SRC') - if src and os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, src_file) - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) - if os.path.exists(src): - return src - return src_ex - -''' - -IMPL_API = """ -@tbe_register.register_operator("{}", trans_bool_to_s8=False) -@para_check.check_op_params({}) -def {}({}, kernel_name="{}", impl_mode=""): -{} - if get_current_build_config("enable_op_prebuild"): - return - __inputs__, __outputs__, __attrs__ = _build_args({}) - options = get_dtype_fmt_options(__inputs__, __outputs__) - options += ["-x", "cce"] - bisheng = os.environ.get('BISHENG_REAL_PATH') - if bisheng is None: - bisheng = shutil.which("bisheng") - if bisheng != None: - bisheng_path = os.path.dirname(bisheng) - tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) - else: - tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") - options.append("-I" + tikcpp_path) - options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) - options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) - if impl_mode == "high_performance": - options.append("-DHIGH_PERFORMANCE=1") - elif impl_mode == "high_precision": - options.append("-DHIGH_PRECISION=1") - if get_current_build_config("enable_deterministic_mode") == 1: - options.append("-DDETERMINISTIC_MODE=1") - else: - options.append("-DDETERMINISTIC_MODE=0") - - custom_compile_options = {}, - custom_all_compile_options = {}, - soc_version = get_soc_spec("SOC_VERSION") - soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() - custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) - custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) - options += custom_all_compile_options_soc - options += custom_compile_options_soc - - origin_func_name = "{}" - ascendc_src_dir_ex = "{}" - ascendc_src_dir = "{}" - ascendc_src_file = "{}" - src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) -""" - -REPLAY_OP_API = """ - msg = "start replay Ascend C Operator {}, kernel name is {}" - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" - tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version - msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" - replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" - if PYF_PATH.endswith("dynamic"): - op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") - else: - op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") - replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") - load_dso(codegen_so_path) - load_dso(replaystub_so_path) - load_dso(replayapi_so_path) - op_type = "{}" - entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) - res, msg = replay_op(op_info, entry_obj, code_channel, src, options) - if not res: - print("call replay op failed for %s and get into call compile op" %(msg)) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -""" - -COMPILE_OP_API = """ - msg = "start compile Ascend C Operator {}, kernel name is " + kernel_name - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - op_type = "{}" - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ - param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ - output_shape_depend_on_compute = {}) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -""" - -COMPILE_OP_API_BUILT_IN = """ - msg = "start compile Ascend C Operator {}, kernel name is " + kernel_name - CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) - op_type = "{}" - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ - param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ - output_shape_depend_on_compute = {}) - - op_compile_option = '{}' - opp_path = os.environ.get('ASCEND_OPP_PATH') - dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) - if opp_path and os.path.exists(dat_path): - # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op - abs_rel_kernel_src_path = "{}" - extend_options = {{}} - extend_options['opp_kernel_hidden_dat_path'] = dat_path - compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ - extend_options) - else: - raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) -""" - -SUP_API = """ -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - ret_dict = json.loads(ret_str) - err_code = ret_dict.get("ret_code") - sup = "Unknown" - reason = "Unknown reason" - if err_code is not None: - if err_code == 0: - sup = "True" - reason = "" - elif err_code == 1: - sup = "False" - reason = ret_dict.get("reason") - else: - sup = "Unknown" - reason = ret_dict.get("reason") - return sup, reason -""" -CAP_API = """ -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - return result.decode("utf-8") -""" -GLZ_API = """ -@tbe_register.register_param_generalization("{}") -def {}_generalization({}, generalize_config=None): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) - return [json.loads(ret_str)] -""" - -ATTR_DEFAULT = { - "bool": "False", - "int": "0", - "float": "0.0", - "list_int": "[]", - "list_float": "[]", - "list_bool": "[]", - "list_list_int": "[[]]", - "str": "", -} - - -def optype_snake(origin_str): - temp_str = origin_str[0].lower() + origin_str[1:] - new_str = re.sub(r"([A-Z])", r"_\1", temp_str).lower() - return new_str - - -def optype_snake_ex(s): - snake_case = "" - for i, c in enumerate(s): - if i == 0: - snake_case += c.lower() - elif c.isupper(): - if s[i - 1] != "_": - if not s[i - 1].isupper(): - snake_case += "_" - elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): - snake_case += "_" - snake_case += c.lower() - else: - snake_case += c - return snake_case - - -class AdpBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - self.argsdefv = [] - self.op_compile_option: str = "{}" - super().__init__(op_type) - - def write_adapt( - self: any, impl_path, path: str, op_compile_option_all: list = None - ): - self._build_paradefault() - if os.environ.get("BUILD_BUILTIN_OPP") != "1" and impl_path != "": - src_file = os.path.join(impl_path, self.op_file + ".cpp") - if not os.path.exists(src_file): - print( - f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check." - ) - return - out_path = os.path.abspath(path) - if self.dynamic_shape and not out_path.endswith("dynamic"): - out_path = os.path.join(path, "dynamic") - os.makedirs(out_path, exist_ok=True) - adpfile = os.path.join(out_path, self.op_file + ".py") - self._gen_op_compile_option(op_compile_option_all) - with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), "w") as fd: - self._write_head(fd) - self._write_argparse(fd) - self._write_impl(fd, impl_path) - if self.op_chk_support: - self._write_cap("check_supported", fd) - self._write_cap("get_op_support_info", fd) - if self.op_fmt_sel: - self._write_cap("op_select_format", fd) - self._write_cap("get_op_specific_info", fd) - if self.op_range_limit == "limited" or self.op_range_limit == "dynamic": - self._write_glz(fd) - - def _gen_op_compile_option(self: any, op_compile_option_all: list = None): - if op_compile_option_all is not None: - if self.op_type in op_compile_option_all: - self.op_compile_option = op_compile_option_all[self.op_type] - elif "__all__" in op_compile_option_all: - self.op_compile_option = op_compile_option_all["__all__"] - - def _ip_argpack(self: any, default: bool = True) -> list: - args = [] - for i in range(len(self.input_name)): - arg = self.input_name[i] - if default and self.argsdefv[i] is not None: - arg += "=" + self.argsdefv[i] - args.append(arg) - return args - - def _op_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) - for i in range(len(self.output_name)): - arg = self.output_name[i] - if default and self.argsdefv[i + argidx] is not None: - arg += "=" + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _attr_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) + len(self.output_name) - for i in range(len(self.attr_list)): - att = self.attr_list[i] - arg = att - if default and self.argsdefv[i + argidx] is not None: - if self.attr_val.get(att).get("type") == "str": - arg += '="' + self.argsdefv[i + argidx] + '"' - elif self.attr_val.get(att).get("type") == "bool": - arg += "=" + self.argsdefv[i + argidx].capitalize() - else: - arg += "=" + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _build_paralist(self: any, default: bool = True) -> str: - args = [] - args.extend(self._ip_argpack(default)) - args.extend(self._op_argpack(default)) - args.extend(self._attr_argpack(default)) - return ", ".join(args) - - def _io_parachk(self: any, types: list, type_name: str) -> list: - chk = [] - for iot in types: - if iot == "optional": - ptype = "OPTION" - else: - ptype = iot.upper() - chk.append("para_check.{}_{}".format(ptype, type_name)) - return chk - - def _attr_parachk(self: any) -> list: - chk = [] - for att in self.attr_list: - att_type = self.attr_val.get(att).get("type").upper() - chk.append("para_check.{}_ATTR_{}".format("OPTION", att_type)) - return chk - - def _build_parachk(self: any) -> str: - chk = [] - chk.extend(self._io_parachk(self.input_type, "INPUT")) - chk.extend(self._io_parachk(self.output_type, "OUTPUT")) - chk.extend(self._attr_parachk()) - chk.append("para_check.KERNEL_NAME") - return ", ".join(chk) - - def _build_virtual(self: any) -> str: - virt_exp = [] - for index in range(len(self.input_name)): - if self.input_virt.get(index) is None: - continue - val = [] - val.append('"param_name":"{}"'.format(self.input_name[index])) - val.append('"index":{}'.format(index)) - val.append('"dtype":"{}"'.format(self.input_dtype[index].split(",")[0])) - val.append('"format":"{}"'.format(self.input_fmt[index].split(",")[0])) - val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(",")[0])) - val.append('"paramType":"optional"') - val.append('"shape":[1]') - val.append('"ori_shape":[1]') - virt_exp.append( - " " + self.input_name[index] + " = {" + ",".join(val) + "}" - ) - if len(virt_exp) > 0: - return "\n".join(virt_exp) - else: - return " # do ascendc build step" - - def _build_mc2_ctx(self: any): - if len(self.mc2_ctx) != 0: - return '["' + '", "'.join(self.mc2_ctx) + '"]' - return "[]" - - def _build_paradefault(self: any): - optional = False - argtypes = [] - argtypes.extend(self.input_type) - argtypes.extend(self.output_type) - in_idx = 0 - for atype in argtypes: - if atype == "optional": - optional = True - if optional: - self.argsdefv.append("None") - else: - self.argsdefv.append(None) - in_idx += 1 - for attr in self.attr_list: - atype = self.attr_val.get(attr).get("paramType") - if atype == "optional": - optional = True - attrval = self.attr_val.get(attr).get("defaultValue") - if attrval is not None: - optional = True - if type == "bool": - attrval = attrval.capitalize() - elif type == "str": - attrval = '"' + attrval + '"' - self.argsdefv.append(attrval) - continue - if optional: - self.argsdefv.append( - ATTR_DEFAULT.get(self.attr_val.get(attr).get("type")) - ) - else: - self.argsdefv.append(None) - - def _write_head(self: any, fd: object): - now = datetime.datetime.now() - curr_year = now.year - former_year = curr_year - 1 - fd.write( - IMPL_HEAD.format( - former_year, curr_year, self.input_ori_name, self.output_ori_name - ) - ) - - def _write_argparse(self: any, fd: object): - args = self._build_paralist(False) - fd.write("def _build_args({}):\n".format(args)) - fd.write(" __inputs__ = []\n") - fd.write(" for arg in [{}]:\n".format(", ".join(self.input_name))) - fd.write(" if arg != None:\n") - fd.write(" if isinstance(arg, (list, tuple)):\n") - fd.write(" if len(arg) == 0:\n") - fd.write(" continue\n") - fd.write(" __inputs__.append(arg[0])\n") - fd.write(" else:\n") - fd.write(" __inputs__.append(arg)\n") - fd.write(" else:\n") - fd.write(" __inputs__.append(arg)\n") - fd.write(" __outputs__ = []\n") - fd.write(" for arg in [{}]:\n".format(", ".join(self.output_name))) - fd.write(" if arg != None:\n") - fd.write(" if isinstance(arg, (list, tuple)):\n") - fd.write(" if len(arg) == 0:\n") - fd.write(" continue\n") - fd.write(" __outputs__.append(arg[0])\n") - fd.write(" else:\n") - fd.write(" __outputs__.append(arg)\n") - fd.write(" else:\n") - fd.write(" __outputs__.append(arg)\n") - fd.write(" __attrs__ = []\n") - for attr in self.attr_list: - fd.write(" if {} != None:\n".format(attr)) - fd.write(" attr = {}\n") - fd.write(' attr["name"] = "{}"\n'.format(attr)) - fd.write( - ' attr["dtype"] = "{}"\n'.format( - self.attr_val.get(attr).get("type") - ) - ) - fd.write(' attr["value"] = {}\n'.format(attr)) - fd.write(" __attrs__.append(attr)\n") - fd.write(" return __inputs__, __outputs__, __attrs__\n") - - def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): - src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) - if os.path.exists(src_ex): - return src_ex - src = os.environ.get("BUILD_KERNEL_SRC") - if src and os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, dir_snake, src_file) - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, src_file) - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") - if os.path.exists(src): - return src - src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) - if os.path.exists(src): - return src - return src_ex - - def _write_impl(self: any, fd: object, impl_path: str = ""): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - pchk = self._build_parachk() - if len(self.kern_name) > 0: - kern_name = self.kern_name - else: - kern_name = self.op_intf - src = self.op_file + ".cpp" - virt_exprs = self._build_virtual() - fd.write( - IMPL_API.format( - self.op_type, - pchk, - self.op_intf, - argsdef, - kern_name, - virt_exprs, - argsval, - self.custom_compile_options, - self.custom_all_compile_options, - self.op_intf, - optype_snake_ex(self.op_type), - optype_snake(self.op_type), - src, - ) - ) - if self.op_replay_flag: - fd.write( - REPLAY_OP_API.format( - self.op_type, - kern_name, - self.op_file, - self.op_type, - self.op_file, - self.param_type_dynamic, - self.op_compile_option, - ) - ) - else: - if os.environ.get("BUILD_BUILTIN_OPP") == "1": - relative_kernel_src_path = os.path.realpath( - self._get_kernel_source( - impl_path, - src, - optype_snake(self.op_type), - optype_snake_ex(self.op_type), - ) - ) - # to match src path in .dat file system, turn relative path into absolute path - abs_rel_kernel_src_path = os.path.join( - "/", os.path.relpath(relative_kernel_src_path, impl_path) - ) - - # compiling hidden src file requires src path before packaging .dat file, - # hard code such src path to .py - fd.write( - COMPILE_OP_API_BUILT_IN.format( - self.op_type, - self.op_type, - ", ".join(self.input_name), - ", ".join(self.output_name), - self.param_type_dynamic, - self._build_mc2_ctx(), - self.input_type + self.output_type, - self.output_init_value, - self.output_shape_depend_on_compute, - self.op_compile_option, - abs_rel_kernel_src_path, - ) - ) - else: - fd.write( - COMPILE_OP_API.format( - self.op_type, - self.op_type, - ", ".join(self.input_name), - ", ".join(self.output_name), - self.param_type_dynamic, - self._build_mc2_ctx(), - self.input_type + self.output_type, - self.output_init_value, - self.output_shape_depend_on_compute, - self.op_compile_option, - ) - ) - - def _write_cap(self: any, cap_name: str, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - if cap_name == "check_supported": - fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - else: - fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - - def _write_glz(self: any, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - fd.write( - GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type) - ) - - -def write_scripts( - cfgfile: str, - cfgs: dict, - dirs: dict, - ops: list = None, - op_compile_option: list = None, -): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(";") - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(";") - file_map = {} - op_descs = opdesc_parser.get_op_desc( - cfgfile, - batch_lists, - iterator_lists, - AdpBuilder, - ops, - dirs.get(const_var.AUTO_GEN_DIR), - ) - for op_desc in op_descs: - op_desc.write_adapt( - dirs.get(const_var.CFG_IMPL_DIR), - dirs.get(const_var.CFG_OUT_DIR), - op_compile_option, - ) - file_map[op_desc.op_type] = op_desc.op_file - return file_map - - -class OpFileNotExistsError(Exception): - """File does not exist error.""" - - def __str__(self) -> str: - return ( - f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" - ) - - -def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: - """Get all ops info files.""" - ops_info_files = [] - for _dir in opsinfo_dir: - ops_info_files.extend(glob.glob(f"{_dir}/aic-*-ops-info.ini")) - return sorted(ops_info_files) - - -def parse_args(argv): - """Command line parameter parsing""" - parser = argparse.ArgumentParser() - parser.add_argument("argv", nargs="+") - parser.add_argument("--opsinfo-dir", nargs="*", default=None) - return parser.parse_args(argv) - - -if __name__ == "__main__": - args = parse_args(sys.argv) - - if len(args.argv) <= 6: - raise RuntimeError("arguments must greater equal than 6") - - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] - - cfg_dir = {} - cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] - cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] - cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] - - ops_infos = [] - if args.opsinfo_dir: - ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) - if not ops_infos: - raise OpFileNotExistsError(args.opsinfo_dir) - else: - ops_infos.append(args.argv[1]) - - for ops_info in ops_infos: - write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/csrc/deepep/ops2/cmake/util/ascendc_op_info.py b/csrc/deepep/ops2/cmake/util/ascendc_op_info.py deleted file mode 100755 index a75404285..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_op_info.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import sys - -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class OpInfo: - def __init__(self: any, op_type: str, cfg_file: str): - op_descs = opdesc_parser.get_op_desc( - cfg_file, [], [], opdesc_parser.OpDesc, [op_type] - ) - if op_descs is None or len(op_descs) != 1: - raise RuntimeError("cannot get op info of {}".format(op_type)) - self.op_desc = op_descs[0] - - def get_op_file(self: any): - return self.op_desc.op_file - - def get_op_intf(self: any): - return self.op_desc.op_intf - - def get_inputs_name(self: any): - return self.op_desc.input_ori_name - - def get_outputs_name(self: any): - return self.op_desc.output_ori_name - - -if __name__ == "__main__": - if len(sys.argv) <= 2: - raise RuntimeError("arguments must greater than 2") - op_info = OpInfo(sys.argv[1], sys.argv[2]) - print(op_info.get_op_file()) - print(op_info.get_op_intf()) - print(op_info.get_inputs_name()) - print(op_info.get_outputs_name()) diff --git a/csrc/deepep/ops2/cmake/util/ascendc_ops_config.py b/csrc/deepep/ops2/cmake/util/ascendc_ops_config.py deleted file mode 100755 index 8c5dd2763..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_ops_config.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import json -import os -import sys - -import const_var - -BINARY_INFO_CONFIG_JSON = "binary_info_config.json" - - -def load_json(json_file: str): - with open(json_file, encoding="utf-8") as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, "**/*.{}".format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return sorted(all_suffix_files) - - -def add_dict_key(dict_to_add, key, value): - if value is None: - return - dict_to_add[key] = value - - -def correct_format_mode(format_mode): - if format_mode == "FormatDefault": - return "nd_agnostic" - if format_mode == "FormatAgnostic": - return "static_nd_agnostic" - if format_mode == "FormatFixed": - return "normal" - return format_mode - - -def get_input_or_output_config(in_or_out): - param_dict = {} - name = in_or_out.get("name") - index = in_or_out.get("index") - param_type = in_or_out.get("paramType") - - format_match_mode = in_or_out.get("format_match_mode") - format_mode = correct_format_mode(format_match_mode) - - dtype_mode = in_or_out.get("dtype_match_mode") - if dtype_mode == "DtypeByte": - dtype_mode = "bit" - - add_dict_key(param_dict, "name", name) - add_dict_key(param_dict, "index", index) - add_dict_key(param_dict, "paramType", param_type) - add_dict_key(param_dict, "dtypeMode", dtype_mode) - add_dict_key(param_dict, "formatMode", format_mode) - return param_dict - - -def get_inputs_or_outputs_config(inputs_or_outputs): - if inputs_or_outputs is None: - return None - inputs_or_outputs_list = [] - - for in_or_out in inputs_or_outputs: - if isinstance(in_or_out, dict): - dict_param_config = get_input_or_output_config(in_or_out) - inputs_or_outputs_list.append(dict_param_config) - elif isinstance(in_or_out, list): - param_info = in_or_out[0] - list_param_config = get_input_or_output_config(param_info) - tmp_list = [list_param_config] - inputs_or_outputs_list.append(tmp_list) - return inputs_or_outputs_list - - -def gen_attrs_config(attrs): - attrs_list = [] - for attr in attrs: - attrs_dict = {} - name = attr.get("name") - mode = attr.get("mode") - add_dict_key(attrs_dict, "name", name) - add_dict_key(attrs_dict, "mode", mode) - attrs_list.append(attrs_dict) - return attrs_list - - -def get_params_config(support_info): - params_dict = {} - - inputs = support_info.get("inputs") - inputs_list = get_inputs_or_outputs_config(inputs) - params_dict["inputs"] = inputs_list - - outputs = support_info.get("outputs") - outputs_list = get_inputs_or_outputs_config(outputs) - params_dict["outputs"] = outputs_list - - attrs = support_info.get("attrs") - if attrs is not None: - attrs_list = gen_attrs_config(attrs) - params_dict["attrs"] = attrs_list - - return params_dict - - -def add_simplified_config( - op_type, support_info, core_type, task_ration, objfile, config -): - simplified_key = support_info.get("simplifiedKey") - - json_path = objfile.split(".")[0] + ".json" - - simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) - op_cfg = simple_cfg.get(op_type) - if not op_cfg: - op_cfg = {"dynamicRankSupport": True} - - simplified_key_mode = support_info.get("simplifiedKeyMode") - add_dict_key(op_cfg, "simplifiedKeyMode", simplified_key_mode) - - optional_input_mode = support_info.get("optionalInputMode") - optional_output_mode = support_info.get("optionalOutputMode") - add_dict_key(op_cfg, "optionalInputMode", optional_input_mode) - if optional_output_mode is not None: - add_dict_key(op_cfg, "optionalOutputMode", optional_output_mode) - - params_info = get_params_config(support_info) - op_cfg["params"] = params_info - op_cfg["binaryList"] = [] - simple_cfg[op_type] = op_cfg - - bin_list = op_cfg.get("binaryList") - if core_type == 0 and task_ration == "tilingKey": - bin_list.append( - { - "coreType": core_type, - "simplifiedKey": simplified_key, - "multiKernelType": 1, - "binPath": objfile, - "jsonPath": json_path, - } - ) - else: - bin_list.append( - { - "coreType": core_type, - "simplifiedKey": simplified_key, - "binPath": objfile, - "jsonPath": json_path, - } - ) - - -def add_op_config(op_file, bin_info, config): - op_cfg = config.get(op_file) - if not op_cfg: - op_cfg = {"binList": []} - config[op_file] = op_cfg - op_cfg.get("binList").append(bin_info) - - -def gen_ops_config(json_file, soc, config): - core_type_map = { - "MIX": 0, - "AiCore": 1, - "VectorCore": 2, - "MIX_AICORE": 3, - "MIX_VECTOR_CORE": 4, - "MIX_AIV": 4, - } - contents = load_json(json_file) - if ("binFileName" not in contents) or ("supportInfo" not in contents): - return - json_base_name = os.path.basename(json_file) - op_dir = os.path.basename(os.path.dirname(json_file)) - - support_info = contents.get("supportInfo") - bin_name = contents.get("binFileName") - bin_suffix = contents.get("binFileSuffix") - core_type = contents.get("coreType") - task_ration = contents.get("taskRation") - core_type = core_type_map.get(core_type, -1) - if core_type == -1 and soc != "ascend310b": - raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") - - bin_file_name = bin_name + bin_suffix - op_type = bin_name.split("_")[0] - op_file = op_dir + ".json" - bin_info = {} - - add_dict_key(bin_info, "implMode", support_info.get("implMode")) - add_dict_key(bin_info, "int64Mode", support_info.get("int64Mode")) - add_dict_key(bin_info, "simplifiedKeyMode", support_info.get("simplifiedKeyMode")) - - simplified_key = support_info.get("simplifiedKey") - if simplified_key is not None: - bin_info["simplifiedKey"] = simplified_key - obj_file = os.path.join(soc, op_dir, bin_file_name) - add_simplified_config( - op_type, support_info, core_type, task_ration, obj_file, config - ) - - add_dict_key(bin_info, "dynamicParamMode", support_info.get("dynamicParamMode")) - bin_info["staticKey"] = support_info.get("staticKey") - bin_info["inputs"] = support_info.get("inputs") - bin_info["outputs"] = support_info.get("outputs") - if support_info.get("attrs"): - bin_info["attrs"] = support_info.get("attrs") - - add_dict_key(bin_info, "opMode", support_info.get("opMode")) - add_dict_key(bin_info, "optionalInputMode", support_info.get("optionalInputMode")) - add_dict_key(bin_info, "deterministic", support_info.get("deterministic")) - if support_info.get("optionalOutputMode") is not None: - add_dict_key( - bin_info, "optionalOutputMode", support_info.get("optionalOutputMode") - ) - - bin_info["binInfo"] = {"jsonFilePath": os.path.join(soc, op_dir, json_base_name)} - add_op_config(op_file, bin_info, config) - - -def check_single_op_is_void(root_dir): - for root, dirs, _ in os.walk(root_dir): - for sub_dir in dirs: - dir_path = os.path.join(root, sub_dir) - if len(os.listdir(dir_path)) == 0: - print(f"[ERROR] op {sub_dir}: not any obj compile success") - sys.exit(1) - - -def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): - suffix = "json" - config = {BINARY_INFO_CONFIG_JSON: {}} - check_single_op_is_void(root_dir) - all_json_files = get_specified_suffix_file(root_dir, suffix) - - for _json in all_json_files: - gen_ops_config(_json, soc, config) - file_path = soc + _json.split(soc)[1] - with open(_json, "r+") as f: - data = json.load(f) - data["filePath"] = file_path - f.seek(0) - json.dump(data, f, indent=" ") - f.truncate() - - for cfg_key in config.keys(): - if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: - continue - cfg_file = os.path.join(out_dir, cfg_key) - with os.fdopen( - os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - json.dump(config.get(cfg_key), fd, indent=" ") - - -# Parse multiple soc_versions ops in single path. -def gen_all_soc_config(all_path): - soc_roots = glob.glob(os.path.join(all_path, "ascend*")) - - for soc_root in soc_roots: - soc = os.path.basename(soc_root) - gen_all_config(soc_root, soc, soc_root, True) - cfg_files = glob.glob(os.path.join(soc_root, "*.json")) - cfg_path = os.path.join(all_path, "config", soc) - os.makedirs(cfg_path, exist_ok=True) - for cfg_file in cfg_files: - new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) - os.rename(cfg_file, new_file) - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-p", - "--path", - nargs="?", - required=True, - help="Parse the path of the json file.", - ) - - parser.add_argument( - "-s", "--soc", nargs="?", required=True, help="Parse the soc_version of ops." - ) - - parser.add_argument("-o", "--out", nargs="?", help="Output directory.") - - parser.add_argument( - "--skip-binary-info-config", - action="store_true", - help="binary_info_config.json file is not parsed.", - ) - - return parser.parse_args() - - -def main(): - args = args_prase() - if args.out is None: - out_dir = args.path - else: - out_dir = args.out - - gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) - - -if __name__ == "__main__": - main() diff --git a/csrc/deepep/ops2/cmake/util/ascendc_pack_kernel.py b/csrc/deepep/ops2/cmake/util/ascendc_pack_kernel.py deleted file mode 100755 index 430c1ba8e..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_pack_kernel.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import json -import math -import os -import subprocess -import sys - -import ascendc_ops_config -import const_var -from tbe.tikcpp.log_utils import AscendCLogLevel, LogUtil - - -class PackKernel: - def __init__(self: any, args: any): - self.in_path = os.path.realpath(args.input_path) - self.out_path = os.path.realpath(args.output_path) - self.is_lib = args.enable_library - self.platform = args.platform - self.op_info = {} - self.file_info = {} - try: - os.makedirs(self.out_path, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {self.out_path} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def load_json(self: any, json_file: str): - with open(json_file, encoding="utf-8") as file: - json_content = json.load(file) - return json_content - - def get_symbol(self: any, name: str): - name = name.replace("/", "_") - return name.replace(".", "_") - - def ascendc_gen_object(self: any, in_file: str, soc: str): - sym = self.get_symbol("_binary_" + in_file) - out_file = os.path.join(self.out_path, sym + ".o") - # ascend610lite only support aarch64 - if soc == "ascend610lite": - try: - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - except Exception as e: - LogUtil.print_compile_log( - "", - " ascend610lite execute objcopy fail!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - uname = os.popen("uname -m").read().strip() - if self.platform is not None: - target_platform = self.platform - else: - target_platform = uname - try: - if target_platform == "x86_64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-x86-64", - "--binary-architecture", - "i386", - in_file, - out_file, - ] - ) - elif target_platform == "aarch64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - else: - subprocess.run(["echo", "unsported environment!"]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"{target_platform} execute objcopy error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - - def ascendc_get_config(self: any): - os.chdir(self.in_path) - soc_vers = os.listdir("config") - for soc in soc_vers: - bin_infos = glob.glob(os.path.join("config", soc, "*.json")) - cfgs = {} - for bin_info in bin_infos: - if bin_info.find("binary_info_config.json") > 0: - continue - jobj = self.load_json(bin_info) - for bin_cfg in jobj.get("binList"): - js_cfg = bin_cfg.get("binInfo").get("jsonFilePath") - op_type = os.path.basename(js_cfg).split("_")[0] - if cfgs.get(op_type) is None: - op_obj = {} - op_obj["obj"] = [] - op_obj["cfg"] = bin_info - cfgs[op_type] = op_obj - op_obj = cfgs.get(op_type) - op_obj.get("obj").append(js_cfg[:-5]) - self.file_info[soc] = cfgs - - def ascendc_pack_kernel(self: any): - for soc in self.file_info.keys(): - os.chdir(self.in_path) - op_cfgs = self.file_info.get(soc) - for op_type in op_cfgs.keys(): - op_obj = op_cfgs.get(op_type) - if self.op_info.get(op_type) is None: - op_info = {} - op_info["op_fun"] = ["nullptr", "nullptr"] - op_info["op_bin"] = {} - op_info["op_rkb"] = [] - self.op_info[op_type] = op_info - op_info = self.op_info.get(op_type) - op_bin = op_info.get("op_bin") - if op_bin.get(soc) is None: - op_bin[soc] = [] - op_bin[soc].append(self.ascendc_gen_object(op_obj["cfg"], soc)) - op_soc = op_bin.get(soc) - for objs in op_obj["obj"]: - op_soc.append(self.ascendc_gen_object(objs + ".json", soc)) - op_soc.append(self.ascendc_gen_object(objs + ".o", soc)) - - def ascendc_gen_header(self: any): - for op_type in self.op_info.keys(): - op_obj = self.op_info.get(op_type) - macro_op = ( - "#define {}_OP_RESOURCES std::make_tuple, \\\n" - " std::map>>, \\\n" - " std::vector>>({{{}}}, \\\n".format( - op_type, ", ".join(op_obj.get("op_fun")) - ) - ) - op_bin = op_obj.get("op_bin") - socs_res = [] - op_syms = [] - for soc in op_bin.keys(): - soc_res = '{{ "{}", {{'.format(soc) - soc_syms = op_bin.get(soc) - soc_pairs = [] - for pair_addr in soc_syms: - pair_addr1 = ["&" + s for s in pair_addr] - op_syms += pair_addr - soc_pairs.append( - " {{ {} }} ".format(", \\\n ".join(pair_addr1)) - ) - soc_res += ", \\\n ".join(soc_pairs) - soc_res += " } }" - socs_res.append(soc_res) - macro_op += " {{ {} }}, \\\n".format(", \\\n ".join(socs_res)) - macro_op += " {{ {} }})\n\n".format(", ".join(op_obj.get("op_rkb"))) - macro_str = '#define {}_RESOURCES {{{{"{}", {}}}}}'.format( - op_type, op_type, "{}_OP_RESOURCES".format(op_type) - ) - var_str = ( - "extern gert::OpImplRegisterV2 op_impl_register_optiling_{};\n".format( - op_type - ) - ) - if len(op_syms) > 0: - var_str += ( - "extern uint8_t " + ";\nextern uint8_t ".join(op_syms) + ";\n" - ) - head_file = os.path.join(self.out_path, "{}_op_resource.h".format(op_type)) - try: - with os.fdopen( - os.open(head_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write('#include "graph/ascend_string.h"\n') - fd.write('#include "register/op_impl_registry.h"\n\n') - fd.write(var_str) - fd.write("\n") - fd.write(macro_op) - fd.write(macro_str) - except Exception as e: - LogUtil.print_compile_log( - "", - f"{op_type}_op_resource.h create error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_gen_lib(self: any): - out_lib = os.path.join(self.out_path, "libkernels.a") - if os.path.exists(out_lib): - os.remove(out_lib) - objs = glob.glob(os.path.join(self.out_path, "*.o")) - start = 0 - batch_size = 100 - for _ in range(math.ceil(len(objs) / batch_size)): - sub_objs = objs[start : start + batch_size] - start += batch_size - try: - subprocess.run(["ar", "qc", out_lib] + sub_objs) - subprocess.run(["ranlib", out_lib]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"execute ar/ranlib command error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_gen_opsinfo(self: any): - ascendc_ops_config.gen_all_soc_config(self.in_path) - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", "--input-path", nargs="?", help="Input path of compile result." - ) - parser.add_argument( - "-o", "--output-path", nargs="?", help="Output path of compile result." - ) - parser.add_argument( - "-l", - "--enable-library", - nargs="?", - default=None, - help="Whether library is enabled.", - ) - parser.add_argument( - "-p", - "--platform", - nargs="?", - default=None, - help="target platform is x86_64 or aarch64.", - ) - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - kernel_packer = PackKernel(args) - if kernel_packer.is_lib is None: - kernel_packer.ascendc_gen_opsinfo() - kernel_packer.ascendc_get_config() - kernel_packer.ascendc_pack_kernel() - kernel_packer.ascendc_gen_header() - kernel_packer.ascendc_gen_lib() diff --git a/csrc/deepep/ops2/cmake/util/ascendc_pack_opregistry.py b/csrc/deepep/ops2/cmake/util/ascendc_pack_opregistry.py deleted file mode 100755 index 1acb7e45e..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_pack_opregistry.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import math -import os -import shutil -import subprocess -import sys - -import const_var -from tbe.tikcpp.log_utils import AscendCLogLevel, LogUtil - - -class PackKernel: - def __init__(self: any, args: any): - self.in_path = os.path.realpath(args.input_path) - self.copy_path = os.path.realpath(args.copy_path) - self.out_path = os.path.realpath(args.output_path) - self.op_soc_ver = args.compute_unit.split("-") - self.vendor_name = args.vendor_name - self.framework_type = args.framework_type - self.platform = args.platform - self.op_info = {} - self.file_info = {} - if os.path.exists(self.copy_path): - try: - shutil.rmtree(self.copy_path) - except OSError as e: - LogUtil.print_compile_log( - "", - f"remove {self.copy_path} error!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - if os.path.exists(self.out_path): - try: - shutil.rmtree(self.out_path) - except OSError as e: - LogUtil.print_compile_log( - "", - f"remove {self.out_path} error!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - try: - os.makedirs(self.copy_path, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {self.copy_path} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - try: - os.makedirs(self.out_path, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {self.out_path} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def get_symbol(self: any, name: str): - name = name.replace("/", "_") - name = name.replace("-", "_") - return name.replace(".", "_") - - def ascendc_gen_object(self: any, in_file: str, path: str): - sym = self.get_symbol("_binary_" + in_file) - out_file = os.path.join(self.out_path, sym + ".o") - # ascend610lite only support aarch64 - if path.find("ascend610lite") != -1: - try: - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - except Exception as e: - LogUtil.print_compile_log( - "", - " ascend610lite execute objcopy fail!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - - uname = os.popen("uname -m").read().strip() - if self.platform is not None: - target_platform = self.platform - else: - target_platform = uname - try: - if target_platform == "x86_64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-x86-64", - "--binary-architecture", - "i386", - in_file, - out_file, - ] - ) - elif target_platform == "aarch64": - subprocess.run( - [ - "llvm-objcopy", - "--input-target", - "binary", - "--output-target", - "elf64-littleaarch64", - "--binary-architecture", - "aarch64", - in_file, - out_file, - ] - ) - else: - subprocess.run(["echo", "unsupported environment!"]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"{target_platform} execute objcopy error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - return None - return [sym + "_start", sym + "_end"] - - def ascendc_get_config(self: any): - os.chdir(self.copy_path) - current_directory = os.getcwd() - catalog_file = os.listdir(current_directory) - for catalog in catalog_file: - if catalog == "op_impl" or catalog == "framework": - files_dict = {} - for root, _, files in os.walk(catalog): - for file in files: - if ( - file.endswith(".json") - or file.endswith(".so") - or file.endswith(".cpp") - or file.endswith(".py") - or file.endswith(".o") - ): - file_path = os.path.join(root, file) - file_name = os.path.basename(file_path) - files_dict[file_name] = file_path - self.file_info[catalog] = files_dict - - def ascendc_pack_kernel(self: any): - op_info = {} - for files in self.file_info.keys(): - os.chdir(self.copy_path) - op_cfgs = self.file_info.get(files) - for file_name in op_cfgs.keys(): - op_info[file_name] = [] - path, filename = os.path.split(op_cfgs[file_name]) - op_info[file_name].append(os.path.join(self.vendor_name, path)) - op_info[file_name].append( - self.ascendc_gen_object(op_cfgs[file_name], path) - ) - self.op_info = op_info - - def ascendc_gen_header(self: any): - socs_res = [] - var_str = "" - macro_op = ( - "std::vector> __ascendc_op_info = \n" - ) - for file_name in self.op_info.keys(): - file_addr = self.op_info.get(file_name) - soc_pairs = [] - op_syms = [] - soc_res = ' {{ "{}", '.format(file_name) - soc_res += '"{}", '.format(file_addr[0]) - for pair_addr in file_addr[1]: - op_syms.append(pair_addr) - pair_addr1 = "&" + pair_addr - soc_pairs.append(pair_addr1) - soc_res += "{}, {}".format(soc_pairs[0], soc_pairs[1]) - soc_res += "}, \n" - socs_res.append(soc_res) - if len(op_syms) > 0: - var_str += "".join( - ["extern uint8_t {};\n".format(sym) for sym in op_syms] - ) - macro_op += "{{\n{}}}; \n".format("".join(socs_res)) - head_file = os.path.join(self.out_path, "ge_table_op_resource.h") - try: - with os.fdopen( - os.open(head_file, const_var.WFLAGS, const_var.WMODES), "w" - ) as fd: - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write("#include \n") - fd.write('#include "graph/ascend_string.h"\n') - fd.write('#include "register/op_impl_registry.h"\n\n') - fd.write(var_str) - fd.write("\n") - fd.write("namespace AscendC {\n") - fd.write(macro_op) - fd.write("}\n") - except Exception as e: - LogUtil.print_compile_log( - "", - f"ge_table_op_resource.h create error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_gen_lib(self: any): - out_lib = os.path.join(self.out_path, "libopregistry.a") - if os.path.exists(out_lib): - os.remove(out_lib) - objs = glob.glob(os.path.join(self.out_path, "*.o")) - start = 0 - batch_size = 100 - for _ in range(math.ceil(len(objs) / batch_size)): - sub_objs = objs[start : start + batch_size] - start += batch_size - try: - subprocess.run(["ar", "qc", out_lib] + sub_objs) - subprocess.run(["ranlib", out_lib]) - except Exception as e: - LogUtil.print_compile_log( - "", - f"execute ar/ranlib command error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_copy_dir(self: any, src_dir: str, target_dir: str): - file_list = os.listdir(src_dir) - for file_name in file_list: - source_file = os.path.join(src_dir, file_name) - target_file = os.path.join(target_dir, file_name) - if os.path.isdir(source_file): - try: - shutil.copytree(source_file, target_file) - except Exception as e: - LogUtil.print_compile_log( - "", - f"copy {source_file} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_copy_file(self: any, src_dir: str, target_dir: str): - file_list = os.listdir(src_dir) - for file_name in file_list: - source_file = os.path.join(src_dir, file_name) - if os.path.isfile(source_file): - try: - os.makedirs(target_dir, exist_ok=True) - except Exception as e: - LogUtil.print_compile_log( - "", - f"make {target_dir} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - try: - shutil.copy(source_file, target_dir) - except Exception as e: - LogUtil.print_compile_log( - "", - f"copy {source_file} error: {e}!", - AscendCLogLevel.LOG_ERROR, - LogUtil.Option.NON_SOC, - ) - - def ascendc_copy_func(self: any): - os.chdir(self.in_path) - framework_catalog = os.listdir("framework") - for catalog_file in framework_catalog: - if ( - catalog_file == "tf_plugin" - or catalog_file == "caffe_plugin" - or catalog_file == "onnx_plugin" - ): - source_dir = "op_kernel/tbe/op_info_cfg/ai_core" - dst_dir = os.path.join(self.copy_path, "framework", self.framework_type) - self.ascendc_copy_file(source_dir, dst_dir) - source_dir = os.path.join("framework", catalog_file) - dst_dir = os.path.join(self.copy_path, "framework", self.framework_type) - self.ascendc_copy_file(source_dir, dst_dir) - source_dir = "op_kernel/tbe/op_info_cfg/ai_core" - dst_dir = os.path.join(self.copy_path, "op_impl/ai_core/tbe/config") - self.ascendc_copy_dir(source_dir, dst_dir) - source_dir = "op_kernel/binary/dynamic" - dst_dir = os.path.join( - self.copy_path, "op_impl/ai_core/tbe", self.vendor_name + "_impl", "dynamic" - ) - self.ascendc_copy_file(source_dir, dst_dir) - for compute_unit in self.op_soc_ver: - source_dir = os.path.join("op_kernel/binary", compute_unit) - dst_dir = os.path.join( - self.copy_path, "op_impl/ai_core/tbe/kernel", compute_unit - ) - self.ascendc_copy_dir(source_dir, dst_dir) - source_dir = "op_kernel/binary/config" - dst_dir = os.path.join(self.copy_path, "op_impl/ai_core/tbe/kernel/config") - self.ascendc_copy_dir(source_dir, dst_dir) - so_file = "op_impl/ai_core/tbe/op_master_device/lib/libcust_opmaster.so" - if os.path.exists(so_file): - dst_dir = os.path.join( - self.copy_path, "op_impl/ai_core/tbe/op_master_device/lib" - ) - os.makedirs(dst_dir, exist_ok=True) - shutil.copy(so_file, dst_dir) - - -def args_parse(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", "--input-path", nargs="?", help="Input path of compile result." - ) - parser.add_argument( - "-c", "--copy-path", nargs="?", help="Copy path of compile result." - ) - parser.add_argument( - "-o", "--output-path", nargs="?", help="Output path of compile result." - ) - parser.add_argument("-n", "--vendor-name", nargs="?", help="Vendor name.") - parser.add_argument("-u", "--compute-unit", nargs="?", help="Compute unit.") - parser.add_argument( - "-t", "--framework-type", nargs="?", help="Framework type, eg:tensorflow." - ) - parser.add_argument( - "-p", - "--platform", - nargs="?", - default=None, - help="target platform is x86_64 or aarch64.", - ) - return parser.parse_args() - - -if __name__ == "__main__": - args = args_parse() - kernel_packer = PackKernel(args) - kernel_packer.ascendc_copy_func() - kernel_packer.ascendc_get_config() - kernel_packer.ascendc_pack_kernel() - kernel_packer.ascendc_gen_header() - kernel_packer.ascendc_gen_lib() diff --git a/csrc/deepep/ops2/cmake/util/ascendc_replay_build.py b/csrc/deepep/ops2/cmake/util/ascendc_replay_build.py deleted file mode 100755 index e07545f5c..000000000 --- a/csrc/deepep/ops2/cmake/util/ascendc_replay_build.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import sys - -import const_var -import opdesc_parser -import replay_codegen -from replay_codegen import ReplayCodeGenParams - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class ReplayBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - - def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): - if not self.op_replay_flag: - print("{} replay not enabled".format(self.op_type)) - return - argn = len(self.input_name) + len(self.output_name) + 1 - if self.op_replay_batch: - print("{} replay in batch mode".format(self.op_type)) - else: - print("{} replay in normal mode".format(self.op_type)) - if impl_path.endswith("op_kernel"): - implf = os.path.join(impl_path, self.op_file + ".cpp") - tiling_file = os.path.join( - impl_path, "../op_host", self.op_file + "_tiling.h" - ) - else: - if self.dynamic_shape: - dyn_path = "dynamic" - else: - dyn_path = "" - implf = os.path.join(impl_path, dyn_path, self.op_file + ".cpp") - tiling_file = os.path.join( - impl_path, "../../op_tiling", self.op_file + "_tiling.h" - ) - rep_conf = replay_codegen.ReplayCodeGen( - ReplayCodeGenParams( - self.op_type, - implf, - tiling_file, - self.op_file, - self.op_intf, - argn, - self.op_replay_batch, - self.max_block_dim, - self.max_shape_size, - ) - ) - rep_conf.set_batch(self.op_replay_batch) - rep_conf.set_outdir(out_path) - rep_conf.gen_replay(ops_product) - - -def gen_replay( - cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None -): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(";") - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(";") - op_descs = opdesc_parser.get_op_desc( - cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops - ) - for op_desc in op_descs: - op_desc.gen_replay_source( - dirs.get(const_var.CFG_IMPL_DIR), - dirs.get(const_var.CFG_OUT_DIR), - ops_product, - ) - - -if __name__ == "__main__": - if len(sys.argv) <= 6: - raise RuntimeError("arguments must greater than 6") - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] - rep_dir = {} - rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] - rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] - gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/csrc/deepep/ops2/cmake/util/batch_replay_impl.temp b/csrc/deepep/ops2/cmake/util/batch_replay_impl.temp deleted file mode 100644 index 0e8834664..000000000 --- a/csrc/deepep/ops2/cmake/util/batch_replay_impl.temp +++ /dev/null @@ -1,117 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -#include - -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], - int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N]; - int len[KERNEL_N]; - block_idx = 0; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - //__OP_SET_KERNEL__ - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, true); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[i] = (char *)pos; - len[i] = CodeLen(); - pos += len[i]; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/csrc/deepep/ops2/cmake/util/code_channel_infer.py b/csrc/deepep/ops2/cmake/util/code_channel_infer.py deleted file mode 100755 index c9042d4d3..000000000 --- a/csrc/deepep/ops2/cmake/util/code_channel_infer.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import collections -import copy -import ctypes -import os -import shutil -import stat -import subprocess - -"""CODE_* is used to cube/vector api is called in operator code -CODE_MIX means both cube and vector api is called -CODE_CUBE means only cube api is called -CODE_VEC means only vector api is called -""" -CODE_MIX = 0 -CODE_CUBE = 1 -CODE_VEC = 2 - - -def _is_v220(op_product: str): - """return if current soc version is V220 - - Returns: - res: True means V220 - """ - if op_product == "ascend910_93" or op_product == "ascend910b": - return True - return False - - -InfoCodeChanelParams = collections.namedtuple( - "InfoCodeChanelParams", - [ - "src_file", - "tiling_header", - "kernel_name", - "outdir", - "op_product", - "compile_options", - ], -) - - -def infer_code_channel(params: InfoCodeChanelParams): - """get code channel for v220, return CODE_MIX if soc version is not V220 - - Args: - src_file (str): AscendC operator code file - src_file (str): AscendC operator tiling header file - kernel_name (str): kernel function name - optype (str): operator type - compile_options (list): compile options for bisheng cmd - - Raises: - Exception: if not exist L1/L0/UB if code, it's not a aicore code - - Returns: - res (int): CODE_MIX/CODE_CUBE/CODE_VEC - """ - if not _is_v220(params.op_product): - return CODE_MIX - return CODE_VEC diff --git a/csrc/deepep/ops2/cmake/util/const_var.py b/csrc/deepep/ops2/cmake/util/const_var.py deleted file mode 100755 index bc8f33bf9..000000000 --- a/csrc/deepep/ops2/cmake/util/const_var.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import stat - -REPLAY_BATCH = "batch" -REPLAY_ITERATE = "iterate" -CFG_IMPL_DIR = "impl_dir" -CFG_OUT_DIR = "out_dir" -AUTO_GEN_DIR = "auto_gen_dir" -WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC -WMODES = stat.S_IWUSR | stat.S_IRUSR -SOC_MAP_EXT = { - "ascend310p": "Ascend310P3", - "ascend310b": "Ascend310B1", - "ascend910": "Ascend910A", - "ascend910b": "Ascend910B1", - "ascend910_93": "Ascend910_9391", - "ascend610lite": "Ascend610Lite", -} -BIN_CMD = "opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ ---output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n" -SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" -SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" -SRC_ENV = """ -while true; do - case "$1" in - --kernel-src=*) - export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) - shift - ;; - -*) - shift - ;; - *) - break - ;; - esac -done -""" -CHK_CMD = """ -if ! test -f $2/{res_file} ; then - echo "$2/{res_file} not generated!" - exit 1 -fi -""" -ATTR_DEF_VAL = { - "str": "", - "int": 0, - "float": 0.0, - "bool": False, - "list_bool": [], - "list_int": [], - "list_float": [], - "list_list_int": [[]], -} - - -def conv_soc_ver(ver: str): - return SOC_MAP_EXT.get(ver) diff --git a/csrc/deepep/ops2/cmake/util/gen_impl_and_mrege_json.sh b/csrc/deepep/ops2/cmake/util/gen_impl_and_mrege_json.sh deleted file mode 100755 index 93d7ec845..000000000 --- a/csrc/deepep/ops2/cmake/util/gen_impl_and_mrege_json.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No project path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -# copy aicpu kernel so operators -if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then - cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl - rm -rf ${project_path}/cpukernel/aicpu_kernel_lib -fi diff --git a/csrc/deepep/ops2/cmake/util/gen_ops_filter.sh b/csrc/deepep/ops2/cmake/util/gen_ops_filter.sh deleted file mode 100755 index b06a4e9fc..000000000 --- a/csrc/deepep/ops2/cmake/util/gen_ops_filter.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -if [[ -z "$1" ]]; then - echo -e "[ERROR] No source dir provided" - exit 1 -fi - -if [[ -z "$2" ]]; then - echo -e "[ERROR] No destination dir provided" - exit 1 -fi - -src=$1 -dest_file=$2/npu_supported_ops.json - -if [ -f "$dest_file" ];then - chmod u+w $dest_file -fi - -echo $* - -add_ops() { - name=$1 - isHeavy=$2 - file=$3 - grep -w "\"$name\"" ${file} >/dev/null - if [ $? == 0 ];then - return - fi - echo " \"${name}\": {" >> ${file} - echo " \"isGray\": false," >> ${file} - echo " \"isHeavy\": ${isHeavy}" >> ${file} - echo " }," >> ${file} -} - -echo "{" > ${dest_file} -ini_files=$(find ${src} -name "*.ini") -for file in ${ini_files} ; do - name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') - grep 'heavyOp.flag' ${file} >/dev/null - if [ $? == 0 ];then - isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') - else - isHeavy="false" - fi - for op in ${name} ; do - add_ops ${op} "false" ${dest_file} - done -done -echo "}" >> ${dest_file} -file_count=$(cat ${dest_file} | wc -l) -line=$(($file_count-1)) -sed -i "${line}{s/,//g}" ${dest_file} - -chmod 640 "${dest_file}" -echo -e "[INFO] Succeed generated ${dest_file}" - -exit 0 diff --git a/csrc/deepep/ops2/cmake/util/gen_version_info.sh b/csrc/deepep/ops2/cmake/util/gen_version_info.sh deleted file mode 100755 index 8468f949a..000000000 --- a/csrc/deepep/ops2/cmake/util/gen_version_info.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -ascend_install_dir=$1 -gen_file_dir=$2 - -# create version.info -compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') -echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info diff --git a/csrc/deepep/ops2/cmake/util/insert_op_info.py b/csrc/deepep/ops2/cmake/util/insert_op_info.py deleted file mode 100644 index ca7562e7a..000000000 --- a/csrc/deepep/ops2/cmake/util/insert_op_info.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- - -import json -import os -import stat -import sys - -import const_var - -if __name__ == "__main__": - if len(sys.argv) != 3: - print(sys.argv) - print("argv error, inert_op_info.py your_op_file lib_op_file") - sys.exit(2) - - with open(sys.argv[1], "r") as load_f: - insert_operator = json.load(load_f) - - all_operators = {} - if os.path.exists(sys.argv[2]): - if os.path.getsize(sys.argv[2]) != 0: - with open(sys.argv[2], "r") as load_f: - all_operators = json.load(load_f) - - for k in insert_operator.keys(): - if k in all_operators.keys(): - print("replace op:[", k, "] success") - else: - print("insert op:[", k, "] success") - all_operators[k] = insert_operator[k] - - with os.fdopen( - os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), "w" - ) as json_file: - json_file.write(json.dumps(all_operators, indent=4)) diff --git a/csrc/deepep/ops2/cmake/util/insert_simplified_keys.py b/csrc/deepep/ops2/cmake/util/insert_simplified_keys.py deleted file mode 100755 index 599ebe974..000000000 --- a/csrc/deepep/ops2/cmake/util/insert_simplified_keys.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import argparse -import glob -import json -import os -import re -import sys - -import const_var - -DATA_TPYE_DICT = { - "float32": 0, - "float16": 1, - "int8": 2, - "int16": 6, - "uint16": 7, - "uint8": 4, - "int32": 3, - "int64": 9, - "uint32": 8, - "uint64": 10, - "bool": 12, - "double": 11, - "string": 13, - "dual": 14, - "dual": 15, - "complex64": 16, - "complex128": 17, - "qint8": 18, - "qint16": 19, - "qint32": 20, - "quint8": 21, - "quint16": 22, - "resource": 23, - "string": 24, - "dual": 25, - "variant": 26, - "bf16": 27, - "bfloat16": 27, - "undefined": 28, - "int4": 29, - "uint1": 30, - "int2": 31, -} - -FORMAT_DICT = { - "NCHW": 0, - "NHWC": 1, - "ND": 2, - "NC1HWC0": 3, - "FRACTAL_Z": 4, - "NC1C0HWPAD": 5, - "NHWC1C0": 6, - "FSR_NCHW": 7, - "FRACTAL_DECONV": 8, - "C1HWNC0": 9, - "FRACTAL_DECONV_TRANSPOSE": 10, - "FRACTAL_DECONV_SP_STRIDE_TRANS": 11, - "NC1HWC0_C04": 12, - "FRACTAL_Z_C04": 13, - "CHWN": 14, - "FRACTAL_DECONV_SP_STRIDE8_TRANS": 15, - "HWCN": 16, - "NC1KHKWHWC0": 17, - "BN_WEIGHT": 18, - "FILTER_HWCK": 19, - "HASHTABLE_LOOKUP_LOOKUPS": 20, - "HASHTABLE_LOOKUP_KEYS": 21, - "HASHTABLE_LOOKUP_VALUE": 22, - "HASHTABLE_LOOKUP_OUTPUT": 23, - "HASHTABLE_LOOKUP_HITS": 24, - "C1HWNCoC0": 25, - "MD": 26, - "NDHWC": 27, - "FRACTAL_ZZ": 28, - "FRACTAL_NZ": 29, - "NCDHW": 30, - "DHWCN": 31, - "NDC1HWC0": 32, - "FRACTAL_Z_3D": 33, - "CN": 34, - "NC": 35, - "DHWNC": 36, - "FRACTAL_Z_3D_TRANSPOSE": 37, - "FRACTAL_ZN_LSTM": 38, - "FRACTAL_Z_G": 39, - "RESERVED": 40, - "ALL": 41, - "NULL": 42, - "ND_RNN_BIAS": 43, - "FRACTAL_ZN_RNN": 44, - "NYUV": 45, - "NYUV_A": 46, -} - - -def load_json(json_file: str): - with open(json_file, encoding="utf-8") as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, "**/*.{}".format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return all_suffix_files - - -def get_deterministic_value(support_info): - deterministic_key = "deterministic" - if deterministic_key not in support_info: - return 0 - deterministic_value = support_info.get(deterministic_key) - if deterministic_value == "true": - return 1 - else: - return 0 - - -def get_precision_value(support_info): - precision_key = "implMode" - precision_value = support_info.get(precision_key) - if precision_value == "high_performance": - _value = 1 - elif precision_value == "high_precision": - _value = 2 - else: - _value = 0 - return _value - - -def get_overflow_value(support_info): - return 0 - - -def get_parameters(info): - if info: - if "dtype" in info: - data_type = info["dtype"] - data_type_value = DATA_TPYE_DICT.get(data_type) - else: - data_type_value = 0 - if "format" in info: - _format = info["format"] - _format_value = FORMAT_DICT.get(_format) - else: - _format_value = 0 - else: - data_type_value = 0 - _format_value = 0 - return str(data_type_value), str(_format_value) - - -def get_dynamic_parameters(info): - # 动态输入时只需获取第一个参数 - return get_parameters(info[0]) - - -def get_all_parameters(support_info, _type): - result_list = list() - info_lists = support_info.get(_type) - if info_lists: - for _info in info_lists: - # 输入为列表时是动态输入 - if isinstance(_info, (list, tuple)): - data_type_value, _format_value = get_dynamic_parameters(_info) - else: - data_type_value, _format_value = get_parameters(_info) - result_list.append("{},{}".format(data_type_value, _format_value)) - return result_list - - -def get_all_input_parameters(support_info): - result = get_all_parameters(support_info, "inputs") - return "/".join(result) - - -def insert_content_into_file(input_file, content): - with open(input_file, "r+") as file: - lines = file.readlines() - for index, line in enumerate(lines): - match_result = re.search(r'"staticKey":', line) - if match_result: - count = len(line) - len(line.lstrip()) - new_content = "{}{}".format(" " * count, content) - # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 - lines.insert(index, new_content) - break - file.seek(0) - file.write("".join(lines)) - - -def insert_simplified_keys(json_file): - contents = load_json(json_file) - # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 - if ("binFileName" not in contents) or ("supportInfo" not in contents): - return - support_info = contents.get("supportInfo") - bin_file_name = contents.get("binFileName") - # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 - if "simplifiedKey" in support_info: - return - op_type = bin_file_name.split("_")[0] - deterministic = str(get_deterministic_value(support_info)) - precision = str(get_precision_value(support_info)) - overflow = str(get_overflow_value(support_info)) - input_parameters = get_all_input_parameters(support_info) - key = "{}/d={},p={},o={}/{}/".format( - op_type, deterministic, precision, overflow, input_parameters - ) - result = '"simplifiedKey": "' + key + '",\n' - insert_content_into_file(json_file, result) - - -def insert_all_simplified_keys(root_dir): - suffix = "json" - all_json_files = get_specified_suffix_file(root_dir, suffix) - for _json in all_json_files: - insert_simplified_keys(_json) - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-p", - "--path", - nargs="?", - required=True, - help="Parse the path of the json file.", - ) - return parser.parse_args() - - -def main(): - args = args_prase() - insert_all_simplified_keys(args.path) - - -if __name__ == "__main__": - main() diff --git a/csrc/deepep/ops2/cmake/util/kernel_entry.py b/csrc/deepep/ops2/cmake/util/kernel_entry.py deleted file mode 100755 index 255266fd9..000000000 --- a/csrc/deepep/ops2/cmake/util/kernel_entry.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - - -def gen_fun_def(title, kernel, argn, arg_type, arg_name): - entry = [] - entry.append(title) - entry.append(kernel) - entry.append("(") - args = [] - for i in range(0, argn): - args.append(arg_type + " " + arg_name + str(i)) - entry.append(", ".join(args)) - entry.append(")") - return " ".join(entry) - - -def gen_batch_kernel_body(fname, argn, arg_name): - body = [] - body.append("{") - fun = [] - fun.append(fname) - fun.append("(") - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(", ".join(args)) - fun.append(");") - body.append(" ".join(fun)) - body.append("}") - return "\n".join(body) - - -def gen_mc_kernel_body(kn, argn, arg_name, blknum): - body = [] - body.append("{") - body.append(" switch(block_idx) {") - for blk in range(0, blknum): - fun = [] - fun.append("{}_blk{:02d}".format(kn, blk)) - fun.append("(") - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(", ".join(args)) - fun.append(")") - body.append(" case {}: {}; break;".format(blk, " ".join(fun))) - body.append(" default: break;") - body.append(" }") - body.append("}") - return "\n".join(body) - - -def gen_proc_body(argn, arg_name): - body = [] - body.append("{") - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - body.append("uint64_t __x = (uint64_t)" + " + (uint64_t)".join(args) + ";") - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append("}") - return "\n".join(body) - - -def batch_code_gen(kn, argn, argt): - codes = [] - kernel_name = kn - proc_name = kernel_name + "_percore" - arg_num = int(argn) - data_type = argt - arg_type = "__gm__ " + data_type + "* __restrict__" - arg_name = "arg" - kernel_title = 'extern "C" __global__ __aicore__ void' - proc_title = 'extern "C" __attribute__((noinline)) __aicore__ void' - codes.append("#ifndef __aicore__") - codes.append("#define __aicore__ [aicore]") - codes.append("#endif") - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ";") - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return "\n".join(codes) + "\n" - - -def mc_code_gen(kn, argn, argt, blknum): - codes = [] - kernel_name = kn - core_num = int(blknum) - arg_num = int(argn) - data_type = argt - arg_type = "__gm__ " + data_type + "* __restrict__" - arg_name = "arg" - kernel_title = 'extern "C" __global__ __aicore__ void' - proc_title = 'extern "C" __attribute__((noinline)) __aicore__ void' - codes.append("#ifndef __aicore__") - codes.append("#define __aicore__ [aicore]") - codes.append("#endif") - for i in range(0, core_num): - proc_name = "{}_blk{:02d}".format(kernel_name, i) - codes.append( - gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ";" - ) - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) - for i in range(0, core_num): - proc_name = "{}_blk{:02d}".format(kernel_name, i) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return "\n".join(codes) + "\n" diff --git a/csrc/deepep/ops2/cmake/util/kernel_impl.temp b/csrc/deepep/ops2/cmake/util/kernel_impl.temp deleted file mode 100644 index 5079a1043..000000000 --- a/csrc/deepep/ops2/cmake/util/kernel_impl.temp +++ /dev/null @@ -1,10 +0,0 @@ -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#define __ASCENDC_REPLAY_CODE__ -#include "__CCE_FILE__" diff --git a/csrc/deepep/ops2/cmake/util/makeself/COPYING b/csrc/deepep/ops2/cmake/util/makeself/COPYING deleted file mode 100644 index d159169d1..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/csrc/deepep/ops2/cmake/util/makeself/README.md b/csrc/deepep/ops2/cmake/util/makeself/README.md deleted file mode 100644 index 9d3d4b86f..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/README.md +++ /dev/null @@ -1,246 +0,0 @@ -[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) -![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) - -# makeself - Make self-extractable archives on Unix - -[makeself.sh][1] is a small shell script that generates a self-extractable -compressed tar archive from a directory. The resulting file appears as a shell script -(many of those have a **.run** suffix), and can be launched as is. The archive -will then uncompress itself to a temporary directory and an optional arbitrary -command will be executed (for example an installation script). This is pretty -similar to archives generated with WinZip Self-Extractor in the Windows world. -Makeself archives also include checksums for integrity self-validation (CRC -and/or MD5/SHA256 checksums). - -The makeself.sh script itself is used only to create the archives from a -directory of files. The resultant archive is actually a compressed (using -gzip, bzip2, or compress) TAR archive, with a small shell script stub at the -beginning. This small stub performs all the steps of extracting the files, -running the embedded command, and removing the temporary files when done. -All the user has to do to install the software contained in such an -archive is to "run" the archive, i.e **sh nice-software.run**. I recommend -using the ".run" (which was introduced by some Makeself archives released by -Loki Software) or ".sh" suffix for such archives not to confuse the users, -so that they will know they are actually shell scripts (with quite a lot of binary data -attached to them though!). - -I am trying to keep the code of this script as portable as possible, i.e it is -not relying on any bash-specific features and only calls commands that are -installed on any functioning UNIX-compatible system. This script as well as -the archives it generates should run on any Unix flavor, with any compatible -Bourne shell, provided of course that the compression programs are available. - -As of version 2.1, Makeself has been rewritten and tested on the following -platforms : - - * Linux (all distributions) - * Sun Solaris (8 and above) - * HP-UX (tested on 11.0 and 11i on HPPA RISC) - * SCO OpenUnix and OpenServer - * IBM AIX 5.1L - * macOS (Darwin) - * SGI IRIX 6.5 - * FreeBSD - * UnicOS / Cray - * Cygwin (Windows) - -If you successfully run Makeself and/or archives created with it on another -system, then please [let me know][2]! - -Examples of publicly available archives made using makeself are : - - * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; - * All game patches released by [Loki Software][4] for the Linux version of popular games ; - * The [nVidia drivers][5] for Linux - * The installer for the Linux version of [Google Earth][6] - * The [VirtualBox][7] installers for Linux - * The [Makeself][1] distribution itself ;-) - * and countless others... - -**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : - -`AddType application/x-makeself .run` - -**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : - -`export _POSIX2_VERSION=199209` - -## Usage - -The syntax of makeself is the following: - -``` -makeself.sh [args] archive_dir file_name label startup_script [script_args] -``` - - * _args_ are optional options for Makeself. The available ones are : - - * **`--version`** : Prints the version number on stdout, then exits immediately - * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) - * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. - * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. - * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. - * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. - * **`--pigz`** : Use pigz for compression. - * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). - * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. - * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. - * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. - * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. - * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) - * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. - * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). - * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. - * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. - * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. - * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. - * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. - * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. - * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. - * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. - * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. - * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. - * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. - * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). - * **`--tar-extra opt`** : Append more options to the tar command line. - - For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` - - * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. - * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. - * **`--license`** : Append a license file. - * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. - * **`--help-header file`** : Add a header to the archive's `--help` output. - * `archive_dir` is the name of the directory that contains the files to be archived - * `file_name` is the name of the archive to be created - * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. - * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. - -Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named -**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : - -`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -` - -Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : - -`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` - -Archives generated with Makeself can be passed the following arguments: - - * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. - * **`--verbose`** : Will prompt the user before executing the embedded command - * **`--target dir`** : Allows to extract the archive in an arbitrary place. - * **`--nox11`** : Do not spawn a X11 terminal. - * **`--confirm`** : Prompt the user for confirmation before running the embedded command. - * **`--info`** : Print out general information about the archive (does not extract). - * **`--lsm`** : Print out the LSM entry, if it is present. - * **`--list`** : List the files in the archive. - * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. - * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. - * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. - * **`--noexec`** : Do not run the embedded script after extraction. - * **`--noexec-cleanup`** : Do not run the embedded cleanup script. - * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. - * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. - -Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. - -## Startup Script - -The startup script must be a regular Shell script. - -Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. - -`my-self-extracting-script.sh --fooBarFileParameter foo.bar` - -## Building and Testing - -Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. - -* To make a release: `make` -* To run all tests: `make test` - -## Maven Usage - -Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. - -## License - -Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. - -## Contributing - -I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: - - * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. - * Please explain clearly what the purpose of the patch is, and how you achieved it. - -## Download - -Get the latest official distribution [here][9] (version 2.4.2). - -The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. - -## Version history - - * **v1.0:** Initial public release - * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan - * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. - * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. - * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. - * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. - * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. - * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. - * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. - * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. - * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. - * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. - * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. - * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). - * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) - * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. - * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overridden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. - * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. - * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. - * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. - * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. - * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. - * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) - -## Links - - * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. - * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] - -## Contact - -This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. - -Contributions were included from John C. Quillan, Bjarni R. Einarsson, -Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot -your name, don't hesitate to contact me. - -This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. - -* * * - -[Stephane Peter][2] - - [1]: http://makeself.io/ - [2]: mailto:megastep@megastep.org - [3]: http://www.idsoftware.com/ - [4]: http://www.lokigames.com/products/myth2/updates.php3 - [5]: http://www.nvidia.com/ - [6]: http://earth.google.com/ - [7]: http://www.virtualbox.org/ - [8]: http://www.gnu.org/copyleft/gpl.html - [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run - [10]: https://github.com/megastep/makeself - [11]: https://github.com/megastep/loki_setup/ - [12]: http://www.unrealtournament2003.com/ - [13]: http://www.icculus.org/ - [14]: http://bre.klaki.net/programs/setup.sh/ - [15]: https://stephanepeter.com/ diff --git a/csrc/deepep/ops2/cmake/util/makeself/VERSION b/csrc/deepep/ops2/cmake/util/makeself/VERSION deleted file mode 100644 index 59aa62c1f..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/VERSION +++ /dev/null @@ -1 +0,0 @@ -2.4.5 diff --git a/csrc/deepep/ops2/cmake/util/makeself/make-release.sh b/csrc/deepep/ops2/cmake/util/makeself/make-release.sh deleted file mode 100755 index 65d698f25..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/make-release.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# -# Create a distributable archive of the current version of Makeself - -VER=`cat VERSION` -mkdir -p /tmp/makeself-$VER release -cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ -./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" diff --git a/csrc/deepep/ops2/cmake/util/makeself/makeself-header.sh b/csrc/deepep/ops2/cmake/util/makeself/makeself-header.sh deleted file mode 100644 index 23ffc483e..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/makeself-header.sh +++ /dev/null @@ -1,660 +0,0 @@ -cat << EOF > "$archname" -#!/bin/bash -# This script was generated using Makeself $MS_VERSION -# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) -# 2022.3.19-Modified the MS_Help function and some options -# Huawei Technologies Co., Ltd. - -ORIG_UMASK=\`umask\` - -CRCsum="$CRCsum" -MD5="$MD5sum" -SHA="$SHAsum" -SIGNATURE="$Signature" -TMPROOT=\${TMPDIR:="\$HOME"} -if ! test -d "\$TMPROOT"; then - TMPROOT="\$PWD" -fi -export TMPDIR="\$TMPROOT" -USER_PWD="\$PWD" -if ! test -d "\$USER_PWD"; then - exit 1 -fi -export USER_PWD -ARCHIVE_DIR=\`dirname "\$0"\` -export ARCHIVE_DIR - -name_of_file="\$0 " -pwd_of_file="\$PWD" -label="$LABEL" -script="$SCRIPT" -scriptargs="$SCRIPTARGS" -cleanup_script="${CLEANUP_SCRIPT}" -licensetxt="$LICENSE" -helpheader='$HELPHEADER' -targetdir="$archdirname" -filesizes="$filesizes" -totalsize="$totalsize" -keep="$KEEP" -nooverwrite="$NOOVERWRITE" -quiet="n" -accept="n" -nodiskspace="n" -export_conf="$EXPORT_CONF" -decrypt_cmd="$DECRYPT_CMD" -skip="$SKIP" - -print_cmd_arg="" -if type printf > /dev/null; then - print_cmd="printf" -elif test -x /usr/ucb/echo; then - print_cmd="/usr/ucb/echo" -else - print_cmd="echo" -fi - -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:\$PATH - export PATH -fi - -if test -d /usr/sfw/bin; then - PATH=\$PATH:/usr/sfw/bin - export PATH -fi - -unset CDPATH - -MS_Printf() -{ - \$print_cmd \$print_cmd_arg "\$1" -} - -MS_PrintLicense() -{ - PAGER=\${PAGER:=more} - if test x"\$licensetxt" != x; then - PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` - if test -x "\$PAGER_PATH"; then - echo "\$licensetxt" | \$PAGER - else - echo "\$licensetxt" - fi - if test x"\$accept" != xy; then - while true - do - MS_Printf "Please type y to accept, n otherwise: " - read yn - if test x"\$yn" = xn; then - keep=n - eval \$finish; exit 1 - break; - elif test x"\$yn" = xy; then - break; - fi - done - fi - fi -} - -MS_diskspace() -{ - ( - df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' - ) -} - -MS_dd() -{ - blocks=\`expr \$3 / 1024\` - bytes=\`expr \$3 % 1024\` - # Test for ibs, obs and conv feature - if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then - dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ - { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ - test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null - else - dd if="\$1" bs=\$2 skip=1 2> /dev/null - fi -} - -MS_dd_Progress() -{ - if test x"\$noprogress" = xy; then - MS_dd "\$@" - return \$? - fi - file="\$1" - offset=\$2 - length=\$3 - pos=0 - bsize=4194304 - while test \$bsize -gt \$length; do - bsize=\`expr \$bsize / 4\` - done - blocks=\`expr \$length / \$bsize\` - bytes=\`expr \$length % \$bsize\` - ( - dd ibs=\$offset skip=1 2>/dev/null - pos=\`expr \$pos \+ \$bsize\` - MS_Printf " 0%% " 1>&2 - if test \$blocks -gt 0; then - while test \$pos -le \$length; do - dd bs=\$bsize count=1 2>/dev/null - pcent=\`expr \$length / 100\` - pcent=\`expr \$pos / \$pcent\` - if test \$pcent -lt 100; then - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - if test \$pcent -lt 10; then - MS_Printf " \$pcent%% " 1>&2 - else - MS_Printf " \$pcent%% " 1>&2 - fi - fi - pos=\`expr \$pos \+ \$bsize\` - done - fi - if test \$bytes -gt 0; then - dd bs=\$bytes count=1 2>/dev/null - fi - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - MS_Printf " 100%% " 1>&2 - ) < "\$file" -} - -MS_Help() -{ - cat << EOH >&2 -Usage: \$0 [options] -Options: - --help | -h Print this message - --info Print embedded info : title, default target directory, embedded script ... - --list Print the list of files in the archive - --check Checks integrity and version dependency of the archive - --quiet Quiet install mode, skip human-computer interactions - --nox11 Do not spawn an xterm - --noexec Do not run embedded script - --extract= Extract directly to a target directory (absolute or relative) - Usually used with --noexec to just extract files without running - --tar arg1 [arg2 ...] Access the contents of the archive through the tar command -\${helpheader} -EOH -} - -MS_Verify_Sig() -{ - GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - temp_sig=\`mktemp -t XXXXX\` - echo \$SIGNATURE | base64 --decode > "\$temp_sig" - gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` - gpg_res=\$? - rm -f "\$temp_sig" - if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then - if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then - test x"\$quiet" = xn && echo "GPG signature is good" >&2 - else - echo "GPG Signature key does not match" >&2 - exit 2 - fi - else - test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 - exit 2 - fi -} - -MS_Check() -{ - OLD_PATH="\$PATH" - PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` - PATH="\$OLD_PATH" - - SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` - test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` - - if test x"\$quiet" = xn; then - MS_Printf "Verifying archive integrity..." - fi - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - fsize=\`cat "\$1" | wc -c | tr -d " "\` - if test \$totalsize -ne \`expr \$fsize - \$offset\`; then - echo " Unexpected archive size." >&2 - exit 2 - fi - verb=\$2 - i=1 - for s in \$filesizes - do - crc=\`echo \$CRCsum | cut -d" " -f\$i\` - if test -x "\$SHA_PATH"; then - if test x"\`basename \$SHA_PATH\`" = xshasum; then - SHA_ARG="-a 256" - fi - sha=\`echo \$SHA | cut -d" " -f\$i\` - if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 - else - shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; - if test x"\$shasum" != x"\$sha"; then - echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " SHA256 checksums are OK." >&2 - fi - crc="0000000000"; - fi - fi - if test -x "\$MD5_PATH"; then - if test x"\`basename \$MD5_PATH\`" = xdigest; then - MD5_ARG="-a md5" - fi - md5=\`echo \$MD5 | cut -d" " -f\$i\` - if test x"\$md5" = x00000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 - else - md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; - if test x"\$md5sum" != x"\$md5"; then - echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " MD5 checksums are OK." >&2 - fi - crc="0000000000"; verb=n - fi - fi - if test x"\$crc" = x0000000000; then - test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 - else - sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` - if test x"\$sum1" != x"\$crc"; then - echo "Error in checksums: \$sum1 is different from \$crc" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " CRC checksums are OK." >&2 - fi - fi - i=\`expr \$i + 1\` - offset=\`expr \$offset + \$s\` - done - if test x"\$quiet" = xn; then - echo " All good." - fi -} - -MS_Decompress() -{ - if test x"\$decrypt_cmd" != x""; then - { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" - else - eval "$GUNZIP_CMD" - fi - - if test \$? -ne 0; then - echo " ... Decompression failed." >&2 - fi -} - -UnTAR() -{ - if test x"\$quiet" = xn; then - tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } - else - tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } - fi -} - -MS_exec_cleanup() { - if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then - cleanup=n - cd "\$tmpdir" - eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" - fi -} - -MS_cleanup() -{ - echo 'Signal caught, cleaning up' >&2 - MS_exec_cleanup - cd "\$TMPROOT" - rm -rf "\$tmpdir" - eval \$finish; exit 15 -} - -Script_Args_Check() -{ - script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) - arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) - - for arg in \${script_supported_args}; - do - if test x"\$arg_to_test" = x"\$arg" ;then - return - fi - done - - MS_Help - exit 1 -} - -finish=true -xterm_loop= -noprogress=$NOPROGRESS -nox11=$NOX11 -copy=$COPY -ownership=$OWNERSHIP -verbose=n -cleanup=y -cleanupargs= -sig_key= - -initargs="\$@" - -while [ -n "\$*" ] -do - case "\$1" in - -h | --help) - MS_Help - exit 0 - ;; - -q | --quiet) - quiet=y - noprogress=y - shift - ;; - --info) - echo Identification: "\$label" - echo Target directory: "\$targetdir" - echo Uncompressed size: $USIZE KB - echo Compression: $COMPRESS - if test x"$ENCRYPT" != x""; then - echo Encryption: $ENCRYPT - fi - echo Date of packaging: $DATE - echo Built with Makeself version $MS_VERSION - echo Build command was: "$MS_COMMAND" - if test x"\$script" != x; then - echo Script run after extraction: - echo " " \$script \$scriptargs - fi - if test x"$copy" = xcopy; then - echo "Archive will copy itself to a temporary location" - fi - if test x"$NEED_ROOT" = xy; then - echo "Root permissions required for extraction" - fi - if test x"$KEEP" = xy; then - echo "directory \$targetdir is permanent" - else - echo "\$targetdir will be removed after extraction" - fi - exit 0 - ;; - --list) - echo Target directory: \$targetdir - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --tar) - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - arg1="\$2" - shift 2 || { MS_Help; exit 1; } - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --check) - MS_Check "\$0" y - scriptargs="\$scriptargs \$1" - shift - ;; - --noexec) - script="" - cleanup_script="" - shift - ;; - --extract=*) - keep=y - targetdir=\`echo \$1 | cut -d"=" -f2 \` - if ! shift; then MS_Help; exit 1; fi - ;; - --nox11) - nox11=y - shift - ;; - --xwin) - if test "$NOWAIT" = n; then - finish="echo Press Return to close this window...; read junk" - fi - xterm_loop=1 - shift - ;; - --phase2) - copy=phase2 - shift - ;; - --repack | --repack-path=*) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - if [[ ! "\$1" =~ ^-.* ]]; then - scriptargs="\$scriptargs '\$1'" - shift - fi - ;; - *) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - ;; - esac -done - -quiet_para="" -if test x"\$quiet" = xy; then - quiet_para="--quiet " -fi -scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" - -if test x"\$quiet" = xy -a x"\$verbose" = xy; then - echo Cannot be verbose and quiet at the same time. >&2 - exit 1 -fi - -if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then - echo "Administrative privileges required for this archive (use su or sudo)" >&2 - exit 1 -fi - -if test x"\$copy" \!= xphase2; then - MS_PrintLicense -fi - -case "\$copy" in -copy) - tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ - mkdir "\$tmpdir" || { - echo "Could not create temporary directory \$tmpdir" >&2 - exit 1 - } - SCRIPT_COPY="\$tmpdir/makeself" - echo "Copying to a temporary location..." >&2 - cp "\$0" "\$SCRIPT_COPY" - chmod +x "\$SCRIPT_COPY" - cd "\$TMPROOT" - exec "\$SCRIPT_COPY" --phase2 -- \$initargs - ;; -phase2) - finish="\$finish ; rm -rf \`dirname \$0\`" - ;; -esac - -if test x"\$nox11" = xn; then - if tty -s; then # Do we have a terminal? - : - else - if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? - if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable - GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" - for a in \$GUESS_XTERMS; do - if type \$a >/dev/null 2>&1; then - XTERM=\$a - break - fi - done - chmod a+x \$0 || echo Please add execution rights on \$0 - if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! - exec \$XTERM -e "\$0 --xwin \$initargs" - else - exec \$XTERM -e "./\$0 --xwin \$initargs" - fi - fi - fi - fi -fi - -if test x"\$targetdir" = x.; then - tmpdir="." -else - if test x"\$keep" = xy; then - if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then - echo "Target directory \$targetdir already exists, aborting." >&2 - exit 1 - fi - if test x"\$quiet" = xn; then - echo "Creating directory \$targetdir" >&2 - fi - tmpdir="\$targetdir" - dashp="-p" - else - tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" - dashp="" - fi - mkdir \$dashp "\$tmpdir" || { - echo 'Cannot create target directory' \$tmpdir >&2 - echo 'You should try option --extract=' >&2 - eval \$finish - exit 1 - } -fi - -location="\`pwd\`" -if test x"\$SETUP_NOCHECK" != x1; then - MS_Check "\$0" -fi -offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - -if test x"\$verbose" = xy; then - MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " - read yn - if test x"\$yn" = xn; then - eval \$finish; exit 1 - fi -fi - -if test x"\$quiet" = xn; then - # Decrypting with openssl will ask for password, - # the prompt needs to start on new line - if test x"$ENCRYPT" = x"openssl"; then - echo "Decrypting and uncompressing \$label..." - else - MS_Printf "Uncompressing \$label" - fi -fi -res=3 -if test x"\$keep" = xn; then - trap MS_cleanup 1 2 3 15 -fi - -if test x"\$nodiskspace" = xn; then - leftspace=\`MS_diskspace "\$tmpdir"\` - if test -n "\$leftspace"; then - if test "\$leftspace" -lt $USIZE; then - echo - echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 - if test x"\$keep" = xn; then - echo "Consider setting TMPDIR to a directory with more free space." - fi - eval \$finish; exit 1 - fi - fi -fi - -for s in \$filesizes -do - if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then - if test x"\$ownership" = xy; then - (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) - fi - else - echo >&2 - echo "Unable to decompress \$0" >&2 - eval \$finish; exit 1 - fi - offset=\`expr \$offset + \$s\` -done -if test x"\$quiet" = xn; then - echo -fi - -cd "\$tmpdir" -res=0 -if test x"\$script" != x; then - if test x"\$export_conf" = x"y"; then - MS_BUNDLE="\$0" - MS_LABEL="\$label" - MS_SCRIPT="\$script" - MS_SCRIPTARGS="\$scriptargs" - MS_ARCHDIRNAME="\$archdirname" - MS_KEEP="\$KEEP" - MS_NOOVERWRITE="\$NOOVERWRITE" - MS_COMPRESS="\$COMPRESS" - MS_CLEANUP="\$cleanup" - export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS - export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS - fi - - if test x"\$verbose" = x"y"; then - yn="x" - while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN - do - MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " - read yn - if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; - elif test x"\$yn" = xn -o x"\$yn" = xN; then - echo "Unable to decompress \$script ,because of aborting! ";res=\$? - else - echo "Input value is unacceptable,please try again." - fi - done - else - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? - fi - if test "\$res" -ne 0; then - test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 - fi -fi - -MS_exec_cleanup - -if test x"\$keep" = xn; then - cd "\$TMPROOT" - rm -rf "\$tmpdir" -fi -eval \$finish; exit \$res -EOF diff --git a/csrc/deepep/ops2/cmake/util/makeself/makeself.1 b/csrc/deepep/ops2/cmake/util/makeself/makeself.1 deleted file mode 100644 index 81bf6e4ff..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/makeself.1 +++ /dev/null @@ -1,110 +0,0 @@ -.TH "MAKESELF" "1" "2.4.5" -.SH "NAME" -makeself \- An utility to generate self-extractable archives. -.SH "SYNTAX" -.B makeself [\fIoptions\fP] archive_dir file_name label -.B [\fIstartup_script\fP] [\fIargs\fP] -.SH "DESCRIPTION" -This program is a free (GPL) utility designed to create self-extractable -archives from a directory. -.SH "OPTIONS" -The following options are supported. -.TP 15 -.B -v, --version -Prints out the makeself version number and exits. -.TP -.B -h, --help -Print out help information. -.TP -.B --tar-quietly -Suppress verbose output from the tar command -.TP -.B --quiet -Do not print any messages other than errors -.TP -.B --gzip -Compress using gzip (default if detected). -.TP -.B --bzip2 -Compress using bzip2. -.TP -.B --pbzip2 -Compress using pbzip2. -.TP -.B --xz -Compress using xz. -.TP -.B --lzo -Compress using lzop. -.TP -.B --lz4 -Compress using lz4. -.TP -.B --compress -Compress using the UNIX 'compress' command. -.TP -.B --nocomp -Do not compress the data. -.TP -.B --complevel lvl -Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 -.TP -.B --notemp -The archive will create archive_dir in the current directory and -uncompress in ./archive_dir. -.TP -.B --copy -Upon extraction, the archive will first copy itself to a temporary directory. -.TP -.B --append -Append more files to an existing makeself archive. The label and startup scripts will then be ignored. -.TP -.B --current -Files will be extracted to the current directory. Both --current and --target dir imply --notemp. -.TP -.B --target dir -Extract directly to a target directory. Directory path can be either absolute or relative. -.TP -.B --header file -Specify location of the header script. -.TP -.B --cleanup file -Specify a cleanup script that executes on interrupt and when finished successfully. -.TP -.B --follow -Follow the symlinks in the archive. -.TP -.B --noprogress -Do not show the progress during the decompression. -.TP -.B --nox11 -Disable automatic spawn of an xterm if running in X11. -.TP -.B --nowait -Do not wait for user input after executing embedded program from an xterm. -.TP -.B --nomd5 -Do not create a MD5 checksum for the archive. -.TP -.B --nocrc -Do not create a CRC32 checksum for the archive. -.TP -.B --lsm file -LSM file describing the package. -.B --packaging-date date -Use provided string as the packaging date instead of the current date. -.SH "EXAMPLES" -Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, -and he wants to generate a self-extracting package named mysoft.sh, which will launch -the "setup" script initially stored in /home/joe/mysoft: -.TP -makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -.TP -Here is also how I created the makeself.run archive which contains the Makeself distribution: -.TP -makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" -.SH "AUTHORS" -Makeself has been written by Stéphane Peter . -.BR -This man page was originally written by Bartosz Fenski for the -Debian GNU/Linux distribution (but it may be used by others). diff --git a/csrc/deepep/ops2/cmake/util/makeself/makeself.lsm b/csrc/deepep/ops2/cmake/util/makeself/makeself.lsm deleted file mode 100644 index 802cada3a..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/makeself.lsm +++ /dev/null @@ -1,16 +0,0 @@ -Begin3 -Title: makeself.sh -Version: 2.4.5 -Description: makeself.sh is a shell script that generates a self-extractable - tar.gz archive from a directory. The resulting file appears as a shell - script, and can be launched as is. The archive will then uncompress - itself to a temporary directory and an arbitrary command will be - executed (for example an installation script). This is pretty similar - to archives generated with WinZip Self-Extractor in the Windows world. -Keywords: Installation archive tar winzip -Author: Stephane Peter (megastep@megastep.org) -Maintained-by: Stephane Peter (megastep@megastep.org) -Original-site: https://makeself.io/ -Platform: Unix -Copying-policy: GPL -End diff --git a/csrc/deepep/ops2/cmake/util/makeself/makeself.sh b/csrc/deepep/ops2/cmake/util/makeself/makeself.sh deleted file mode 100755 index 60ced4a8a..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/makeself.sh +++ /dev/null @@ -1,822 +0,0 @@ -#!/bin/sh -# -# Makeself version 2.4.x -# by Stephane Peter -# -# Utility to create self-extracting tar.gz archives. -# The resulting archive is a file holding the tar.gz archive with -# a small Shell script stub that uncompresses the archive to a temporary -# directory and then executes a given script from within that directory. -# -# Makeself home page: https://makeself.io/ -# -# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. -# -# Version history : -# - 1.0 : Initial public release -# - 1.1 : The archive can be passed parameters that will be passed on to -# the embedded script, thanks to John C. Quillan -# - 1.2 : Package distribution, bzip2 compression, more command line options, -# support for non-temporary archives. Ideas thanks to Francois Petitjean -# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: -# Support for no compression (--nocomp), script is no longer mandatory, -# automatic launch in an xterm, optional verbose output, and -target -# archive option to indicate where to extract the files. -# - 1.4 : Improved UNIX compatibility (Francois Petitjean) -# Automatic integrity checking, support of LSM files (Francois Petitjean) -# - 1.5 : Many bugfixes. Optionally disable xterm spawning. -# - 1.5.1 : More bugfixes, added archive options -list and -check. -# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big -# archives (Quake III demo) -# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. -# More verbosity in xterms and check for embedded command's return value. -# Bugfix for Debian 2.0 systems that have a different "print" command. -# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. -# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to -# bypass checksum verification of archives. -# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) -# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. -# - 2.0.1 : Added --copy -# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. -# Added --nochown for archives -# Stopped doing redundant checksums when not necessary -# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command -# Cleaned up the code to handle error codes from compress. Simplified the extraction code. -# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. -# - 2.1.3 : Bug fixes with command line when spawning terminals. -# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. -# Added --noexec to prevent execution of embedded scripts. -# Added --nomd5 and --nocrc to avoid creating checksums in archives. -# Added command used to create the archive in --info output. -# Run the embedded script through eval. -# - 2.1.4 : Fixed --info output. -# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) -# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) -# Avoid some race conditions (Ludwig Nussel) -# Unset the $CDPATH variable to avoid problems if it is set. (Debian) -# Better handling of dot files in the archive directory. -# - 2.1.5 : Made the md5sum detection consistent with the header code. -# Check for the presence of the archive directory -# Added --encrypt for symmetric encryption through gpg (Eric Windisch) -# Added support for the digest command on Solaris 10 for MD5 checksums -# Check for available disk space before extracting to the target directory (Andreas Schweitzer) -# Allow extraction to run asynchronously (patch by Peter Hatch) -# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) -# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spinning cursor (Guy Baconniere) -# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) -# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) -# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. -# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) -# - 2.4.0 : Optional support for SHA256 checksums in archives. -# - 2.4.2 : Add support for threads for several compressors. (M. Limber) -# Added zstd support. -# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. -# - 2.4.5 : Added --tar-format to override ustar tar archive format -# -# (C) 1998-2021 by Stephane Peter -# -# This software is released under the terms of the GNU GPL version 2 and above -# Please read the license at http://www.gnu.org/copyleft/gpl.html -# Self-extracting archives created with this script are explicitly NOT released under the term of the GPL -# - -MS_VERSION=2.4.5 -MS_COMMAND="$0" -unset CDPATH - -for f in ${1+"$@"}; do - MS_COMMAND="$MS_COMMAND \\\\ - \\\"$f\\\"" -done - -# For Solaris systems -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:$PATH - export PATH -fi - -# Procedures - -MS_Usage() -{ - echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" - echo "args can be one or more of the following :" - echo " --version | -v : Print out Makeself version number and exit" - echo " --help | -h : Print out this help message" - echo " --tar-quietly : Suppress verbose output from the tar command" - echo " --quiet | -q : Do not print any messages other than errors." - echo " --gzip : Compress using gzip (default if detected)" - echo " --pigz : Compress with pigz" - echo " --zstd : Compress with zstd" - echo " --bzip2 : Compress using bzip2 instead of gzip" - echo " --pbzip2 : Compress using pbzip2 instead of gzip" - echo " --xz : Compress using xz instead of gzip" - echo " --lzo : Compress using lzop instead of gzip" - echo " --lz4 : Compress using lz4 instead of gzip" - echo " --compress : Compress using the UNIX 'compress' command" - echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" - echo " --threads thds : Number of threads to be used by compressors that support parallelization." - echo " Omit to use compressor's default. Most useful (and required) for opting" - echo " into xz's threading, usually with '--threads=0' for all available cores." - echo " pbzip2 and pigz are parallel by default, and setting this value allows" - echo " limiting the number of threads they use." - echo " --base64 : Instead of compressing, encode the data using base64" - echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" - echo " --gpg-asymmetric-encrypt-sign" - echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" - echo " --gpg-extra opt : Append more options to the gpg command line" - echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" - echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" - echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" - echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." - echo " If this option is not supplied, the user will be asked to enter" - echo " encryption password on the current terminal." - echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." - echo " --nochown : Do not give the target folder to the current user (default)" - echo " --chown : Give the target folder to the current user recursively" - echo " --nocomp : Do not compress the data" - echo " --notemp : The archive will create archive_dir in the" - echo " current directory and uncompress in ./archive_dir" - echo " --needroot : Check that the root user is extracting the archive before proceeding" - echo " --copy : Upon extraction, the archive will first copy itself to" - echo " a temporary directory" - echo " --append : Append more files to an existing Makeself archive" - echo " The label and startup scripts will then be ignored" - echo " --target dir : Extract directly to a target directory" - echo " directory path can be either absolute or relative" - echo " --nooverwrite : Do not extract the archive if the specified target directory exists" - echo " --current : Files will be extracted to the current directory" - echo " Both --current and --target imply --notemp" - echo " --tar-format opt : Specify a tar archive format (default is ustar)" - echo " --tar-extra opt : Append more options to the tar command line" - echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" - echo " --nomd5 : Don't calculate an MD5 for archive" - echo " --nocrc : Don't calculate a CRC for archive" - echo " --sha256 : Compute a SHA256 checksum for the archive" - echo " --header file : Specify location of the header script" - echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." - echo " --follow : Follow the symlinks in the archive" - echo " --noprogress : Do not show the progress during the decompression" - echo " --nox11 : Disable automatic spawn of a xterm" - echo " --nowait : Do not wait for user input after executing embedded" - echo " program from an xterm" - echo " --sign passphrase : Signature private key to sign the package with" - echo " --lsm file : LSM file describing the package" - echo " --license file : Append a license file" - echo " --help-header file : Add a header to the archive's --help output" - echo " --packaging-date date" - echo " : Use provided string as the packaging date" - echo " instead of the current date." - echo - echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." - echo " --export-conf : Export configuration variables to startup_script" - echo - echo "Do not forget to give a fully qualified startup script name" - echo "(i.e. with a ./ prefix if inside the archive)." - exit 1 -} - -# Default settings -if type gzip >/dev/null 2>&1; then - COMPRESS=gzip -elif type compress >/dev/null 2>&1; then - COMPRESS=compress -else - echo "ERROR: missing commands: gzip, compress" >&2 - MS_Usage -fi -ENCRYPT=n -PASSWD="" -PASSWD_SRC="" -OPENSSL_NO_MD=n -COMPRESS_LEVEL=9 -DEFAULT_THREADS=123456 # Sentinel value -THREADS=$DEFAULT_THREADS -KEEP=n -CURRENT=n -NOX11=n -NOWAIT=n -APPEND=n -TAR_QUIETLY=n -KEEP_UMASK=n -QUIET=n -NOPROGRESS=n -COPY=none -NEED_ROOT=n -TAR_ARGS=rvf -TAR_FORMAT=ustar -TAR_EXTRA="" -GPG_EXTRA="" -DU_ARGS=-ks -HEADER=`dirname "$0"`/makeself-header.sh -SIGNATURE="" -TARGETDIR="" -NOOVERWRITE=n -DATE=`LC_ALL=C date` -EXPORT_CONF=n -SHA256=n -OWNERSHIP=n -SIGN=n -GPG_PASSPHRASE="" - -# LSM file stuff -LSM_CMD="echo No LSM. >> \"\$archname\"" - -while true -do - case "$1" in - --version | -v) - echo Makeself version $MS_VERSION - exit 0 - ;; - --pbzip2) - COMPRESS=pbzip2 - shift - ;; - --bzip2) - COMPRESS=bzip2 - shift - ;; - --gzip) - COMPRESS=gzip - shift - ;; - --pigz) - COMPRESS=pigz - shift - ;; - --zstd) - COMPRESS=zstd - shift - ;; - --xz) - COMPRESS=xz - shift - ;; - --lzo) - COMPRESS=lzo - shift - ;; - --lz4) - COMPRESS=lz4 - shift - ;; - --compress) - COMPRESS=compress - shift - ;; - --base64) - COMPRESS=base64 - shift - ;; - --gpg-encrypt) - COMPRESS=gpg - shift - ;; - --gpg-asymmetric-encrypt-sign) - COMPRESS=gpg-asymmetric - shift - ;; - --gpg-extra) - GPG_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-encrypt) - ENCRYPT=openssl - shift - ;; - --ssl-passwd) - PASSWD=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-pass-src) - PASSWD_SRC=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-no-md) - OPENSSL_NO_MD=y - shift - ;; - --nocomp) - COMPRESS=none - shift - ;; - --complevel) - COMPRESS_LEVEL="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --threads) - THREADS="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nochown) - OWNERSHIP=n - shift - ;; - --chown) - OWNERSHIP=y - shift - ;; - --notemp) - KEEP=y - shift - ;; - --copy) - COPY=copy - shift - ;; - --current) - CURRENT=y - KEEP=y - shift - ;; - --tar-format) - TAR_FORMAT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --tar-extra) - TAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --untar-extra) - UNTAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --target) - TARGETDIR="$2" - KEEP=y - shift 2 || { MS_Usage; exit 1; } - ;; - --sign) - SIGN=y - GPG_PASSPHRASE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nooverwrite) - NOOVERWRITE=y - shift - ;; - --needroot) - NEED_ROOT=y - shift - ;; - --header) - HEADER="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --cleanup) - CLEANUP_SCRIPT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --license) - # We need to escape all characters having a special meaning in double quotes - LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") - shift 2 || { MS_Usage; exit 1; } - ;; - --follow) - TAR_ARGS=rvhf - DU_ARGS=-ksL - shift - ;; - --noprogress) - NOPROGRESS=y - shift - ;; - --nox11) - NOX11=y - shift - ;; - --nowait) - NOWAIT=y - shift - ;; - --nomd5) - NOMD5=y - shift - ;; - --sha256) - SHA256=y - shift - ;; - --nocrc) - NOCRC=y - shift - ;; - --append) - APPEND=y - shift - ;; - --lsm) - LSM_CMD="cat \"$2\" >> \"\$archname\"" - shift 2 || { MS_Usage; exit 1; } - ;; - --packaging-date) - DATE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --help-header) - HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` - shift 2 || { MS_Usage; exit 1; } - [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER -" - ;; - --tar-quietly) - TAR_QUIETLY=y - shift - ;; - --keep-umask) - KEEP_UMASK=y - shift - ;; - --export-conf) - EXPORT_CONF=y - shift - ;; - -q | --quiet) - QUIET=y - shift - ;; - -h | --help) - MS_Usage - ;; - -*) - echo Unrecognized flag : "$1" - MS_Usage - ;; - *) - break - ;; - esac -done - -if test $# -lt 1; then - MS_Usage -else - if test -d "$1"; then - archdir="$1" - else - echo "Directory $1 does not exist." >&2 - exit 1 - fi -fi -archname="$2" - -if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then - if test "$TAR_ARGS" = "rvf"; then - TAR_ARGS="rf" - elif test "$TAR_ARGS" = "rvhf"; then - TAR_ARGS="rhf" - fi -fi - -if test "$APPEND" = y; then - if test $# -lt 2; then - MS_Usage - fi - - # Gather the info from the original archive - OLDENV=`sh "$archname" --dumpconf` - if test $? -ne 0; then - echo "Unable to update archive: $archname" >&2 - exit 1 - else - eval "$OLDENV" - OLDSKIP=`expr $SKIP + 1` - fi -else - if test "$KEEP" = n -a $# = 3; then - echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 - echo >&2 - MS_Usage - fi - # We don't want to create an absolute directory unless a target directory is defined - if test "$CURRENT" = y; then - archdirname="." - elif test x"$TARGETDIR" != x; then - archdirname="$TARGETDIR" - else - archdirname=`basename "$1"` - fi - - if test $# -lt 3; then - MS_Usage - fi - - LABEL="$3" - SCRIPT="$4" - test "x$SCRIPT" = x || shift 1 - shift 3 - SCRIPTARGS="$*" -fi - -if test "$KEEP" = n -a "$CURRENT" = y; then - echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 - exit 1 -fi - -case $COMPRESS in -gzip) - GZIP_CMD="gzip -c$COMPRESS_LEVEL" - GUNZIP_CMD="gzip -cd" - ;; -pigz) - GZIP_CMD="pigz -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --processes $THREADS" - fi - GUNZIP_CMD="gzip -cd" - ;; -zstd) - GZIP_CMD="zstd -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="zstd -cd" - ;; -pbzip2) - GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD -p$THREADS" - fi - GUNZIP_CMD="bzip2 -d" - ;; -bzip2) - GZIP_CMD="bzip2 -$COMPRESS_LEVEL" - GUNZIP_CMD="bzip2 -d" - ;; -xz) - GZIP_CMD="xz -c$COMPRESS_LEVEL" - # Must opt-in by specifying a value since not all versions of xz support threads - if test $THREADS -ne $DEFAULT_THREADS; then - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="xz -d" - ;; -lzo) - GZIP_CMD="lzop -c$COMPRESS_LEVEL" - GUNZIP_CMD="lzop -d" - ;; -lz4) - GZIP_CMD="lz4 -c$COMPRESS_LEVEL" - GUNZIP_CMD="lz4 -d" - ;; -base64) - GZIP_CMD="base64" - GUNZIP_CMD="base64 --decode -i -" - ;; -gpg) - GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" - GUNZIP_CMD="gpg -d" - ENCRYPT="gpg" - ;; -gpg-asymmetric) - GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" - GUNZIP_CMD="gpg --yes -d" - ENCRYPT="gpg" - ;; -compress) - GZIP_CMD="compress -fc" - GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" - ;; -none) - GZIP_CMD="cat" - GUNZIP_CMD="cat" - ;; -esac - -if test x"$ENCRYPT" = x"openssl"; then - if test x"$APPEND" = x"y"; then - echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 - fi - - ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" - DECRYPT_CMD="openssl enc -aes-256-cbc -d" - - if test x"$OPENSSL_NO_MD" != x"y"; then - ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" - DECRYPT_CMD="$DECRYPT_CMD -md sha256" - fi - - if test -n "$PASSWD_SRC"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" - elif test -n "$PASSWD"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" - fi -fi - -tmpfile="${TMPDIR:-/tmp}/mkself$$" - -if test -f "$HEADER"; then - oldarchname="$archname" - archname="$tmpfile" - # Generate a fake header to count its lines - SKIP=0 - . "$HEADER" - SKIP=`cat "$tmpfile" |wc -l` - # Get rid of any spaces - SKIP=`expr $SKIP` - rm -f "$tmpfile" - if test "$QUIET" = "n"; then - echo "Header is $SKIP lines long" >&2 - fi - archname="$oldarchname" -else - echo "Unable to open header file: $HEADER" >&2 - exit 1 -fi - -if test "$QUIET" = "n"; then - echo -fi - -if test "$APPEND" = n; then - if test -f "$archname"; then - echo "WARNING: Overwriting existing file: $archname" >&2 - fi -fi - -USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` - -if test "." = "$archdirname"; then - if test "$KEEP" = n; then - archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" - fi -fi - -test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } -if test "$QUIET" = "n"; then - echo "About to compress $USIZE KB of data..." - echo "Adding files to archive named \"$archname\"..." -fi - -# See if we have GNU tar -TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` -test -x "$TAR" || TAR=tar - -tmparch="${TMPDIR:-/tmp}/mkself$$.tar" -( - if test "$APPEND" = "y"; then - tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" - fi - cd "$archdir" - # "Determining if a directory is empty" - # https://www.etalabs.net/sh_tricks.html - find . \ - \( \ - ! -type d \ - -o \ - \( -links 2 -exec sh -c ' - is_empty () ( - cd "$1" - set -- .[!.]* ; test -f "$1" && return 1 - set -- ..?* ; test -f "$1" && return 1 - set -- * ; test -f "$1" && return 1 - return 0 - ) - is_empty "$0"' {} \; \ - \) \ - \) -print \ - | LC_ALL=C sort \ - | sed 's/./\\&/g' \ - | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" -) || { - echo "ERROR: failed to create temporary archive: $tmparch" - rm -f "$tmparch" "$tmpfile" - exit 1 -} - -USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` - -eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { - echo "ERROR: failed to create temporary file: $tmpfile" - rm -f "$tmparch" "$tmpfile" - exit 1 -} -rm -f "$tmparch" - -if test x"$ENCRYPT" = x"openssl"; then - echo "About to encrypt archive \"$archname\"..." - { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ - { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } -fi - -fsize=`cat "$tmpfile" | wc -c | tr -d " "` - -# Compute the checksums - -shasum=0000000000000000000000000000000000000000000000000000000000000000 -md5sum=00000000000000000000000000000000 -crcsum=0000000000 - -if test "$NOCRC" = y; then - if test "$QUIET" = "n"; then - echo "skipping crc at user request" - fi -else - crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` - if test "$QUIET" = "n"; then - echo "CRC: $crcsum" - fi -fi - -if test "$SHA256" = y; then - SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` - if test -x "$SHA_PATH"; then - shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` - else - SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` - shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` - fi - if test "$QUIET" = "n"; then - if test -x "$SHA_PATH"; then - echo "SHA256: $shasum" - else - echo "SHA256: none, SHA command not found" - fi - fi -fi -if test "$NOMD5" = y; then - if test "$QUIET" = "n"; then - echo "Skipping md5sum at user request" - fi -else - # Try to locate a MD5 binary - OLD_PATH=$PATH - PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` - PATH=$OLD_PATH - if test -x "$MD5_PATH"; then - if test `basename ${MD5_PATH}`x = digestx; then - MD5_ARG="-a md5" - fi - md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` - if test "$QUIET" = "n"; then - echo "MD5: $md5sum" - fi - else - if test "$QUIET" = "n"; then - echo "MD5: none, MD5 command not found" - fi - fi -fi -if test "$SIGN" = y; then - GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` - if test -x "$GPG_PATH"; then - SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` - if test "$QUIET" = "n"; then - echo "Signature: $SIGNATURE" - fi - else - echo "Missing gpg command" >&2 - fi -fi - -totalsize=0 -for size in $fsize; -do - totalsize=`expr $totalsize + $size` -done - -if test "$APPEND" = y; then - mv "$archname" "$archname".bak || exit - - # Prepare entry for new archive - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - # Generate the header - . "$HEADER" - # Append the new data - cat "$tmpfile" >> "$archname" - - chmod +x "$archname" - rm -f "$archname".bak - if test "$QUIET" = "n"; then - echo "Self-extractable archive \"$archname\" successfully updated." - fi -else - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - - # Generate the header - . "$HEADER" - - # Append the compressed tar data after the stub - if test "$QUIET" = "n"; then - echo - fi - cat "$tmpfile" >> "$archname" - chmod +x "$archname" - if test "$QUIET" = "n"; then - echo Self-extractable archive \"$archname\" successfully created. - fi -fi -rm -f "$tmpfile" diff --git a/csrc/deepep/ops2/cmake/util/makeself/run-tests.sh b/csrc/deepep/ops2/cmake/util/makeself/run-tests.sh deleted file mode 100755 index 31ee16511..000000000 --- a/csrc/deepep/ops2/cmake/util/makeself/run-tests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# Run every available test - Bash needed -cd test -for test in *test; -do - echo "Running test $test ..." - bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } -done diff --git a/csrc/deepep/ops2/cmake/util/merge_aicpu_info_json.sh b/csrc/deepep/ops2/cmake/util/merge_aicpu_info_json.sh deleted file mode 100755 index 970a44bf1..000000000 --- a/csrc/deepep/ops2/cmake/util/merge_aicpu_info_json.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -echo $@ -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No project path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -if [[ ! -d "$ASCEND_OPP_PATH" ]]; then - echo "[ERROR] No opp install path is provided" - exit 1 -fi -custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json - -if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then - cp -f $custom_exist_info_json $temp_info_json - chmod +w $temp_info_json - python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} - cp -f $temp_info_json $custom_new_info_json - rm -f $temp_info_json -fi diff --git a/csrc/deepep/ops2/cmake/util/opdesc_parser.py b/csrc/deepep/ops2/cmake/util/opdesc_parser.py deleted file mode 100755 index 7b789567a..000000000 --- a/csrc/deepep/ops2/cmake/util/opdesc_parser.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import os -import sys - -OP_ALL = "__ALLOP__" -SOC_ALL = "__ALLSOC__" -SOC_TO_SHORT_SOC_MAP = { - "ascend910a": "ascend910", - "ascend910proa": "ascend910", - "ascend910b": "ascend910", - "ascend910prob": "ascend910", - "ascend910premiuma": "ascend910", - "ascend910b1": "ascend910b", - "ascend910b2": "ascend910b", - "ascend910b2c": "ascend910b", - "ascend910b3": "ascend910b", - "ascend910b4": "ascend910b", - "ascend910b4-1": "ascend910b", - "ascend910_9391": "ascend910_93", - "ascend910_9381": "ascend910_93", - "ascend910_9372": "ascend910_93", - "ascend910_9392": "ascend910_93", - "ascend910_9382": "ascend910_93", - "ascend910_9361": "ascend910_93", - "ascend310p1": "ascend310p", - "ascend310p3": "ascend310p", - "ascend310p3vir01": "ascend310p", - "ascend310p3vir02": "ascend310p", - "ascend310p3vir04": "ascend310p", - "ascend310p3vir08": "ascend310p", - "ascend310b1": "ascend310b", - "bs9sx1aa": "bs9sx1a", - "ascend610lite": "ascend610lite", -} -CONFLICT_KEYWORDS = { - "and", - "as", - "assert", - "break", - "class", - "continue", - "def", - "del", - "elif", - "else", - "except", - "finally", - "for", - "from", - "global", - "if", - "import", - "in", - "is", - "lambda", - "not", - "or", - "pass", - "raise", - "return", - "try", - "while", - "with", - "yield", - "False", - "None", - "True", - "nonlocal", - "arg", - "__inputs__", - "__outputs__", - "options", - "bisheng", - "bisheng_path", - "tikcpp_path", - "impl_mode", - "custom_compile_options", - "custom_all_compile_options", - "soc_version", - "soc_short", - "custom_compile_options_soc", - "custom_all_compile_options_soc", - "origin_func_name", - "ascendc_src_dir_ex", - "ascendc_src_dir", - "ascendc_src_file", - "src", - "op_type", - "code_channel", - "op_info", - "compile_op", - "get_code_channel", - "result", - "__attrs__", - "isinstance", - "attr", - "get_current_build_config", - "_build_args", - "get_dtype_fmt_options", - "shutil", - "os", - "get_kernel_source", -} - - -class OpDesc: - def __init__(self: any, op_type: str): - self.op_type = op_type - self.attr_list = [] - self.attr_val = {} - self.input_name = [] - self.input_ori_name = [] - self.input_type = [] - self.input_dtype = [] - self.input_dtype_for_bin_list = [] - self.input_dtype_for_bin = {} - self.input_fmt = [] - self.input_fmt_for_bin_list = [] - self.input_fmt_for_bin = {} - self.input_virt = {} - self.output_name = [] - self.output_ori_name = [] - self.output_type = [] - self.output_dtype = [] - self.output_dtype_for_bin_list = [] - self.output_dtype_for_bin = {} - self.output_fmt = [] - self.output_fmt_for_bin_list = [] - self.output_fmt_for_bin = {} - self.output_init_value = [] - self.output_shape_depend_on_compute = [] - self.op_fmt_sel = False - self.op_chk_support = False - self.op_intf = "" - self.kern_name = "" - self.op_file = "" - self.op_replay_flag = False - self.op_replay_batch = False - self.input_idx = -1 - self.output_idx = -1 - self.max_block_dim = 32 - self.max_shape_size = 268435456 - self.dynamic_shape = False - self.op_range_limit = "" - self.custom_compile_options = {} - self.custom_all_compile_options = {} - self.param_type_dynamic = False - self.mc2_ctx = [] - self.bin_cprs_list = [] - self.bin_cprs_head = [] - self.bin_save_list = [] - - @staticmethod - def _parse_digit(conf: str) -> int: - return int(conf.split("=")[1]) - - @staticmethod - def _parse_flag(conf: str) -> bool: - if "true" == conf.split("=")[1]: - return True - return False - - @staticmethod - def _parse_str(conf: str) -> str: - return conf.split("=")[1] - - @staticmethod - def _parse_list(conf: str) -> list: - return conf.split("=")[1].split(",") - - def parse_input(self: any, conf: str): - if conf.startswith("input{}.name".format(int(self.input_idx) + 1)): - self.input_idx += 1 - self.input_ori_name.append(self._parse_str(conf)) - self.input_name.append(self.input_ori_name[-1] + "_in__") - elif conf.startswith("input{}.paramType".format(int(self.input_idx))): - param_type = self._parse_str(conf) - self.input_type.append(param_type) - if param_type == "dynamic": - self.param_type_dynamic = True - elif conf.startswith("input{}.dtype".format(int(self.input_idx))): - self.input_dtype.append(self._parse_str(conf)) - elif conf.startswith("input{}.for_bin_dtype".format(int(self.input_idx))): - self.input_dtype_for_bin.update({self.input_idx: self._parse_str(conf)}) - elif conf.startswith("input{}.format".format(int(self.input_idx))): - self.input_fmt.append(self._parse_str(conf)) - elif conf.startswith("input{}.for_bin_format".format(int(self.input_idx))): - self.input_fmt_for_bin.update({self.input_idx: self._parse_str(conf)}) - elif conf.startswith("input{}.virtual".format(int(self.input_idx))): - self.input_virt[self.input_idx] = self._parse_str(conf) - elif conf.startswith("input{}.initValue".format(int(self.input_idx))): - raise Exception( - f"[ERROR]: Op: {{'{self.op_type}'}} input {self.input_ori_name[int(self.input_idx)]}\ - has InitValue, which is not support!" - ) - else: - return - - def parse_output(self: any, conf: str): - if conf.startswith("output{}.name".format(int(self.output_idx) + 1)): - self.output_idx += 1 - self.output_ori_name.append(self._parse_str(conf)) - self.output_name.append(self.output_ori_name[-1] + "_out_") - self.output_init_value.append(None) - elif conf.startswith("output{}.paramType".format(int(self.output_idx))): - param_type = self._parse_str(conf) - self.output_type.append(param_type) - if param_type == "dynamic": - self.param_type_dynamic = True - elif conf.startswith("output{}.dtype".format(int(self.output_idx))): - self.output_dtype.append(self._parse_str(conf)) - elif conf.startswith("output{}.for_bin_dtype".format(int(self.output_idx))): - self.output_dtype_for_bin.update({self.output_idx: self._parse_str(conf)}) - elif conf.startswith("output{}.format".format(int(self.output_idx))): - self.output_fmt.append(self._parse_str(conf)) - elif conf.startswith("output{}.for_bin_format".format(int(self.output_idx))): - self.output_fmt_for_bin.update({self.output_idx: self._parse_str(conf)}) - elif conf.startswith("output{}.initValue".format(int(self.output_idx))): - self.output_init_value[int(self.output_idx)] = self._parse_str(conf) - elif conf.startswith( - "output{}.outputShapeDependOnCompute=true".format(int(self.output_idx)) - ): - self.output_shape_depend_on_compute.append(int(self.output_idx)) - else: - return - - def parse_op_format(self: any, conf: str): - self.op_fmt_sel = self._parse_flag(conf) - - def parse_check_support(self: any, conf: str): - self.op_chk_support = self._parse_flag(conf) - - def parse_range_limit(self: any, conf: str): - self.op_range_limit = self._parse_str(conf) - - def parse_kern_name(self: any, conf: str): - self.kern_name = self._parse_str(conf) - - def parse_op_intf(self: any, conf: str): - self.op_intf = self._parse_str(conf) - - def parse_op_file(self: any, conf: str): - self.op_file = self._parse_str(conf) - - def parse_dynamic_shape(self: any, conf: str): - self.dynamic_shape = self._parse_flag(conf) - - def parse_attr_list(self: any, conf: str): - self.attr_list = self._parse_list(conf) - intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS - if intersection_element: - raise Exception( - f"[ERROR]: The attribute name: {intersection_element} in op: {{'{self.op_type}'}} \ -conflicts with the built-in variable name. Use a complex name or prefix the operator name." - ) - - def parse_mc2_ctx(self: any, conf: str): - self.mc2_ctx = self._parse_list(conf) - - @staticmethod - def _camel_to_snake(camel_case_str: str): - snake_case_str = "" - for i, c in enumerate(camel_case_str): - if i == 0: - snake_case_str += c.lower() - elif c.isupper(): - snake_case_str += "_" + c.lower() - else: - snake_case_str += c - return snake_case_str - - def parse_attr_val(self: any, conf: str): - for attr in self.attr_list: - if self.attr_val.get(attr) is None: - self.attr_val[attr] = {} - if conf.startswith("attr_{}.type".format(attr)): - self.attr_val.get(attr)["type"] = self._camel_to_snake( - self._parse_str(conf) - ) - elif conf.startswith("attr_{}.paramType".format(attr)): - self.attr_val.get(attr)["paramType"] = self._parse_str(conf) - elif conf.startswith("attr_{}.defaultValue".format(attr)): - self.attr_val.get(attr)["defaultValue"] = self._parse_str(conf) - - def parse_replay_val(self: any, batch_list: list, iterator_list: list): - if self.op_type in batch_list: - self.op_replay_flag = True - self.op_replay_batch = True - elif self.op_type in iterator_list: - self.op_replay_flag = True - self.op_replay_batch = False - - -def _is_op_type_in_opdesc(op_descs: list, op_type: str): - for op in op_descs: - if op_type == op.op_type: - return True - return False - - -def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): - for op in op_descs: - op.custom_all_compile_options = soc_ver_compile_options - - -def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): - for op in op_descs: - if op.op_type != op_type: - continue - op.custom_compile_options.update(soc_ver_compile_options) - - -def _trans_soc_ver_to_short(soc_ver: str): - low_soc_ver = soc_ver.lower() - if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: - print( - f"WARNING: caution: {soc_ver} will trans into ascend910, if not your intention," - f"use ascend910b1~4 instead" - ) - return SOC_TO_SHORT_SOC_MAP[low_soc_ver] - - -def _get_op_custom_options(op_descs: list, auto_gen_dir: str): - if auto_gen_dir is None: - return {} - file = os.path.join(auto_gen_dir, "custom_compile_options.ini") - if not os.path.exists(file): - print(f"WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini") - return {} - with open(file, "r") as fd: - lines = fd.readlines() - for line in lines: - param_list = str.split(line.rstrip("\n"), ",") - if len(param_list) != 3: - raise Exception( - f"ERROR: custom compile option {param_list} len is not 3" - ) - op_type = param_list[0] - if op_type.upper() == "ALL": - op_type = OP_ALL - if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: - continue - soc_ver_compile_options = {} - soc_ver = param_list[1] - options_str = param_list[2] - options = str.split(options_str, ";") - if soc_ver == "": - soc_ver_compile_options[SOC_ALL] = options - else: - soc_ver_list = str.split(soc_ver, ";") - for ver in soc_ver_list: - short_ver = _trans_soc_ver_to_short(ver) - soc_ver_compile_options[short_ver] = options - if op_type == OP_ALL: - _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) - else: - _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) - - -def get_op_desc( - file: str, - batch_list: list, - iterator_list: list, - builder: any, - op_type: list, - auto_gen_dir: str = None, -) -> list: - op_descs = [] - op_match = False - with open(file, "r") as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if line.startswith("["): - name = line[1:-1] - if op_type is None or name in op_type: - op_match = True - op_desc = builder(name) - op_desc.parse_replay_val(batch_list, iterator_list) - op_descs.append(op_desc) - else: - op_match = False - if op_type is not None and len(op_descs) == len(op_type): - break - continue - if not op_match: - continue - if line.startswith("input"): - op_desc.parse_input(line) - elif line.startswith("output"): - op_desc.parse_output(line) - elif line.startswith("dynamicFormat.flag"): - op_desc.parse_op_format(line) - elif line.startswith("needCheckSupport.flag"): - op_desc.parse_check_support(line) - elif line.startswith("rangeLimit.value"): - op_desc.parse_range_limit(line) - elif line.startswith("opInterface.value"): - op_desc.parse_op_intf(line) - elif line.startswith("kernel.name"): - op_desc.parse_kern_name(line) - elif line.startswith("opFile.value"): - op_desc.parse_op_file(line) - elif line.startswith("dynamicShapeSupport.flag"): - op_desc.parse_dynamic_shape(line) - elif line.startswith("mc2.ctx"): - op_desc.parse_mc2_ctx(line) - elif line.startswith("attr.list"): - op_desc.parse_attr_list(line) - elif line.startswith("attr_"): - op_desc.parse_attr_val(line) - _get_op_custom_options(op_descs, auto_gen_dir) - return op_descs diff --git a/csrc/deepep/ops2/cmake/util/parse_ini_to_json.py b/csrc/deepep/ops2/cmake/util/parse_ini_to_json.py deleted file mode 100755 index 928acae6f..000000000 --- a/csrc/deepep/ops2/cmake/util/parse_ini_to_json.py +++ /dev/null @@ -1,449 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -""" -parser ini to json -""" - -import json -import os -import stat -import sys - -ATTR_TYPE_LIST = [ - "int", - "float", - "bool", - "str", - "listInt", - "listFloat", - "listBool", - "listStr", - "listListInt", - "type", - "listType", - "tensor", - "listTensor", -] -ATTR_PARAMTYPE_LIST = ["optional", "required"] -BOOL_FLAG_KEY = [ - "dynamicFormat", - "dynamicShapeSupport", - "dynamicRankSupport", - "precision_reduce", - "heavyOp", - "needCheckSupport", - "enableVectorCore", -] -BOOL_LIST = ["true", "false"] -DTYPE_LIST = [ - "float16", - "float", - "float32", - "int8", - "int16", - "int32", - "uint8", - "uint16", - "uint32", - "bool", - "int64", - "uint64", - "qint8", - "qint16", - "qint32", - "quint8", - "quint16", - "double", - "complex64", - "complex128", - "string", - "resource", - "dual", - "dual_sub_int8", - "dual_sub_uint8", - "string_ref", - "int4", - "bfloat16", - "uint1", -] -FORMAT_LIST = [ - "NCHW", - "NHWC", - "ND", - "NC1HWC0", - "FRACTAL_Z", - "NC1C0HWPAD", - "NHWC1C0", - "FSR_NCHW", - "FRACTAL_DECONV", - "C1HWNC0", - "FRACTAL_DECONV_TRANSPOSE", - "FRACTAL_DECONV_SP_STRIDE_TRANS", - "NC1HWC0_C04", - "FRACTAL_Z_C04", - "CHWN", - "FRACTAL_DECONV_SP_STRIDE8_TRANS", - "HWCN", - "NC1KHKWHWC0", - "BN_WEIGHT", - "FILTER_HWCK", - "HASHTABLE_LOOKUP_LOOKUPS", - "HASHTABLE_LOOKUP_KEYS", - "HASHTABLE_LOOKUP_VALUE", - "HASHTABLE_LOOKUP_OUTPUT", - "HASHTABLE_LOOKUP_HITS", - "C1HWNCoC0", - "MD", - "NDHWC", - "FRACTAL_ZZ", - "FRACTAL_NZ", - "NCDHW", - "DHWCN", - "NDC1HWC0", - "FRACTAL_Z_3D", - "CN", - "NC", - "DHWNC", - "FRACTAL_Z_3D_TRANSPOSE", - "FRACTAL_ZN_LSTM", - "FRACTAL_ZN_RNN", - "FRACTAL_Z_G", - "NULL", -] - - -def parse_ini_files(ini_files): - """ - parse ini files to json - Parameters: - ---------------- - ini_files:input file list - return:ops_info - ---------------- - """ - tbe_ops_info = {} - for ini_file in ini_files: - check_file_size(ini_file) - parse_ini_to_obj(ini_file, tbe_ops_info) - return tbe_ops_info - - -def check_file_size(input_file): - try: - file_size = os.path.getsize(input_file) - except OSError as os_error: - print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) - raise OSError from os_error - if file_size > 10 * 1024 * 1024: - print( - "[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait." - % input_file - ) - - -def parse_ini_to_obj(ini_file, tbe_ops_info): - """ - parse ini file to json obj - Parameters: - ---------------- - ini_file:ini file path - tbe_ops_info:ops_info - ---------------- - """ - with open(ini_file) as ini_file: - lines = ini_file.readlines() - op_dict = {} - op_name = "" - find_op_type = False - for line in lines: - line = line.rstrip() - if line == "": - continue - if line.startswith("["): - if line.endswith("]"): - op_name = line[1:-1] - op_dict = {} - tbe_ops_info[op_name] = op_dict - find_op_type = True - elif "=" in line: - key1 = line[: line.index("=")] - key2 = line[line.index("=") + 1 :] - key1_0, key1_1 = key1.split(".") - if key1_0 not in op_dict: - op_dict[key1_0] = {} - if key1_1 in op_dict.get(key1_0): - raise RuntimeError( - "Op:" + op_name + " " + key1_0 + " " + key1_1 + " is repeated!" - ) - dic_key = op_dict.get(key1_0) - dic_key[key1_1] = key2 - else: - continue - if not find_op_type: - raise RuntimeError("Not find OpType in .ini file.") - - -def check_output_exist(op_dict, is_valid): - """ - Function Description: - Check output is exist - Parameter: op_dict - Parameter: is_valid - """ - if "output0" in op_dict: - output0_dict = op_dict.get("output0") - if output0_dict.get("name", None) is None: - is_valid = False - print("output0.name is required in .ini file!") - else: - is_valid = False - print("output0 is required in .ini file!") - return is_valid - - -def check_attr_dict(attr_dict, is_valid, attr): - """ - Function Description: - Check attr_dict - Parameter: attr_dict - Parameter: is_valid - Parameter: attr - """ - attr_type = attr_dict.get("type") - value = attr_dict.get("value") - param_type = attr_dict.get("paramType") - if attr_type is None or value is None: - is_valid = False - print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) - if param_type and param_type not in ATTR_PARAMTYPE_LIST: - is_valid = False - print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) - if attr_type and attr_type not in ATTR_TYPE_LIST: - is_valid = False - print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) - return is_valid - - -def check_attr(op_dict, is_valid): - """ - Function Description: - Check attr - Parameter: op_dict - Parameter: is_valid - """ - if "attr" in op_dict: - attr_dict = op_dict.get("attr") - attr_list_str = attr_dict.get("list", None) - if attr_list_str is None: - is_valid = False - print("attr.list is required in .ini file!") - else: - attr_list = attr_list_str.split(",") - for attr_name in attr_list: - attr = "attr_" + attr_name.strip() - attr_dict = op_dict.get(attr) - if attr_dict: - is_valid = check_attr_dict(attr_dict, is_valid, attr) - else: - is_valid = False - print( - "%s is required in .ini file, when attr.list is %s!" - % (attr, attr_list_str) - ) - return is_valid - - -def check_bool_flag(op_dict, is_valid): - """ - Function Description: - check_bool_flag - Parameter: op_dict - Parameter: is_valid - """ - for key in BOOL_FLAG_KEY: - if key in op_dict: - op_bool_key = op_dict.get(key) - if op_bool_key.get("flag").strip() not in BOOL_LIST: - is_valid = False - print("{0}.flag only support {1}.".format(key, BOOL_LIST)) - return is_valid - - -def check_type_format(op_info, is_valid, op_info_key): - """ - Function Description: - Check type and format - Parameter: op_info - Parameter: is_valid - Parameter: op_info_key - """ - op_info_dtype_str = op_info.get("dtype") - op_info_dtype_num = 0 - op_info_format_num = 0 - if op_info_dtype_str: - op_info_dtype = op_info_dtype_str.split(",") - op_info_dtype_num = len(op_info_dtype) - for dtype in op_info_dtype: - if dtype.strip() not in DTYPE_LIST: - is_valid = False - print("{0}.dtype not support {1}.".format(op_info_key, dtype)) - op_info_format_str = op_info.get("format") - if op_info_format_str: - op_info_format = op_info_format_str.split(",") - op_info_format_num = len(op_info_format) - for op_format in op_info_format: - if op_format.strip() not in FORMAT_LIST: - is_valid = False - print("{0}.format not support {1}.".format(op_info_key, op_format)) - if op_info_dtype_num > 0 and op_info_format_num > 0: - if op_info_dtype_num != op_info_format_num: - is_valid = False - print( - "The number of {0}.dtype not match the number of {0}.format.".format( - op_info_key - ) - ) - return is_valid - - -def check_op_info(tbe_ops): - """ - Function Description: - Check info. - Parameter: tbe_ops - Return Value: is_valid - """ - print("\n\n==============check valid for ops info start==============") - required_op_input_info_keys = ["paramType", "name"] - required_op_output_info_keys = ["paramType", "name"] - param_type_valid_value = ["dynamic", "optional", "required"] - is_valid = True - for op_key in tbe_ops: - op_dict = tbe_ops[op_key] - for op_info_key in op_dict: - if op_info_key.startswith("input"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_input_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print( - "op: " - + op_key - + " " - + op_info_key - + " missing: " - + ",".join(missing_keys) - ) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print( - "op: " - + op_key - + " " - + op_info_key - + " paramType not valid, valid key:[dynamic, " - "optional, required]" - ) - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - if op_info_key.startswith("output"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_output_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print( - "op: " - + op_key - + " " - + op_info_key - + " missing: " - + ",".join(missing_keys) - ) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print( - "op: " - + op_key - + " " - + op_info_key - + " paramType not valid, valid key:[dynamic, " - "optional, required]" - ) - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - is_valid = check_attr(op_dict, is_valid) - is_valid = check_bool_flag(op_dict, is_valid) - print("==============check valid for ops info end================\n\n") - return is_valid - - -def write_json_file(tbe_ops_info, json_file_path): - """ - Save info to json file - Parameters: - ---------------- - tbe_ops_info: ops_info - json_file_path: json file path - ---------------- - """ - json_file_real_path = os.path.realpath(json_file_path) - wr_flag = os.O_WRONLY | os.O_CREAT - wr_mode = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), "w") as file_path: - # The owner have all rights£¬group only have read rights - os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + stat.S_IRUSR) - json.dump( - tbe_ops_info, file_path, sort_keys=True, indent=4, separators=(",", ":") - ) - print("Compile op info cfg successfully.") - - -def parse_ini_to_json(ini_file_paths, outfile_path): - """ - parse ini files to json file - Parameters: - ---------------- - ini_file_paths: list of ini file path - outfile_path: output file path - ---------------- - """ - tbe_ops_info = parse_ini_files(ini_file_paths) - if not check_op_info(tbe_ops_info): - print("Compile op info cfg failed.") - return False - write_json_file(tbe_ops_info, outfile_path) - return True - - -if __name__ == "__main__": - args = sys.argv - - OUTPUT_FILE_PATH = "tbe_ops_info.json" - ini_file_path_list = [] - parse_ini_list = [] - - for arg in args: - if arg.endswith("ini"): - ini_file_path_list.append(arg) - OUTPUT_FILE_PATH = arg.replace(".ini", ".json") - if arg.endswith("json"): - OUTPUT_FILE_PATH = arg - - if not ini_file_path_list: - ini_file_path_list.append("tbe_ops_info.ini") - - for ini_file in ini_file_path_list: - if os.path.exists(ini_file): - parse_ini_list.append(ini_file) - - if parse_ini_list: - if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): - sys.exit(1) - sys.exit(0) diff --git a/csrc/deepep/ops2/cmake/util/preset_parse.py b/csrc/deepep/ops2/cmake/util/preset_parse.py deleted file mode 100755 index 983f16e74..000000000 --- a/csrc/deepep/ops2/cmake/util/preset_parse.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import json -import os -import sys - - -def read_json(file): - with open(file, "r") as fd: - config = json.load(fd) - return config - - -def get_config_opts(file): - config = read_json(file) - - src_dir = os.path.abspath(os.path.dirname(file)) - opts = "" - - for conf in config: - if conf == "configurePresets": - for node in config[conf]: - macros = node.get("cacheVariables") - if macros is not None: - for key in macros: - opts += "-D{}={} ".format(key, macros[key]["value"]) - - opts = opts.replace("${sourceDir}", src_dir) - print(opts) - - -if __name__ == "__main__": - get_config_opts(sys.argv[1]) diff --git a/csrc/deepep/ops2/cmake/util/replay_codegen.py b/csrc/deepep/ops2/cmake/util/replay_codegen.py deleted file mode 100755 index 6f896a098..000000000 --- a/csrc/deepep/ops2/cmake/util/replay_codegen.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- - -import collections -import os -import stat - -import code_channel_infer -import const_var -import kernel_entry as keb -from tiling_data_def_build import gen_tiling - -PYF_PATH = os.path.dirname(__file__) - -ReplayCodeGenParams = collections.namedtuple( - "ReplayCodeGenParams", - [ - "op_type", - "impl", - "tiling_file", - "kernel", - "entry", - "argn", - "op_replay_batch", - "max_block_dim", - "max_shape_size", - ], -) - - -class ReplayCodeGen: - def __init__(self, replayCodeGenParams): - self.op_type = replayCodeGenParams.op_type - self.impl = replayCodeGenParams.impl - self.tiling_file = replayCodeGenParams.tiling_file - self.tiling_data_file = "" - self.kernel = replayCodeGenParams.kernel - self.entry = replayCodeGenParams.entry - self.argn = replayCodeGenParams.argn - self.batch = False - self.outdir = "" - self.data_type = "uint8_t" - self.blknum = 32 - self.op_replay_batch = replayCodeGenParams.op_replay_batch - self.max_block_dim = replayCodeGenParams.max_block_dim - self.max_shape_size = replayCodeGenParams.max_shape_size - - def set_batch(self, is_batch): - self.batch = is_batch - - def set_outdir(self, outdir): - self.outdir = outdir - - def gen_replay(self, ops_product: str): - kerentry = os.path.join(self.outdir, self.kernel + "_entry.cce") - kerimpl = os.path.join(self.outdir, self.kernel + "_impl.cpp") - replayimpl = os.path.join(self.outdir, self.kernel + "_replay.cpp") - if self.batch: - reptmp = os.path.join(PYF_PATH, "batch_replay_impl.temp") - else: - reptmp = os.path.join(PYF_PATH, "replay_impl.temp") - kertmp = os.path.join(PYF_PATH, "kernel_impl.temp") - self._gen_kentry(kerentry) - self._gen_kimpl_code(kerimpl, kertmp) - self._gen_tiling_data_header() - self._gen_replay_code(replayimpl, reptmp, ops_product) - - def _gen_tiling_data_header(self): - self.tiling_data_file = os.path.join( - self.outdir, self.kernel + "_tiling_data.h" - ) - gen_tiling(self.tiling_file, self.tiling_data_file) - - def _gen_kimpl_code(self, src, tmpfile): - with open(tmpfile, "r") as fd: - temp = fd.read() - temp = temp.replace("__CCE_FILE__", self.impl) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), "w") as ofd: - ofd.write(temp) - - def _gen_replay_code(self, src, tmpfile, ops_product: str): - with open(tmpfile, "r") as fd: - temp = fd.read() - temp = temp.replace("__ARG_NUM__", str(self.argn)) - argdef = [] - kargs = [] - for i in range(0, self.argn): - argdef.append("{} *".format(self.data_type)) - kargs.append("({} *)GetArg({})".format(self.data_type, i)) - temp = temp.replace("__ARGS_DEF__", ", ".join(argdef)) - temp = temp.replace("__KERNEL_ARGS__", ", ".join(kargs)) - temp = temp.replace("__KERNEL_FUN__", self.entry) - core_type_infer = "core_type" - code_channel = code_channel_infer.infer_code_channel( - code_channel_infer.InfoCodeChanelParams( - self.impl, - self.tiling_data_file, - self.kernel, - self.outdir, - ops_product, - None, - ) - ) - if code_channel == code_channel_infer.CODE_VEC: - core_type_infer = "0" - elif code_channel == code_channel_infer.CODE_CUBE: - core_type_infer = "1" - temp = temp.replace("__CORE_TYPE__", core_type_infer) - # register function - temp = temp.replace("__OPS_PRODUCT__", ops_product) - temp = temp.replace("__OPTYPE__", self.op_type) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), "w") as ofd: - ofd.write(temp) - - def _gen_kentry(self, src): - kf = "" - pre_alloc_str = "A" * 256 - if self.batch: - kf += keb.batch_code_gen( - "K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), - self.argn, - self.data_type, - ) - else: - kf += keb.mc_code_gen( - "K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), - self.argn, - self.data_type, - self.blknum, - ) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), "w") as ofd: - ofd.write(kf) diff --git a/csrc/deepep/ops2/cmake/util/replay_impl.temp b/csrc/deepep/ops2/cmake/util/replay_impl.temp deleted file mode 100644 index 1d30dd865..000000000 --- a/csrc/deepep/ops2/cmake/util/replay_impl.temp +++ /dev/null @@ -1,120 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], - int alen[], int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N * 32]; - int len[KERNEL_N * 32]; - int blknum[KERNEL_N]; - int max; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); - for (block_idx = 0; block_idx < block_num; block_idx++) { - //__OP_SET_KERNEL__ - int code_idx = i * block_num + block_idx; -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, false); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[code_idx] = (char *)pos; - len[code_idx] = CodeLen(); - pos += len[code_idx]; - printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); - } - blknum[i] = block_num; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/csrc/deepep/ops2/cmake/util/tiling_data_def_build.py b/csrc/deepep/ops2/cmake/util/tiling_data_def_build.py deleted file mode 100755 index 0576b202d..000000000 --- a/csrc/deepep/ops2/cmake/util/tiling_data_def_build.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Function: -The replay function entry -""" - -import os -import re -import stat -import sys - -import const_var - - -def gen_tiling(tiling_header_file: str, tiling_file_out: str): - if not os.path.exists(tiling_header_file): - print("warning: no userdef tiling header file: ", tiling_header_file) - return - print("generate tiling def header file: ", tiling_file_out) - tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() - tiling_source = "#ifndef __{}_H__\n".format(tmp_name) - tiling_source += "#define __{}_H__\n\n".format(tmp_name) - tiling_source += "#include \n" - tiling_source += "#include \n\n" - tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' - end_source = "" - pattern = re.compile(r"[(](.*)[)]", re.S) - with open(tiling_header_file, "r") as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if line.startswith("BEGIN_TILING_DATA_DEF"): - tiling_source += "#pragma pack(1)\n" - tiling_source += "struct " - struct_def = re.findall(pattern, line)[0] - tiling_source += struct_def + " {\n" - elif line.startswith("TILING_DATA_FIELD_DEF_ARR"): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(",") - tiling_source += " {} {}[{}] = {{}};\n".format( - fds[0].strip(), fds[2].strip(), fds[1].strip() - ) - elif line.startswith("TILING_DATA_FIELD_DEF_STRUCT"): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(",") - tiling_source += " {} {};\n".format(fds[0].strip(), fds[1].strip()) - elif line.startswith("TILING_DATA_FIELD_DEF"): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(",") - tiling_source += " {} {} = 0;\n".format( - fds[0].strip(), fds[1].strip() - ) - elif line.startswith("END_TILING_DATA_DEF"): - tiling_source += "};\n" - tiling_source += "#pragma pack()\n\n" - tiling_source += "#ifdef __NPU_TILING__\n" - tiling_source += "inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n".format( - stru=struct_def - ) - tiling_source += "{\n" - tiling_source += " const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n" - tiling_source += " uint32_t *dst = (uint32_t *)const_data;\n" - tiling_source += " for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n".format( - struct_def - ) - tiling_source += "}\n" - tiling_source += "#else\n" - tiling_source += "inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n".format( - stru=struct_def - ) - tiling_source += "{\n" - tiling_source += " uint64_t *src = (uint64_t *)tiling;\n" - tiling_source += " uint64_t *dst = (uint64_t *)const_data;\n" - tiling_source += " for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n".format( - struct_def - ) - tiling_source += "}\n" - tiling_source += "#endif\n\n" - end_source = """ -#undef GET_TILING_DATA -#define GET_TILING_DATA(tiling_data, tiling_arg) \\ -{stru} tiling_data; \\ -Init{stru}(tiling_arg, &tiling_data)\n -""".format( - stru=struct_def - ) - tiling_source += end_source - tiling_source += "#endif" - with os.fdopen( - os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), "w" - ) as ofd: - ofd.write(tiling_source) - - -if __name__ == "__main__": - if len(sys.argv) <= 2: - raise RuntimeError("arguments must greater than 2") - gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/csrc/deepep/ops2/cmake/func.cmake b/csrc/deepep/ops2/func.cmake similarity index 100% rename from csrc/deepep/ops2/cmake/func.cmake rename to csrc/deepep/ops2/func.cmake diff --git a/csrc/deepep_cmake_build.sh b/csrc/deepep_cmake_build.sh new file mode 100755 index 000000000..07c806ddf --- /dev/null +++ b/csrc/deepep_cmake_build.sh @@ -0,0 +1,75 @@ +#!/bin/bash +echo "Start the cmake script..." +SCRIPT_PATH=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") +export ROOT_PATH=$(dirname "$SCRIPT_PATH") +echo ROOT_PATH: $ROOT_PATH +if [ ! -d "./build_out" ]; then + mkdir build_out +fi +export SRC_PATH="${ROOT_PATH}" +export BUILD_OUT_PATH="${ROOT_PATH}/build_out" +export SCRIPTS_PATH="${ROOT_PATH}" +export TEST_PATH="${ROOT_PATH}/test" + +export BUILD_TYPE="Release" +MODULE_NAME="all" +MODULE_BUILD_ARG="" +IS_MODULE_EXIST=0 + +function PrintHelp() { + echo " +./build.sh [module name] ... +If there are no parameters, all modules are compiled in default mode +module list: [deepep] + +opt: +-d: Enable debug +" +} + +function ProcessArg() { + while getopts "dh" opt; do + case $opt in + d) + export BUILD_TYPE="Debug" + ;; + h) + PrintHelp + exit 0 + ;; + esac + done + shift $(($OPTIND-1)) +} + +function IsModuleName() { + if [ -z "$1" ]; then + return 1 + fi + + if [[ $1 == -* ]]; then + return 1 + else + return 0 + fi +} + +if IsModuleName $@; then + MODULE_NAME=$1 + shift +else + ProcessArg $@ +fi + +if [[ "$MODULE_NAME" == "all" || "$MODULE_NAME" == "deepep" ]]; then + IS_MODULE_EXIST=1 + echo "./deepep/build.sh $@" + ./deepep/build.sh $@ + if [ $? -ne 0 ]; then + exit 1 + fi +fi + +if [ $IS_MODULE_EXIST -eq 0 ]; then + echo "module not exist" +fi