diff --git a/ports/tensorflow-common/convert_lib_params_macos.py b/ports/tensorflow-common/convert_lib_params_macos.py new file mode 100644 index 00000000000000..b3c419363e6cf4 --- /dev/null +++ b/ports/tensorflow-common/convert_lib_params_macos.py @@ -0,0 +1,18 @@ +import sys + +version = sys.argv[1] +lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2] + +with open(f"libtensorflow{lib_suffix}.{version}.a-2.params", "w") as f_out: + parts = [] + with open(f"libtensorflow_framework.{version}.dylib-2.params", "r") as f_in: + for line in f_in: + if line.startswith("-Wl,-force_load,"): + f_out.write(line[16:]) + parts.append(line[16:]) + parts = set(parts) + with open(f"libtensorflow{lib_suffix}.{version}.dylib-2.params", "r") as f_in: + for line in f_in: + if line.startswith("-Wl,-force_load,"): + if line[16:] not in parts: + f_out.write(line[16:]) diff --git a/ports/tensorflow-common/generate_static_link_cmd_linux.py b/ports/tensorflow-common/generate_static_link_cmd_linux.py index 34c8db13400919..7b17552154b6b3 100644 --- a/ports/tensorflow-common/generate_static_link_cmd_linux.py +++ b/ports/tensorflow-common/generate_static_link_cmd_linux.py @@ -4,7 +4,7 @@ lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2] with open(sys.argv[1], "r") as f_in: with open("static_link.sh", "w") as f_out: - p_cd = re.compile("^\\((cd .*) && \\\\$") + p_cd = re.compile(r"^\((cd .*) && \\$") p_linker = re.compile(fr"^\s*(.+)gcc.+(@bazel-out\S+libtensorflow{lib_suffix}\.so\.\d\.\d\.\d-2\.params).*") f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n") env = [] diff --git a/ports/tensorflow-common/generate_static_link_cmd_macos.py b/ports/tensorflow-common/generate_static_link_cmd_macos.py index bb8f6faa21f43a..5894c99b00232c 100644 --- a/ports/tensorflow-common/generate_static_link_cmd_macos.py +++ b/ports/tensorflow-common/generate_static_link_cmd_macos.py @@ -4,44 +4,20 @@ lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2] with open(sys.argv[1], "r") as f_in: with open("static_link.sh", "w") as f_out: - p_cd = re.compile("^\\((cd .*) && \\\\$") - p_linker1 = re.compile(fr"^.*cc_wrapper.sh.+-shared.+-o (bazel-out\S+libtensorflow{lib_suffix}\.\d\.\d\.\d\.dylib)") - p_linker2 = re.compile("^.*cc_wrapper.sh.+-shared.+-o (bazel-out\\S+libtensorflow_framework\\.\\d\\.\\d\\.\\d\\.dylib)") + p_cd = re.compile(r"^\((cd .*) && \\$") + p_linker = re.compile(fr"^\s*.+cc_wrapper.sh.+(@bazel-out\S+libtensorflow{lib_suffix}\.\d\.\d\.\d\.dylib-2\.params).*") f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n") env = [] - parts = None for line in f_in: if line.startswith("(cd"): # new command, reset env = [line] else: - m1 = p_linker1.match(line) - m2 = p_linker2.match(line) + m1 = p_linker.match(line) if m1: - tokens = line.split() - if parts is None: - parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] - else: - m = p_cd.match(env[0]) - f_out.write(m.group(1) + "\n") - tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] - old = set(parts) - parts += [t for t in tmp if t not in old] - line = f"libtool -static -o {m1.group(1).replace('.dylib', '.a')} {' '.join(parts)}\n" - f_out.write(line) - break - elif m2 and len(env) > 6: - tokens = line.split() - if parts is None: - parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] - else: - m = p_cd.match(env[0]) - f_out.write(m.group(1) + "\n") - tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] - old = set(parts) - parts += [t for t in tmp if t not in old] - line = f"libtool -static -o {m2.group(1).replace('_framework', lib_suffix).replace('.dylib', '.a')} {' '.join(parts)}\n" - f_out.write(line) - break + m2 = p_cd.match(env[0]) + f_out.write(m2.group(1) + "\n") + line = f'"/usr/bin/libtool" -static -o {m1.group(1)[1:-9].replace(".dylib", ".a")} {m1.group(1).replace(".dylib", ".a")}\n' + f_out.write(line) else: env.append(line) diff --git a/ports/tensorflow-common/generate_static_link_cmd_windows.py b/ports/tensorflow-common/generate_static_link_cmd_windows.py index 780334ad98ab83..84ec9eede2186a 100644 --- a/ports/tensorflow-common/generate_static_link_cmd_windows.py +++ b/ports/tensorflow-common/generate_static_link_cmd_windows.py @@ -5,7 +5,7 @@ lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2] with open(sys.argv[1], "r") as f_in: with open("static_link.bat", "w") as f_out: - p_setenv = re.compile("^\s*(SET .+=.*)$") + p_setenv = re.compile(r"^\s*(SET .+=.*)$") p_linker = re.compile(fr".+link\.exe.+tensorflow{lib_suffix}\.dll-2\.params.*") env = [] for line in f_in: diff --git a/ports/tensorflow-common/portfile.cmake b/ports/tensorflow-common/portfile.cmake index e505c4430ed748..edd171aa48e322 100644 --- a/ports/tensorflow-common/portfile.cmake +++ b/ports/tensorflow-common/portfile.cmake @@ -3,6 +3,7 @@ set(VCPKG_POLICY_EMPTY_INCLUDE_FOLDER enabled) set(TENSORFLOW_FILES "${CMAKE_CURRENT_LIST_DIR}/change-macros-for-static-lib.patch" "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_linux.py" + "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_macos.py" "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_windows.py" "${CMAKE_CURRENT_LIST_DIR}/fix-build-error.patch" "${CMAKE_CURRENT_LIST_DIR}/fix-linux-build.patch" diff --git a/ports/tensorflow-common/tensorflow-common.cmake b/ports/tensorflow-common/tensorflow-common.cmake index 776cb9c0069829..403f32c6cb8f0e 100644 --- a/ports/tensorflow-common/tensorflow-common.cmake +++ b/ports/tensorflow-common/tensorflow-common.cmake @@ -45,7 +45,16 @@ else() get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY) vcpkg_add_to_path(PREPEND ${PYTHON3_DIR}) - vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET}) + if(VCPKG_TARGET_IS_OSX) + # acceleration libs currently broken on macOS => force numpy user space reinstall without BLAS/LAPACK/ATLAS + # remove this work-around again, i.e. default to "else" branch, once acceleration libs are fixed upstream + set(ENV{BLAS} "None") + set(ENV{LAPACK} "None") + set(ENV{ATLAS} "None") + vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U --force-reinstall numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET}) + else() + vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET}) + endif() vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getusersitepackages())" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH) endif() set(ENV{PYTHON_BIN_PATH} "${PYTHON3}") @@ -265,20 +274,18 @@ foreach(BUILD_TYPE dbg rel) LOGNAME build-${TARGET_TRIPLET}-${BUILD_TYPE} ) endif() - if(NOT VCPKG_TARGET_IS_OSX) - if(VCPKG_TARGET_IS_WINDOWS) - vcpkg_execute_build_process( - COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_LIB_SUFFIX} - WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow - LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE} - ) - else() - vcpkg_execute_build_process( - COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_VERSION} ${TF_LIB_SUFFIX} - WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow - LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE} - ) - endif() + if(VCPKG_TARGET_IS_WINDOWS) + vcpkg_execute_build_process( + COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_LIB_SUFFIX} + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow + LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + else() + vcpkg_execute_build_process( + COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_VERSION} ${TF_LIB_SUFFIX} + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow + LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE} + ) endif() # for some reason stdout of bazel ends up in stderr, so use err log file in the following command vcpkg_execute_build_process( diff --git a/ports/tensorflow-common/vcpkg.json b/ports/tensorflow-common/vcpkg.json index d97e2c228c5518..b0dfa7959bea25 100644 --- a/ports/tensorflow-common/vcpkg.json +++ b/ports/tensorflow-common/vcpkg.json @@ -1,7 +1,7 @@ { "name": "tensorflow-common", "version-semver": "2.4.1", - "port-version": 3, + "port-version": 4, "description": "This meta package holds common files for the C [tensorflow] and the C++ [tensorflow-cc] API version of TensorFlow but is not installable on its own.", "homepage": "https://github.com/tensorflow/tensorflow" } diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 47a65e1fc22c18..dec70ed3b734ba 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1757,7 +1757,3 @@ dimcli:x64-windows-static=fail # cppgraphqlgen triggers an ICE on Apple Clang that comes with MacOS 11. cppgraphqlgen:x64-osx=fail - -# Changes in Python have broken tensorflow on our osx hardware -tensorflow:x64-osx=fail -tensorflow-cc:x64-osx=fail diff --git a/versions/baseline.json b/versions/baseline.json index 39630f5d526a31..cd2851b7aa4537 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -6262,7 +6262,7 @@ }, "tensorflow-common": { "baseline": "2.4.1", - "port-version": 3 + "port-version": 4 }, "tensorpipe": { "baseline": "2021-04-26", diff --git a/versions/t-/tensorflow-common.json b/versions/t-/tensorflow-common.json index cbe798a9e67d36..59c201491da010 100644 --- a/versions/t-/tensorflow-common.json +++ b/versions/t-/tensorflow-common.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "3d84b641f07ef269a213e0382e719a0c338f4ed7", + "version-semver": "2.4.1", + "port-version": 4 + }, { "git-tree": "dd652b405ef53658c13af438e8414110f2977520", "version-semver": "2.4.1",