diff --git a/pkgs/development/python-modules/keras/default.nix b/pkgs/development/python-modules/keras/default.nix index 87f241d3b2ab2..1c4a75be728ea 100644 --- a/pkgs/development/python-modules/keras/default.nix +++ b/pkgs/development/python-modules/keras/default.nix @@ -50,7 +50,7 @@ buildPythonPackage rec { dependencies = [ absl-py h5py - ml-dtypes + ({ ml-dtypesTF = ml-dtypes; } // tensorflow).ml-dtypesTF namex numpy optree diff --git a/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch b/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch deleted file mode 100644 index 1b130e22662e1..0000000000000 --- a/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff -Naurd x/third_party/systemlibs/absl_py.absl.flags.BUILD y/third_party/systemlibs/absl_py.absl.flags.BUILD ---- x/third_party/systemlibs/absl_py.absl.flags.BUILD 2023-09-20 06:02:15.079683729 +0000 -+++ y/third_party/systemlibs/absl_py.absl.flags.BUILD 2023-09-20 06:04:58.491897303 +0000 -@@ -5,3 +5,7 @@ - py_library( - name = "flags", - ) -+ -+py_library( -+ name = "argparse_flags", -+) diff --git a/pkgs/development/python-modules/tensorflow/absl_to_std.patch b/pkgs/development/python-modules/tensorflow/absl_to_std.patch deleted file mode 100644 index b834ec983c94e..0000000000000 --- a/pkgs/development/python-modules/tensorflow/absl_to_std.patch +++ /dev/null @@ -1,389 +0,0 @@ -diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor.cc y/tensorflow/c/experimental/stream_executor/stream_executor.cc ---- x/tensorflow/c/experimental/stream_executor/stream_executor.cc 2023-10-19 14:25:48.648646416 +0000 -+++ y/tensorflow/c/experimental/stream_executor/stream_executor.cc 2023-10-19 14:29:11.700743574 +0000 -@@ -23,6 +23,7 @@ - - #include - #include -+#include - - #include "absl/functional/any_invocable.h" - #include "tensorflow/c/c_api_macros.h" -@@ -275,17 +276,17 @@ - stream_executor_->unified_memory_deallocate(&device_, mem); - } - -- absl::optional GetAllocatorStats() override { -+ std::optional GetAllocatorStats() override { - SP_AllocatorStats c_stats{SP_ALLOCATORSTATS_STRUCT_SIZE}; - TF_Bool has_stats = - stream_executor_->get_allocator_stats(&device_, &c_stats); - if (!has_stats) { -- return absl::nullopt; -+ return std::nullopt; - } - tsl::Status status = ValidateSPAllocatorStats(c_stats); - if (!status.ok()) { - LOG(ERROR) << status.message(); -- return absl::nullopt; -+ return std::nullopt; - } - ::stream_executor::AllocatorStats stats; - stats.num_allocs = c_stats.num_allocs; -diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc ---- x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc 2023-10-19 14:25:48.648646416 +0000 -+++ y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc 2023-10-19 14:29:11.700743574 +0000 -@@ -15,6 +15,7 @@ - #include "tensorflow/c/experimental/stream_executor/stream_executor.h" - - #include -+#include - - #include "tensorflow/c/experimental/stream_executor/stream_executor_internal.h" - #include "tensorflow/c/experimental/stream_executor/stream_executor_test_util.h" -@@ -239,7 +240,7 @@ - }; - - StreamExecutor* executor = GetExecutor(0); -- absl::optional optional_stats = executor->GetAllocatorStats(); -+ std::optional optional_stats = executor->GetAllocatorStats(); - ASSERT_TRUE(optional_stats.has_value()); - AllocatorStats stats = optional_stats.value(); - ASSERT_EQ(stats.bytes_in_use, 123); -diff -Naurd x/tensorflow/compiler/xla/stream_executor/allocator_stats.h y/tensorflow/compiler/xla/stream_executor/allocator_stats.h ---- x/tensorflow/compiler/xla/stream_executor/allocator_stats.h 2023-10-19 14:25:55.064649379 +0000 -+++ y/tensorflow/compiler/xla/stream_executor/allocator_stats.h 2023-10-19 14:29:11.700743574 +0000 -@@ -17,6 +17,7 @@ - #define TENSORFLOW_COMPILER_XLA_STREAM_EXECUTOR_ALLOCATOR_STATS_H_ - - #include -+#include - - #include "absl/types/optional.h" - #include "tensorflow/compiler/xla/stream_executor/platform/port.h" -diff -Naurd x/tensorflow/core/common_runtime/mkl_cpu_allocator.h y/tensorflow/core/common_runtime/mkl_cpu_allocator.h ---- x/tensorflow/core/common_runtime/mkl_cpu_allocator.h 2023-10-19 14:25:58.996651199 +0000 -+++ y/tensorflow/core/common_runtime/mkl_cpu_allocator.h 2023-10-19 14:29:11.700743574 +0000 -@@ -22,6 +22,7 @@ - #ifdef INTEL_MKL - - #include -+#include - - #include "tensorflow/core/common_runtime/bfc_allocator.h" - #include "tensorflow/core/common_runtime/pool_allocator.h" -@@ -80,7 +81,7 @@ - port::AlignedFree(ptr); - } - -- absl::optional GetStats() override { -+ std::optional GetStats() override { - mutex_lock l(mutex_); - return stats_; - } -@@ -242,7 +243,7 @@ - large_size_allocator_->DeallocateRaw(ptr); - } - } -- absl::optional GetStats() override { -+ std::optional GetStats() override { - auto s_stats = small_size_allocator_->GetStats(); - auto l_stats = large_size_allocator_->GetStats(); - -diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc ---- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc 2023-10-19 14:25:59.236651310 +0000 -+++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc 2023-10-19 14:29:11.700743574 +0000 -@@ -32,7 +32,7 @@ - return sub_allocator_->Free(ptr, 0); - } - --absl::optional PluggableDeviceSimpleAllocator::GetStats() { -+std::optional PluggableDeviceSimpleAllocator::GetStats() { - AllocatorStats stats_; - stats_.num_allocs = 0; - stats_.peak_bytes_in_use = 0; -diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h ---- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h 2023-10-19 14:25:59.236651310 +0000 -+++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h 2023-10-19 14:29:11.700743574 +0000 -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include - - #include "tensorflow/core/common_runtime/device/device_mem_allocator.h" - #include "tensorflow/core/platform/thread_annotations.h" -@@ -37,7 +38,7 @@ - - bool TracksAllocationSizes() const override { return false; } - string Name() override { return "Simple allocator"; } -- absl::optional GetStats() override; -+ std::optional GetStats() override; - - AllocatorMemoryType GetMemoryType() const override { - return sub_allocator_->GetMemoryType(); -diff -Naurd x/tensorflow/core/common_runtime/process_state.h y/tensorflow/core/common_runtime/process_state.h ---- x/tensorflow/core/common_runtime/process_state.h 2023-10-19 14:25:59.076651236 +0000 -+++ y/tensorflow/core/common_runtime/process_state.h 2023-10-19 14:29:11.704743576 +0000 -@@ -20,6 +20,7 @@ - #include - #include - #include -+#include - - #include "tensorflow/core/framework/allocator.h" - #include "tensorflow/core/framework/allocator_registry.h" -@@ -144,7 +145,7 @@ - size_t AllocatedSize(const void* p) const override { - return a_->AllocatedSize(p); - } -- absl::optional GetStats() override { return a_->GetStats(); } -+ std::optional GetStats() override { return a_->GetStats(); } - bool ClearStats() override { return a_->ClearStats(); } - - AllocatorMemoryType GetMemoryType() const override { -diff -Naurd x/tensorflow/core/common_runtime/step_stats_collector.cc y/tensorflow/core/common_runtime/step_stats_collector.cc ---- x/tensorflow/core/common_runtime/step_stats_collector.cc 2023-10-19 14:25:59.112651253 +0000 -+++ y/tensorflow/core/common_runtime/step_stats_collector.cc 2023-10-19 14:29:11.704743576 +0000 -@@ -15,6 +15,7 @@ - #include "tensorflow/core/common_runtime/step_stats_collector.h" - - #include -+#include - - #include "tensorflow/core/common_runtime/costmodel_manager.h" - #include "tensorflow/core/framework/allocation_description.pb.h" -@@ -175,7 +176,7 @@ - memory->set_peak_bytes(std::get<1>(sizes)); - memory->set_live_bytes(std::get<2>(sizes)); - -- absl::optional stats = allocator->GetStats(); -+ std::optional stats = allocator->GetStats(); - if (stats) { - memory->set_allocator_bytes_in_use(stats->bytes_in_use); - } -diff -Naurd x/tensorflow/core/framework/allocator_test.cc y/tensorflow/core/framework/allocator_test.cc ---- x/tensorflow/core/framework/allocator_test.cc 2023-10-19 14:25:59.524651443 +0000 -+++ y/tensorflow/core/framework/allocator_test.cc 2023-10-19 14:29:11.704743576 +0000 -@@ -17,6 +17,7 @@ - - #include - #include -+#include - - #include "tensorflow/core/framework/typed_allocator.h" - #include "tensorflow/core/platform/logging.h" -@@ -33,7 +34,7 @@ - - static void CheckStats(Allocator* a, int64_t num_allocs, int64_t bytes_in_use, - int64_t peak_bytes_in_use, int64_t largest_alloc_size) { -- absl::optional stats = a->GetStats(); -+ std::optional stats = a->GetStats(); - EXPECT_TRUE(stats); - if (!stats) { - return; -@@ -255,7 +256,7 @@ - EXPECT_EQ(e0.Name(), "MemoryAllocation") - << "XSpace: " << xspace.DebugString(); - { -- absl::optional bytes_allocated, peak_bytes_in_use, -+ std::optional bytes_allocated, peak_bytes_in_use, - requested_bytes, allocation_bytes; - e0.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) { - LOG(ERROR) << "STAT " << stat.Name() << ": " << stat.ToString(); -@@ -282,7 +283,7 @@ - EXPECT_EQ(e1.Name(), "MemoryDeallocation") - << "XSpace: " << xspace.DebugString(); - { -- absl::optional bytes_allocated, peak_bytes_in_use, -+ std::optional bytes_allocated, peak_bytes_in_use, - allocation_bytes; - e1.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) { - if (stat.Name() == "bytes_allocated") { -diff -Naurd x/tensorflow/core/framework/tracking_allocator_test.cc y/tensorflow/core/framework/tracking_allocator_test.cc ---- x/tensorflow/core/framework/tracking_allocator_test.cc 2023-10-19 14:25:59.700651525 +0000 -+++ y/tensorflow/core/framework/tracking_allocator_test.cc 2023-10-19 14:29:11.704743576 +0000 -@@ -16,6 +16,7 @@ - #include "tensorflow/core/framework/tracking_allocator.h" - - #include -+#include - - #include "tensorflow/core/framework/allocator.h" - #include "tensorflow/core/platform/logging.h" -@@ -44,7 +45,7 @@ - EXPECT_NE(size_map_.end(), iter); - return iter->second; - } -- absl::optional GetStats() override { return absl::nullopt; } -+ std::optional GetStats() override { return std::nullopt; } - - private: - std::unordered_map size_map_; -@@ -58,7 +59,7 @@ - } - void DeallocateRaw(void* ptr) override {} - bool TracksAllocationSizes() const override { return true; } -- absl::optional GetStats() override { return absl::nullopt; } -+ std::optional GetStats() override { return std::nullopt; } - }; - - TEST(TrackingAllocatorTest, SimpleNoTracking) { -diff -Naurd x/tensorflow/core/grappler/clusters/single_machine.cc y/tensorflow/core/grappler/clusters/single_machine.cc ---- x/tensorflow/core/grappler/clusters/single_machine.cc 2023-10-19 14:25:59.964651648 +0000 -+++ y/tensorflow/core/grappler/clusters/single_machine.cc 2023-10-19 14:29:11.704743576 +0000 -@@ -17,6 +17,7 @@ - - #include - #include -+#include - - #include "tensorflow/cc/training/queue_runner.h" - #include "tensorflow/core/common_runtime/device.h" -@@ -230,7 +231,7 @@ - return Status(absl::StatusCode::kInvalidArgument, - "Tracking allocation is not enabled."); - } -- absl::optional stats = allocator->GetStats(); -+ std::optional stats = allocator->GetStats(); - (*device_peak_memory)[device->name()] = - (stats ? stats->peak_bytes_in_use : 0); - } -diff -Naurd x/tensorflow/core/kernels/stack.cc y/tensorflow/core/kernels/stack.cc ---- x/tensorflow/core/kernels/stack.cc 2023-10-19 14:26:01.668652437 +0000 -+++ y/tensorflow/core/kernels/stack.cc 2023-10-19 14:29:11.704743576 +0000 -@@ -18,6 +18,7 @@ - #include - #include - #include -+#include - - #include "tensorflow/core/common_runtime/device.h" - #include "tensorflow/core/framework/device_base.h" -@@ -245,7 +246,7 @@ - DeviceContext* device_ctxt = ctx->op_device_context(); - auto device = static_cast(ctx->device()); - Allocator* allocator = device->GetAllocator(alloc_attrs); -- absl::optional stats = allocator->GetStats(); -+ std::optional stats = allocator->GetStats(); - if (stats && *stats->bytes_limit && - stats->bytes_in_use > (*stats->bytes_limit * kOccupancy)) { - // Asynchronously copy the tensor from GPU to CPU memory. -diff -Naurd x/tensorflow/python/tfe_wrapper.cc y/tensorflow/python/tfe_wrapper.cc ---- x/tensorflow/python/tfe_wrapper.cc 2023-10-19 14:26:10.716656639 +0000 -+++ y/tensorflow/python/tfe_wrapper.cc 2023-10-19 14:29:11.708743578 +0000 -@@ -14,6 +14,7 @@ - ==============================================================================*/ - - #include -+#include - - #include "Python.h" - #include "absl/strings/match.h" -@@ -691,7 +692,7 @@ - tensorflow::AllocatorAttributes attrs; - tensorflow::Allocator* allocator = matched_device->GetAllocator(attrs); - -- if (absl::optional stats = -+ if (std::optional stats = - allocator->GetStats()) { - return std::map{{"current", stats->bytes_in_use}, - {"peak", stats->peak_bytes_in_use}}; -diff -Naurd x/tensorflow/tsl/framework/allocator.h y/tensorflow/tsl/framework/allocator.h ---- x/tensorflow/tsl/framework/allocator.h 2023-10-19 14:26:15.884659044 +0000 -+++ y/tensorflow/tsl/framework/allocator.h 2023-10-19 14:29:11.708743578 +0000 -@@ -216,7 +216,7 @@ - } - - // Fills in 'stats' with statistics collected by this allocator. -- virtual absl::optional GetStats() { return absl::nullopt; } -+ virtual std::optional GetStats() { return std::nullopt; } - - // If implemented, clears the internal stats except for the `in_use` fields - // and sets the `peak_bytes_in_use` to be equal to the `bytes_in_use`. Returns -diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.cc y/tensorflow/tsl/framework/bfc_allocator.cc ---- x/tensorflow/tsl/framework/bfc_allocator.cc 2023-10-19 14:26:15.900659052 +0000 -+++ y/tensorflow/tsl/framework/bfc_allocator.cc 2023-10-19 14:29:11.708743578 +0000 -@@ -1205,7 +1205,7 @@ - return md; - } - --absl::optional BFCAllocator::GetStats() { -+std::optional BFCAllocator::GetStats() { - mutex_lock l(lock_); - return stats_; - } -diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.h y/tensorflow/tsl/framework/bfc_allocator.h ---- x/tensorflow/tsl/framework/bfc_allocator.h 2023-10-19 14:26:15.900659052 +0000 -+++ y/tensorflow/tsl/framework/bfc_allocator.h 2023-10-19 14:29:11.708743578 +0000 -@@ -22,6 +22,7 @@ - #include - #include - #include -+#include - - #include "absl/container/flat_hash_set.h" - #include "tensorflow/tsl/framework/allocator.h" -@@ -93,7 +94,7 @@ - - int64_t AllocationId(const void* ptr) const override; - -- absl::optional GetStats() override; -+ std::optional GetStats() override; - - bool ClearStats() override; - -diff -Naurd x/tensorflow/tsl/framework/cpu_allocator_impl.cc y/tensorflow/tsl/framework/cpu_allocator_impl.cc ---- x/tensorflow/tsl/framework/cpu_allocator_impl.cc 2023-10-19 14:26:15.928659065 +0000 -+++ y/tensorflow/tsl/framework/cpu_allocator_impl.cc 2023-10-19 14:29:11.708743578 +0000 -@@ -15,6 +15,7 @@ - - #include - #include -+#include - - #include "tensorflow/tsl/framework/allocator.h" - #include "tensorflow/tsl/framework/allocator_registry.h" -@@ -145,8 +146,8 @@ - /*level=*/tsl::profiler::TraceMeLevel::kInfo); - } - -- absl::optional GetStats() override { -- if (!cpu_allocator_collect_stats) return absl::nullopt; -+ std::optional GetStats() override { -+ if (!cpu_allocator_collect_stats) return std::nullopt; - mutex_lock l(mu_); - return stats_; - } -diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.cc y/tensorflow/tsl/framework/tracking_allocator.cc ---- x/tensorflow/tsl/framework/tracking_allocator.cc 2023-10-19 14:26:15.968659084 +0000 -+++ y/tensorflow/tsl/framework/tracking_allocator.cc 2023-10-19 14:29:11.708743578 +0000 -@@ -152,7 +152,7 @@ - } - } - --absl::optional TrackingAllocator::GetStats() { -+std::optional TrackingAllocator::GetStats() { - return allocator_->GetStats(); - } - -diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.h y/tensorflow/tsl/framework/tracking_allocator.h ---- x/tensorflow/tsl/framework/tracking_allocator.h 2023-10-19 14:26:15.968659084 +0000 -+++ y/tensorflow/tsl/framework/tracking_allocator.h 2023-10-19 14:29:11.712743580 +0000 -@@ -17,6 +17,7 @@ - #define TENSORFLOW_TSL_FRAMEWORK_TRACKING_ALLOCATOR_H_ - - #include -+#include - - #include "tensorflow/tsl/framework/allocator.h" - #include "tensorflow/tsl/lib/gtl/inlined_vector.h" -@@ -66,7 +67,7 @@ - size_t RequestedSize(const void* ptr) const override; - size_t AllocatedSize(const void* ptr) const override; - int64_t AllocationId(const void* ptr) const override; -- absl::optional GetStats() override; -+ std::optional GetStats() override; - bool ClearStats() override; - - AllocatorMemoryType GetMemoryType() const override { diff --git a/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch index ebb2fb129c06c..b9f0901c9b6c8 100644 --- a/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch +++ b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch @@ -1,7 +1,16 @@ diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system.absl.base.BUILD --- x/third_party/absl/system.absl.base.BUILD 2023-09-17 09:12:05.499753364 +0000 +++ y/third_party/absl/system.absl.base.BUILD 2023-09-17 09:16:01.200082822 +0000 -@@ -22,7 +22,12 @@ +@@ -12,6 +12,8 @@ + "atomic_hook", + "errno_saver", + "fast_type_id", ++ "nullability", ++ "prefetch", + "pretty_function", + ]] + +@@ -22,7 +24,13 @@ cc_library( name = "raw_logging_internal", @@ -11,14 +20,40 @@ diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system. + "-labsl_log_internal_conditions", + "-labsl_log_internal_message", + "-labsl_log_internal_nullguard", ++ "-labsl_log_internal_check_op", + ], visibility = [ "//absl:__subpackages__", ], +diff -Naurd x/third_party/absl/system.absl.crc.BUILD y/third_party/absl/system.absl.crc.BUILD +--- x/third_party/absl/system.absl.crc.BUILD 1970-01-01 00:00:00.000000000 +0000 ++++ y/third_party/absl/system.absl.crc.BUILD 2024-10-10 20:08:56.809477492 +0000 +@@ -0,0 +1,7 @@ ++load("@rules_cc//cc:defs.bzl", "cc_library") ++ ++cc_library( ++ name = "crc32c", ++ linkopts = ["-labsl_crc32c"], ++ visibility = ["//visibility:public"], ++) +diff -Naurd x/third_party/absl/system.absl.functional.BUILD y/third_party/absl/system.absl.functional.BUILD +--- x/third_party/absl/system.absl.functional.BUILD 2024-10-10 10:15:40.640185040 +0000 ++++ y/third_party/absl/system.absl.functional.BUILD 2024-10-10 10:17:59.221375986 +0000 +@@ -4,10 +4,6 @@ + + cc_library( + name = "any_invocable", +-) +- +-cc_library( +- name = "any_invocable", + deps = [ + "//absl/base:base_internal", + "//absl/base:config", diff -Naurd x/third_party/absl/system.absl.log.BUILD y/third_party/absl/system.absl.log.BUILD --- x/third_party/absl/system.absl.log.BUILD 1970-01-01 00:00:00.000000000 +0000 +++ y/third_party/absl/system.absl.log.BUILD 2023-09-17 09:12:11.795762177 +0000 -@@ -0,0 +1,134 @@ +@@ -0,0 +1,135 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) @@ -43,6 +78,7 @@ diff -Naurd x/third_party/absl/system.absl.log.BUILD y/third_party/absl/system.a + +cc_library( + name = "die_if_null", ++ linkopts = ["-labsl_die_if_null"], + deps = [ + ":log", + "//absl/base:config", @@ -153,10 +189,29 @@ diff -Naurd x/third_party/absl/system.absl.log.BUILD y/third_party/absl/system.a + "//absl/strings", + ], +) +diff -Naurd x/third_party/absl/system.absl.strings.BUILD y/third_party/absl/system.absl.strings.BUILD +--- x/third_party/absl/system.absl.strings.BUILD 2024-10-10 10:15:40.644185074 +0000 ++++ y/third_party/absl/system.absl.strings.BUILD 2024-10-10 18:52:37.648334488 +0000 +@@ -3,6 +3,10 @@ + package(default_visibility = ["//visibility:public"]) + + cc_library( ++ name = "string_view", ++) ++ ++cc_library( + name = "strings", + linkopts = ["-labsl_strings"], + deps = [ diff -Naurd x/third_party/absl/workspace.bzl y/third_party/absl/workspace.bzl ---- x/third_party/absl/workspace.bzl 2023-09-17 09:12:05.499753364 +0000 -+++ y/third_party/absl/workspace.bzl 2023-09-17 09:12:11.795762177 +0000 -@@ -20,6 +20,7 @@ +--- x/third_party/absl/workspace.bzl 2024-10-10 10:16:56.508840929 +0000 ++++ y/third_party/absl/workspace.bzl 2024-10-10 20:09:16.685514935 +0000 +@@ -16,10 +16,12 @@ + "base", + "cleanup", + "container", ++ "crc", + "debugging", "flags", "functional", "hash", diff --git a/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch b/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch new file mode 100644 index 0000000000000..8129b0119bd71 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch @@ -0,0 +1,11 @@ +diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl +--- x/tensorflow/workspace2.bzl 2023-12-07 09:42:06.557980143 +0000 ++++ y/tensorflow/workspace2.bzl 2023-12-07 09:43:50.613924139 +0000 +@@ -862,6 +862,7 @@ + tf_http_archive( + name = "riegeli", + sha256 = "870ca080cdfc5eba696a72ccc3a54cbf0f2271befc0d459eafa8f065edfaadb2", ++ patch_file = ["//third_party:riegeli-proto.patch"], + strip_prefix = "riegeli-264ef7b4a1314d97265b37544b27cd3923ea72d2", + urls = tf_mirror_urls("https://github.com/google/riegeli/archive/264ef7b4a1314d97265b37544b27cd3923ea72d2.zip"), + ) diff --git a/pkgs/development/python-modules/tensorflow/core-rules-python-use-nix.patch b/pkgs/development/python-modules/tensorflow/core-rules-python-use-nix.patch new file mode 100644 index 0000000000000..366d1d5937bb4 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/core-rules-python-use-nix.patch @@ -0,0 +1,13 @@ +diff -Naurd x/WORKSPACE y/WORKSPACE +--- x/WORKSPACE 2023-11-16 08:55:37.370373002 +0000 ++++ y/WORKSPACE 2023-11-20 01:55:57.812931627 +0000 +@@ -17,6 +17,9 @@ + sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", + strip_prefix = "rules_python-0.26.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", ++ patches = [ ++ "//tensorflow:rules-python-use-nix.patch" ++ ], + ) + + load("@rules_python//python:repositories.bzl", "python_register_toolchains") diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index ac2430d1a2b64..d60a57eee5d16 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -1,6 +1,6 @@ { stdenv, - bazel_5, + bazel_6, buildBazelPackage, lib, fetchFromGitHub, @@ -14,6 +14,7 @@ pythonAtLeast, pythonOlder, python, + pythonRelaxDepsHook, # Python libraries numpy, tensorboard, @@ -27,6 +28,7 @@ opt-einsum, astunparse, h5py, + ml-dtypes, termcolor, grpcio, six, @@ -38,6 +40,11 @@ portpicker, tblib, typing-extensions, + distutils, + charset-normalizer, + markupsafe, + psutil, + dm-tree, # Common deps git, pybind11, @@ -46,6 +53,12 @@ glibcLocales, cython, perl, + llvmPackages, + pkg-config, + cmake, + lndir, + writeShellScriptBin, + writeText, # Common libraries jemalloc, mpi, @@ -64,6 +77,7 @@ libjpeg_turbo, giflib, protobuf-core, + hdf5, # Upstream by default includes cuda support since tensorflow 1.15. We could do # that in nix as well. It would make some things easier and less confusing, but # it would also make the default tensorflow package unfree. See @@ -84,7 +98,7 @@ Foundation, Security, cctools, - llvmPackages, + xcbuild, }: let @@ -115,7 +129,7 @@ let # use compatible cuDNN (https://www.tensorflow.org/install/source#gpu) # cudaPackages.cudnn led to this: # https://github.com/tensorflow/tensorflow/issues/60398 - cudnnAttribute = "cudnn_8_6"; + cudnnAttribute = "cudnn_8_7"; cudnnMerged = symlinkJoin { name = "cudnn-merged"; paths = [ @@ -181,7 +195,7 @@ let tfFeature = x: if x then "1" else "0"; - version = "2.13.0"; + version = "2.16.2"; format = "setuptools"; variant = lib.optionalString cudaSupport "-gpu"; pname = "tensorflow${variant}"; @@ -192,6 +206,7 @@ let absl-py astunparse dill + distutils flatbuffers-python gast google-pasta @@ -213,6 +228,11 @@ let wrapt ]); + python_major = lib.take 2 (lib.splitString "." python.version); + python_dotted = lib.concatStringsSep "." python_major; + python_underscored = lib.concatStringsSep "_" python_major; + python_undotted = lib.concatStringsSep "" python_major; + rules_cc_darwin_patched = stdenv.mkDerivation { name = "rules_cc-${pname}-${version}"; @@ -273,9 +293,348 @@ let runHook postInstall ''; }; + + python-x86_64-nix = stdenv.mkDerivation { + name = "python-x86_64-${pname}-${version}"; + + src = _bazel-build.deps; + + dontConfigure = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + + mkdir $out + cp python_x86_64-unknown-linux-gnu/BUILD.bazel $out + substituteInPlace $out/BUILD.bazel \ + --replace-fail @version_long@ ${python_dotted} \ + --replace-fail @version_short@ ${python_undotted} + cp python_x86_64-unknown-linux-gnu/WORKSPACE $out + ${lndir}/bin/lndir ${pythonEnv} $out + + runHook postInstall + ''; + }; + + python-aarch64-nix = stdenv.mkDerivation { + name = "python-aarch64-${pname}-${version}"; + + src = _bazel-build.deps; + + dontConfigure = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + + mkdir $out + cp python_aarch64-unknown-linux-gnu/BUILD.bazel $out + substituteInPlace $out/BUILD.bazel \ + --replace-fail @version_long@ ${python_dotted} \ + --replace-fail @version_short@ ${python_undotted} + cp python_aarch64-unknown-linux-gnu/WORKSPACE $out + ${lndir}/bin/lndir ${pythonEnv} $out + + runHook postInstall + ''; + }; + + # In order to keep the deps hash stable, we put everything that depends + # on the Python version into overrides. + # TODO: figure out whether we can create these overrides without building + # the deps tarball first, e.g. using our Nix-vendored packages and + # `rules_python` + nccl_archive-fix = stdenv.mkDerivation { + name = "nccl_archive-fix"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp -r nccl_archive/* $out + pushd $out + rm BUILD.bazel + ln -s ${_bazel-build.src}/third_party/nccl/archive.BUILD BUILD.bazel + popd + runHook postInstall + ''; + }; + + eigen_archive-fix = stdenv.mkDerivation { + name = "eigen_archive-fix"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp -r eigen_archive/* $out + pushd $out + rm BUILD.bazel + ln -s ${_bazel-build.src}/third_party/eigen3/eigen_archive.BUILD \ + BUILD.bazel + popd + runHook postInstall + ''; + }; + + ml_dtypes-fix = stdenv.mkDerivation { + name = "ml_dtypes-fix"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp -r ml_dtypes/* $out + pushd $out + rm BUILD.bazel + ln -s ${_bazel-build.src}/third_party/py/ml_dtypes/ml_dtypes.BUILD \ + BUILD.bazel + rm tests/BUILD.bazel + ln -s ${_bazel-build.src}/third_party/py/ml_dtypes/ml_dtypes.tests.BUILD \ + tests/BUILD.bazel + popd + runHook postInstall + ''; + }; + + charset-normalizer-bazel = stdenv.mkDerivation { + name = "${charset-normalizer.pname}-bazel-${charset-normalizer.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_charset_normalizer/* $out + mkdir $out/bin $out/site-packages + ${lndir}/bin/lndir ${charset-normalizer}/bin $out/bin + ${lndir}/bin/lndir \ + ${charset-normalizer}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + numpy-bazel = stdenv.mkDerivation { + name = "${numpy.pname}-bazel-${numpy.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_numpy/WORKSPACE $out + cp pypi_numpy/rules_python_wheel_entry_point_f2py.py $out + cp pypi_numpy/BUILD.bazel $out + mkdir $out/bin $out/site-packages + ${lndir}/bin/lndir ${numpy}/bin $out/bin + ${lndir}/bin/lndir ${numpy}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + grpcio-bazel = stdenv.mkDerivation { + name = "${grpcio.pname}-bazel-${grpcio.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_grpcio/* $out + mkdir $out/site-packages + ${lndir}/bin/lndir ${grpcio}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + h5py-bazel = stdenv.mkDerivation { + name = "${h5py.pname}-bazel-${h5py.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_h5py/* $out + mkdir $out/site-packages + ${lndir}/bin/lndir ${h5py}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + wrapt-bazel = stdenv.mkDerivation { + name = "${wrapt.pname}-bazel-${wrapt.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_wrapt/* $out + mkdir $out/site-packages + ${lndir}/bin/lndir ${wrapt}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + markupsafe-bazel = stdenv.mkDerivation { + name = "${markupsafe.pname}-bazel-${markupsafe.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_markupsafe/* $out + mkdir $out/site-packages + ${lndir}/bin/lndir \ + ${markupsafe}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + psutil-bazel = stdenv.mkDerivation { + name = "${psutil.pname}-bazel-${psutil.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_psutil/* $out + mkdir $out/site-packages + ${lndir}/bin/lndir \ + ${psutil}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + dm-tree-bazel = stdenv.mkDerivation { + name = "${dm-tree.pname}-bazel-${dm-tree.version}"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi_dm_tree/* $out + mkdir $out/site-packages + ${lndir}/bin/lndir \ + ${dm-tree}/lib/python${python_dotted}/site-packages \ + $out/site-packages + runHook postInstall + ''; + }; + + pypi-bazel = stdenv.mkDerivation { + name = "pypi-bazel"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp pypi/* $out + substituteInPlace $out/requirements.bzl \ + --replace-fail @version_underscored@ ${python_underscored} + runHook postInstall + ''; + }; + + python-bazel = stdenv.mkDerivation { + name = "python-bazel"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp python/* $out + runHook postInstall + ''; + }; + + python_version_repo-bazel = stdenv.mkDerivation { + name = "python-bazel"; + src = _bazel-build.deps; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + cp python_version_repo/* $out + substituteInPlace $out/py_version.bzl \ + --replace-fail @version_long@ ${python_dotted} + runHook postInstall + ''; + }; + + # https://stackoverflow.com/questions/70301375/noinline-macro-conflict-between-glib-and-cuda + cuda_gnu_wrapper_includes = stdenv.mkDerivation { + name = "wrap-gnu-lib"; + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir $out + for wrapping in regex memory string condition_variable + do echo "#ifdef NIX_TF_WRAPPED + #include_next <$wrapping> + #else + #define NIX_TF_WRAPPED 1 + #undef __noinline__ + #include_next <$wrapping> + #define __noinline__ __attribute__((noinline)) + #undef NIX_TF_WRAPPED + #endif" > $out/$wrapping + done + runHook postInstall + ''; + }; + + clang_lib = llvmPackages.libclang.lib; + + clang_major = builtins.elemAt (lib.splitString "." clang_lib.version) 0; + + cuda_wrappers = "${clang_lib}/lib/clang/${clang_major}/include/cuda_wrappers"; + + clang_gnu_wrapper = writeShellScriptBin "clang" '' + ${llvmPackages.clang}/bin/clang \ + -isystem ${cuda_gnu_wrapper_includes} \ + "$@" + ''; + + cuda_clang_wrapper = writeShellScriptBin "clang" '' + ${llvmPackages.clang}/bin/clang \ + -isystem ${cuda_gnu_wrapper_includes} \ + -isystem ${cuda_wrappers} \ + "$@" + ''; + + switching_clang_wrapper = writeShellScriptBin "clang" '' + if echo " $@ " | grep -q " -x \?cuda " + then ${cuda_clang_wrapper}/bin/clang "$@" + else ${clang_gnu_wrapper}/bin/clang "$@" + fi + ''; + + wrapped_clang = if cudaSupport then switching_clang_wrapper else llvmPackages.clang; + bazel-build = if stdenv.hostPlatform.isDarwin then - _bazel-build.overrideAttrs (prev: { + _bazel-build-python.overrideAttrs (prev: { bazelFlags = prev.bazelFlags ++ [ "--override_repository=rules_cc=${rules_cc_darwin_patched}" "--override_repository=llvm-raw=${llvm-raw_darwin_patched}" @@ -284,31 +643,84 @@ let export AR="${cctools}/bin/libtool" ''; }) + else + _bazel-build-python; + + _bazel-build-python = + if (!stdenv.isDarwin) then + _bazel-build.overrideAttrs (prev: { + bazelFlags = + prev.bazelFlags + ++ [ + ( + if stdenv.isx86_64 then + "--override_repository=python_x86_64-unknown-linux-gnu=${python-x86_64-nix}" + else + "--override_repository=python_aarch64-unknown-linux-gnu=${python-aarch64-nix}" + ) + "--override_repository=pypi_charset_normalizer=${charset-normalizer-bazel}" + "--override_repository=pypi_numpy=${numpy-bazel}" + "--override_repository=pypi_grpcio=${grpcio-bazel}" + "--override_repository=pypi_h5py=${h5py-bazel}" + "--override_repository=pypi_wrapt=${wrapt-bazel}" + "--override_repository=pypi_markupsafe=${markupsafe-bazel}" + "--override_repository=pypi_dm_tree=${dm-tree-bazel}" + "--override_repository=pypi=${pypi-bazel}" + "--override_repository=python=${python-bazel}" + "--override_repository=python_version_repo=${python_version_repo-bazel}" + ] + ++ lib.optionals stdenv.isAarch64 [ + "--override_repository=pypi_psutil=${psutil-bazel}" + ] + ++ lib.optionals cudaSupport [ + "--override_repository=nccl_archive=${nccl_archive-fix}" + "--override_repository=eigen_archive=${eigen_archive-fix}" + "--override_repository=ml_dtypes=${ml_dtypes-fix}" + ]; + }) else _bazel-build; + regenerate-compressed = writeText "regenerate.py" '' + from zlib import compress + from base64 import b64encode + with open("find_cuda_config.py", "r") as file: + input_text = file.read() + output_data = b64encode(compress(str.encode(input_text))) + with open("find_cuda_config.py.gz.base64", "w") as file: + file.write(output_data.decode("ASCII")) + ''; + _bazel-build = buildBazelPackage.override { inherit stdenv; } { name = "${pname}-${version}"; - bazel = bazel_5; + bazel = bazel_6; src = fetchFromGitHub { owner = "tensorflow"; repo = "tensorflow"; tag = "v${version}"; - hash = "sha256-Rq5pAVmxlWBVnph20fkAwbfy+iuBNlfFy14poDPd5h0="; + hash = "sha256-yBDS9DFkVLIT+bu/vqh/0U60UwoIzqlTKQsflM7GWCA="; }; # On update, it can be useful to steal the changes from gentoo # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow - nativeBuildInputs = [ - which - pythonEnv - cython - perl - protobuf-core - protobuf-extra - ] ++ lib.optional cudaSupport addDriverRunpath; + nativeBuildInputs = + [ + pkg-config + hdf5 + which + pythonEnv + cython + perl + protobuf-core + protobuf-extra + wrapped_clang + lndir + ] + ++ lib.optional cudaSupport addDriverRunpath + ++ lib.optional stdenv.isDarwin xcbuild + ++ lib.optional stdenv.isAarch64 cmake; buildInputs = [ @@ -330,9 +742,14 @@ let jsoncpp libjpeg_turbo libpng - (pybind11.overridePythonAttrs (_: { - inherit stdenv; - })) + ( + if stdenv.hostPlatform.isDarwin then + pybind11 + else + (pybind11.overridePythonAttrs (_: { + inherit stdenv; + })) + ) snappy sqlite ] @@ -382,7 +799,6 @@ let "jsoncpp_git" "libjpeg_turbo" "nasm" - "opt_einsum_archive" "org_sqlite" "pasta" "png" @@ -412,6 +828,7 @@ let TF_NEED_GCP = true; TF_NEED_HDFS = true; TF_ENABLE_XLA = tfFeature xlaSupport; + TF_PYTHON_VERSION = python.pythonVersion; CC_OPT_FLAGS = " "; @@ -426,39 +843,68 @@ let GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin"; GCC_HOST_COMPILER_PATH = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin/cc"; - patches = [ - "${gentoo-patches}/0002-systemlib-Latest-absl-LTS-has-split-cord-libs.patch" - "${gentoo-patches}/0005-systemlib-Updates-for-Abseil-20220623-LTS.patch" - "${gentoo-patches}/0007-systemlibs-Add-well_known_types_py_pb2-target.patch" - # https://github.com/conda-forge/tensorflow-feedstock/pull/329/commits/0a63c5a962451b4da99a9948323d8b3ed462f461 - (fetchpatch { - name = "fix-layout-proto-duplicate-loading.patch"; - url = "https://raw.githubusercontent.com/conda-forge/tensorflow-feedstock/0a63c5a962451b4da99a9948323d8b3ed462f461/recipe/patches/0001-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch"; - hash = "sha256-/7buV6DinKnrgfqbe7KKSh9rCebeQdXv2Uj+Xg/083w="; - }) - ./com_google_absl_add_log.patch - ./absl_py_argparse_flags.patch - ./protobuf_python.patch - ./pybind11_protobuf_python_runtime_dep.patch - ./pybind11_protobuf_newer_version.patch - ] ++ lib.optionals (stdenv.hostPlatform.system == "aarch64-darwin") [ ./absl_to_std.patch ]; + patches = + [ + "${gentoo-patches}/0002-systemlib-Latest-absl-LTS-has-split-cord-libs.patch" + "${gentoo-patches}/0005-systemlib-Updates-for-Abseil-20220623-LTS.patch" + "${gentoo-patches}/0007-systemlibs-Add-well_known_types_py_pb2-target.patch" + # https://github.com/conda-forge/tensorflow-feedstock/pull/329/commits/0a63c5a962451b4da99a9948323d8b3ed462f461 + (fetchpatch { + name = "fix-layout-proto-duplicate-loading.patch"; + url = "https://raw.githubusercontent.com/conda-forge/tensorflow-feedstock/0a63c5a962451b4da99a9948323d8b3ed462f461/recipe/patches/0001-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch"; + hash = "sha256-/7buV6DinKnrgfqbe7KKSh9rCebeQdXv2Uj+Xg/083w="; + }) + ./fix-syslib-references.patch + ./protobuf_lite.patch + ./protobuf_cc_toolchain.patch + ./pybind11-osx.patch + ./com_google_absl_add_log.patch + ./core-riegeli-proto.patch + ./protobuf_python.patch + ./pybind11_protobuf_python_runtime_dep.patch + ./pybind11_protobuf_newer_version.patch + ] + ++ lib.optionals (!stdenv.isDarwin) [ + # we override Python in the bazel build anyway, but we also need + # a usable Python during deps fetching + ./core-rules-python-use-nix.patch + ] + ++ lib.optionals (cudaSupport) [ + ./system_includes_cuda.patch + ./fix-string-syntax.patch + ]; postPatch = '' # bazel 3.3 should work just as well as bazel 3.1 rm -f .bazelversion patchShebangs . + ln -s ${./riegeli-proto.patch} third_party/riegeli-proto.patch + '' + + lib.optionalString (!stdenv.isDarwin) '' + sed \ + -e 's,@python@,${pythonEnv},g' \ + -e 's,@python_version@,${python.version},g' \ + -e 's,@version_long@,${python_dotted},g' \ + < ${./rules-python-use-nix.patch} \ + > tensorflow/rules-python-use-nix.patch '' + lib.optionalString (!withTensorboard) '' # Tensorboard pulls in a bunch of dependencies, some of which may # include security vulnerabilities. So we make it optional. # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560 sed -i '/tensorboard ~=/d' tensorflow/tools/pip_package/setup.py + '' + + lib.optionalString (cudaSupport) '' + substituteInPlace third_party/gpus/cuda_configure.bzl \ + --replace-fail "@cuda_wrapper_includes@" "${cuda_wrappers}" + substituteInPlace tensorflow/tools/pip_package/BUILD \ + --replace-fail '"@ml_dtypes//:LICENSE",' "" + pushd third_party/gpus + ${python}/bin/python3 ${regenerate-compressed} + popd ''; - # https://github.com/tensorflow/tensorflow/pull/39470 - env.NIX_CFLAGS_COMPILE = toString [ "-Wno-stringop-truncation" ]; - preConfigure = let opt_flags = @@ -476,12 +922,10 @@ let chmod +x dummy-ldconfig/ldconfig export PATH="$PWD/dummy-ldconfig:$PATH" + export PYTHONPATH="${pythonEnv}/lib/${pythonEnv.libPrefix}/site-packages" export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages" export CC_OPT_FLAGS="${lib.concatStringsSep " " opt_flags}" mkdir -p "$PYTHON_LIB_PATH" - - # To avoid mixing Python 2 and Python 3 - unset PYTHONPATH ''; configurePhase = '' @@ -490,22 +934,24 @@ let runHook postConfigure ''; - hardeningDisable = [ "format" ]; - - bazelBuildFlags = + hardeningDisable = [ - "--config=opt" # optimize using the flags set in the configure phase + "format" ] - ++ lib.optionals stdenv.cc.isClang [ - "--cxxopt=-x" - "--cxxopt=c++" - "--host_cxxopt=-x" - "--host_cxxopt=c++" - - # workaround for https://github.com/bazelbuild/bazel/issues/15359 - "--spawn_strategy=sandboxed" - ] - ++ lib.optionals (mklSupport) [ "--config=mkl" ]; + ++ lib.optionals (cudaSupport) [ + "zerocallusedregs" + ]; + + bazelBuildFlags = [ + "--config=opt" # optimize using the flags set in the configure phase + "--cxxopt=-x" + "--cxxopt=c++" + "--host_cxxopt=-x" + "--host_cxxopt=c++" + + # workaround for https://github.com/bazelbuild/bazel/issues/15359 + "--spawn_strategy=sandboxed" + ] ++ lib.optionals (mklSupport) [ "--config=mkl" ]; bazelTargets = [ "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow" @@ -516,20 +962,72 @@ let dontAddBazelOpts = true; fetchAttrs = { + preInstall = lib.optionalString (!stdenv.isDarwin) ( + '' + pushd $bazelOut + rm -rf external/*/*.whl + find external -type f | grep "/__pycache__/[^/]*[.]pyc\?$" | xargs rm + find external -type l | grep "/__pycache__/[^/]*[.]pyc\?$" | xargs rm + find external -type d | grep "/__pycache__$" | xargs rmdir + rm -rf external/pypi_charset_normalizer/bin + rm -rf external/pypi_charset_normalizer/site-packages + rm -rf external/pypi_numpy/bin + rm -rf external/pypi_numpy/site-packages + rm -f external/pypi_numpy/rules_python_wheel_entry_point_f2py3.py + rm -f external/pypi_numpy/rules_python_wheel_entry_point_f2py${python_dotted}.py + rm -rf external/pypi_grpcio/site-packages + rm -rf external/pypi_h5py/site-packages + rm -rf external/pypi_wrapt/site-packages + rm -rf external/pypi_markupsafe/site-packages + rm -rf external/pypi_dm_tree/site-packages + '' + + lib.optionalString (stdenv.isAarch64) '' + rm -rf external/pypi_psutil/site-packages + '' + + '' + substituteInPlace external/pypi/requirements.bzl \ + --replace-fail ${python_underscored} @version_underscored@ + substituteInPlace external/python_version_repo/py_version.bzl \ + --replace-fail ${python_dotted} @version_long@ + sed -i 's/(\("[^(),]*"\), "\([^()]*\)==[^()]*")/(\1, "\2")/g' \ + external/pypi/requirements.bzl + sed -i 's/, "pypi_version=[0-9.]*"//' external/pypi_numpy/BUILD.bazel + rm -rf external/python_*-unknown-linux-gnu/bin + rm -rf external/python_*-unknown-linux-gnu/include + rm -rf external/python_*-unknown-linux-gnu/lib + rm -rf external/python_*-unknown-linux-gnu/nix-support + rm -rf external/python_*-unknown-linux-gnu/python + rm -rf external/python_*-unknown-linux-gnu/share + substituteInPlace external/python_*-unknown-linux-gnu/BUILD.bazel \ + --replace-fail python${python_dotted} python@version_long@ \ + --replace-fail python${python_undotted} python@version_short@ \ + --replace-fail \ + 'python_version = "${python_dotted}"' \ + 'python_version = "@version_long@"' + substituteInPlace external/rules_python/python/repositories.bzl \ + --replace-fail ${pythonEnv} @python@ \ + --replace-fail ${python.version} @python_version@ + popd + '' + ); + sha256 = { x86_64-linux = if cudaSupport then - "sha256-5VFMNHeLrUxW5RTr6EhT3pay9nWJ5JkZTGirDds5QkU=" + "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" else - "sha256-KzgWV69Btr84FdwQ5JI2nQEsqiPg1/+TWdbw5bmxXOE="; + "sha256-SlIIHoD4mLcUbcjkXch/NFA4khbHsj19v05Z6xMX0/Q="; aarch64-linux = if cudaSupport then - "sha256-ty5+51BwHWE1xR4/0WcWTp608NzSAS/iiyN+9zx7/wI=" + "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" else - "sha256-9btXrNHqd720oXTPDhSmFidv5iaZRLjCVX8opmrMjXk="; - x86_64-darwin = "sha256-gqb03kB0z2pZQ6m1fyRp1/Nbt8AVVHWpOJSeZNCLc4w="; - aarch64-darwin = "sha256-WdgAaFZU+ePwWkVBhLzjlNT7ELfGHOTaMdafcAMD5yo="; + "sha256-bGa4OVl0UtD1qZS/zlcnLnm72UbZEr2n2HtJOUprRTs="; + # deps hashes can't be computed for the Darwin platforms as of + # 2024-10-21 as the expressions don't evaluate due to + # python3Packages.jaraco-path being broken on these platforms + x86_64-darwin = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + aarch64-darwin = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; } .${stdenv.hostPlatform.system} or (throw "unsupported system ${stdenv.hostPlatform.system}"); }; @@ -592,11 +1090,13 @@ let description = "Computation using data flow graphs for scalable machine learning"; homepage = "http://tensorflow.org"; license = lib.licenses.asl20; - maintainers = with lib.maintainers; [ abbradar ]; + maintainers = with lib.maintainers; [ + abbradar + gm6k + ]; platforms = with lib.platforms; linux ++ darwin; broken = - stdenv.hostPlatform.isDarwin - || !(xlaSupport -> cudaSupport) + !(xlaSupport -> cudaSupport) || !(cudaSupport -> builtins.hasAttr cudnnAttribute cudaPackages) || !(cudaSupport -> cudaPackages ? cudatoolkit); } @@ -608,7 +1108,7 @@ let in buildPythonPackage { inherit version pname; - disabled = pythonAtLeast "3.12"; + disabled = pythonOlder "3.9"; src = bazel-build.python; @@ -617,16 +1117,17 @@ buildPythonPackage { # - Relax flatbuffers and gast version requirements # - The purpose of python3Packages.libclang is not clear at the moment and we don't have it packaged yet # - keras and tensorlow-io-gcs-filesystem will be considered as optional for now. - postPatch = '' - sed -i setup.py \ - -e '/tensorflow-io-gcs-filesystem/,+1d' \ - -e "s/'flatbuffers[^']*',/'flatbuffers',/" \ - -e "s/'gast[^']*',/'gast',/" \ - -e "/'libclang[^']*',/d" \ - -e "/'keras[^']*')\?,/d" \ - -e "/'tensorflow-io-gcs-filesystem[^']*',/d" \ - -e "s/'protobuf[^']*',/'protobuf',/" \ - ''; + pythonRelaxDeps = [ + "flatbuffers" + "gast" + "protobuf" + ]; + + pythonRemoveDeps = [ + "tensorflow-io-gcs-filesystem" + "libclang" + "keras" + ]; # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow # and the propagated input tensorboard, which causes environment collisions. @@ -643,12 +1144,14 @@ buildPythonPackage { absl-py abseil-cpp astunparse + distutils flatbuffers-python gast google-pasta grpcio h5py keras-preprocessing + ml-dtypes numpy opt-einsum packaging @@ -660,15 +1163,28 @@ buildPythonPackage { wrapt ] ++ lib.optionals withTensorboard [ tensorboard ]; - nativeBuildInputs = lib.optionals cudaSupport [ addDriverRunpath ]; - - postFixup = lib.optionalString cudaSupport '' - find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do - addDriverRunpath "$lib" + nativeBuildInputs = + [ + pythonRelaxDepsHook + ] + ++ lib.optionals cudaSupport [ + addDriverRunpath + ]; - patchelf --set-rpath "${cudatoolkit}/lib:${cudatoolkit.lib}/lib:${cudnnMerged}/lib:${lib.getLib nccl}/lib:$(patchelf --print-rpath "$lib")" "$lib" - done - ''; + postFixup = lib.optionalString cudaSupport ( + '' + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + addDriverRunpath "$lib" + + patchelf --set-rpath "${cudatoolkit}/lib:${cudatoolkit.lib}/lib:${cudnnMerged}/lib:${lib.getLib nccl}/lib:$(patchelf --print-rpath "$lib")" "$lib" + '' + + lib.optionalString stdenv.isLinux '' + patchelf --set-rpath "${cudatoolkit}/lib:${cudatoolkit.lib}/lib:${cudnnMerged}/lib:${nccl}/lib:$(patchelf --print-rpath "$lib")" "$lib" + '' + + '' + done + '' + ); # Actual tests are slow and impure. # TODO try to run them anyway diff --git a/pkgs/development/python-modules/tensorflow/fix-string-syntax.patch b/pkgs/development/python-modules/tensorflow/fix-string-syntax.patch new file mode 100644 index 0000000000000..72124e7a3ba23 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/fix-string-syntax.patch @@ -0,0 +1,30 @@ +diff -Naurd x/third_party/gpus/find_cuda_config.py y/third_party/gpus/find_cuda_config.py +--- x/third_party/gpus/find_cuda_config.py 2024-11-03 00:29:24.245892045 +0000 ++++ y/third_party/gpus/find_cuda_config.py 2024-11-03 00:32:30.813159341 +0000 +@@ -121,7 +121,7 @@ + def _get_header_version(path, name): + """Returns preprocessor defines in C header file.""" + for line in io.open(path, "r", encoding="utf-8").readlines(): +- match = re.match("\s*#\s*define %s\s+(\d+)" % name, line) ++ match = re.match("\\s*#\\s*define %s\\s+(\\d+)" % name, line) + if match: + return match.group(1) + return "" +@@ -260,7 +260,7 @@ + cuda_library_path = _find_library(base_paths, "cudart", cuda_version) + + def get_nvcc_version(path): +- pattern = "Cuda compilation tools, release \d+\.\d+, V(\d+\.\d+\.\d+)" ++ pattern = "Cuda compilation tools, release \\d+\\.\\d+, V(\\d+\\.\\d+\\.\\d+)" + for line in subprocess.check_output([path, "--version"]).splitlines(): + match = re.match(pattern, line.decode("ascii")) + if match: +@@ -549,7 +549,7 @@ + paths. Detect those and return '/usr', otherwise forward to _list_from_env(). + """ + if env_name in os.environ: +- match = re.match("^(/[^/ ]*)+/lib/\w+-linux-gnu/?$", os.environ[env_name]) ++ match = re.match("^(/[^/ ]*)+/lib/\\w+-linux-gnu/?$", os.environ[env_name]) + if match: + return [match.group(1)] + return _list_from_env(env_name, default) diff --git a/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch b/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch new file mode 100644 index 0000000000000..95e3b85b0a08b --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch @@ -0,0 +1,12 @@ +diff -Naurd x/tensorflow/tools/pip_package/BUILD y/tensorflow/tools/pip_package/BUILD +--- x/tensorflow/tools/pip_package/BUILD 2023-10-14 20:09:43.001857258 +0000 ++++ y/tensorflow/tools/pip_package/BUILD 2023-10-14 20:15:17.894102008 +0000 +@@ -209,7 +209,7 @@ + "@com_google_protobuf//:LICENSE", + "@curl//:COPYING", + "@ducc//:LICENSE", +- "@flatbuffers//:LICENSE", ++ "@flatbuffers//:LICENSE.txt", + "@gemmlowp//:LICENSE", + "@libjpeg_turbo//:LICENSE.md", + "@llvm-project//llvm:LICENSE.TXT", diff --git a/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch b/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch new file mode 100644 index 0000000000000..1b63851f095a1 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch @@ -0,0 +1,27 @@ +diff -Naurd x/third_party/systemlibs/protobuf.BUILD y/third_party/systemlibs/protobuf.BUILD +--- x/third_party/systemlibs/protobuf.BUILD 2023-12-03 15:39:27.742087246 +0000 ++++ y/third_party/systemlibs/protobuf.BUILD 2023-12-04 13:25:41.777638732 +0000 +@@ -1,4 +1,8 @@ +-load("@rules_proto//proto:defs.bzl", "proto_library") ++load( ++ "@rules_proto//proto:defs.bzl", ++ "proto_lang_toolchain", ++ "proto_library", ++) + load( + "@com_google_protobuf//:protobuf.bzl", + "cc_proto_library", +@@ -118,3 +122,13 @@ + srcs = [proto[1][0] for proto in WELL_KNOWN_PROTO_MAP.items()], + visibility = ["//visibility:public"], + ) ++ ++proto_lang_toolchain( ++ name = "cc_toolchain", ++ blacklisted_protos = [ ++ "//:compiler_plugin_proto", ++ "//:descriptor_proto", ++ ], ++ command_line = "--cpp_out=$(OUT)", ++ visibility = ["//visibility:public"], ++) diff --git a/pkgs/development/python-modules/tensorflow/protobuf_lite.patch b/pkgs/development/python-modules/tensorflow/protobuf_lite.patch new file mode 100644 index 0000000000000..26d1d47ab3006 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/protobuf_lite.patch @@ -0,0 +1,16 @@ +diff -Naurd x/third_party/systemlibs/protobuf.BUILD y/third_party/systemlibs/protobuf.BUILD +--- x/third_party/systemlibs/protobuf.BUILD 2023-12-03 15:39:27.742087246 +0000 ++++ y/third_party/systemlibs/protobuf.BUILD 2023-12-03 20:23:49.345014695 +0000 +@@ -66,6 +66,12 @@ + ) + + cc_library( ++ name = "protobuf_lite", ++ linkopts = ["-lprotobuf-lite"], ++ visibility = ["//visibility:public"], ++) ++ ++cc_library( + name = "protobuf_headers", + linkopts = ["-lprotobuf"], + visibility = ["//visibility:public"], diff --git a/pkgs/development/python-modules/tensorflow/pybind11-osx.patch b/pkgs/development/python-modules/tensorflow/pybind11-osx.patch new file mode 100644 index 0000000000000..bcff5972273f2 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/pybind11-osx.patch @@ -0,0 +1,12 @@ +diff -Naurd x/third_party/systemlibs/pybind11.BUILD y/third_party/systemlibs/pybind11.BUILD +--- x/third_party/systemlibs/pybind11.BUILD 2023-10-17 19:01:11.988196127 +0000 ++++ y/third_party/systemlibs/pybind11.BUILD 2023-10-17 19:04:09.916356016 +0000 +@@ -6,3 +6,8 @@ + "@org_tensorflow//third_party/python_runtime:headers", + ], + ) ++ ++config_setting( ++ name = "osx", ++ constraint_values = ["@platforms//os:osx"], ++) diff --git a/pkgs/development/python-modules/tensorflow/remove-keras-dependency.patch b/pkgs/development/python-modules/tensorflow/remove-keras-dependency.patch new file mode 100644 index 0000000000000..2865da548aa18 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/remove-keras-dependency.patch @@ -0,0 +1,13 @@ +diff -Naurd x/setup.py y/setup.py +--- x/setup.py 2023-12-01 20:07:55.694564265 +0000 ++++ y/setup.py 2023-12-01 20:09:08.526682512 +0000 +@@ -121,9 +121,6 @@ + 'tensorflow_estimator >= 2.14.0, < 2.15', + 'tf-estimator-nightly ~= 2.15.0.dev', + ), +- standard_or_nightly( +- 'keras >= 2.14.0, < 2.15', 'keras-nightly ~= 2.15.0.dev' +- ), + ] + REQUIRED_PACKAGES = [p for p in REQUIRED_PACKAGES if p is not None] + diff --git a/pkgs/development/python-modules/tensorflow/riegeli-proto.patch b/pkgs/development/python-modules/tensorflow/riegeli-proto.patch new file mode 100644 index 0000000000000..ef39e48ce2225 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/riegeli-proto.patch @@ -0,0 +1,36 @@ +diff -Naurd x/riegeli/records/BUILD y/riegeli/records/BUILD +--- x/riegeli/records/BUILD 2023-12-06 16:24:03.939575600 +0000 ++++ y/riegeli/records/BUILD 2023-12-06 16:23:00.079531687 +0000 +@@ -1,4 +1,4 @@ +-load("@rules_proto//proto:defs.bzl", "proto_library") ++load("@local_tsl//tsl/platform:build_config.bzl", "tf_proto_library") + + package( + default_visibility = ["//visibility:public"], +@@ -14,7 +14,7 @@ + deps = [ + ":chunk_reader", + ":record_position", +- ":records_metadata_cc_proto", ++ ":records_metadata_proto_cc_impl", + ":skipped_region", + "//riegeli/base:arithmetic", + "//riegeli/base:assert", +@@ -184,13 +184,10 @@ + ], + ) + +-proto_library( ++tf_proto_library( + name = "records_metadata_proto", + srcs = ["records_metadata.proto"], +- deps = ["@com_google_protobuf//:descriptor_proto"], +-) +- +-cc_proto_library( +- name = "records_metadata_cc_proto", +- deps = [":records_metadata_proto"], ++ cc_api_version = 2, ++ make_default_target_header_only = True, ++ visibility = ["//visibility:public"], + ) diff --git a/pkgs/development/python-modules/tensorflow/rules-python-use-nix.patch b/pkgs/development/python-modules/tensorflow/rules-python-use-nix.patch new file mode 100644 index 0000000000000..c8ae7b3aba4b7 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/rules-python-use-nix.patch @@ -0,0 +1,115 @@ +diff -Naurd python/private/toolchains_repo.bzl python-use-nix/private/toolchains_repo.bzl +--- python/private/toolchains_repo.bzl 2024-10-07 06:42:33.209319301 +0000 ++++ python-use-nix/private/toolchains_repo.bzl 2024-10-07 06:53:00.702389154 +0000 +@@ -199,7 +199,7 @@ + def py_binary(name, **kwargs): + return _py_binary( + name = name, +- python_version = "{python_version}", ++ python_version = "@python_version_post@", + **kwargs + ) + +@@ -213,7 +213,7 @@ + def py_test(name, **kwargs): + return _py_test( + name = name, +- python_version = "{python_version}", ++ python_version = "@python_version_post@", + **kwargs + ) + +diff -Naurd python/repositories.bzl python-use-nix/repositories.bzl +--- python/repositories.bzl 2023-11-29 23:07:55.169106050 +0000 ++++ python-use-nix/repositories.bzl 2023-12-01 12:00:06.746231531 +0000 +@@ -119,65 +119,23 @@ + fail("Exactly one of (url, urls) must be set.") + + platform = rctx.attr.platform +- python_version = rctx.attr.python_version ++ python_version = "@python_version@" + python_short_version = python_version.rpartition(".")[0] + release_filename = rctx.attr.release_filename + urls = rctx.attr.urls or [rctx.attr.url] + auth = _get_auth(rctx, urls) + +- if release_filename.endswith(".zst"): +- rctx.download( +- url = urls, +- sha256 = rctx.attr.sha256, +- output = release_filename, +- auth = auth, +- ) +- unzstd = rctx.which("unzstd") +- if not unzstd: +- url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version) +- rctx.download_and_extract( +- url = url, +- sha256 = rctx.attr.zstd_sha256, +- auth = auth, +- ) +- working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version) +- +- make_result = rctx.execute( +- [which_with_fail("make", rctx), "--jobs=4"], +- timeout = 600, +- quiet = True, +- working_directory = working_directory, +- ) +- if make_result.return_code: +- fail_msg = ( +- "Failed to compile 'zstd' from source for use in Python interpreter extraction. " + +- "'make' error message: {}".format(make_result.stderr) +- ) +- fail(fail_msg) +- zstd = "{working_directory}/zstd".format(working_directory = working_directory) +- unzstd = "./unzstd" +- rctx.symlink(zstd, unzstd) +- +- exec_result = rctx.execute([ +- which_with_fail("tar", rctx), +- "--extract", +- "--strip-components=2", +- "--use-compress-program={unzstd}".format(unzstd = unzstd), +- "--file={}".format(release_filename), +- ]) +- if exec_result.return_code: +- fail_msg = ( +- "Failed to extract Python interpreter from '{}'. ".format(release_filename) + +- "'tar' error message: {}".format(exec_result.stderr) +- ) +- fail(fail_msg) +- else: +- rctx.download_and_extract( +- url = urls, +- sha256 = rctx.attr.sha256, +- stripPrefix = rctx.attr.strip_prefix, +- auth = auth, ++ lndir_result = rctx.execute([ ++ "lndir", ++ "@python@", ++ ".", ++ ]) ++ if lndir_result.return_code: ++ fail_msg = ( ++ "Failed to link Python interpreter from '{}'. ".format("@python@") + ++ "'lndir' error message: {}".format(lndir_result.stderr) + ) ++ fail(fail_msg) + + patches = rctx.attr.patches + if patches: +diff -Naurd python/pip_install/pip_repository.bzl python-use-nix/pip_install/pip_repository.bzl +--- python/pip_install/pip_repository.bzl 2024-10-08 15:24:23.582898888 +0000 ++++ python-use-nix/pip_install/pip_repository.bzl 2024-10-08 16:33:38.733498958 +0000 +@@ -607,6 +607,10 @@ + # There is an extreme edge-case with entry_points that end with `.py` + # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 + entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name ++ ++ if entry_point_without_py.startswith("f2py3"): ++ continue ++ + entry_point_target_name = ( + _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py + ) diff --git a/pkgs/development/python-modules/tensorflow/system_includes_cuda.patch b/pkgs/development/python-modules/tensorflow/system_includes_cuda.patch new file mode 100644 index 0000000000000..50555ed9828a4 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/system_includes_cuda.patch @@ -0,0 +1,12 @@ +diff -Naurd x/third_party/gpus/cuda_configure.bzl y/third_party/gpus/cuda_configure.bzl +--- x/third_party/gpus/cuda_configure.bzl 2024-10-17 14:10:15.759870088 +0000 ++++ y/third_party/gpus/cuda_configure.bzl 2024-10-17 14:14:42.795467879 +0000 +@@ -1263,7 +1263,7 @@ + repository_ctx, + cc_fullpath, + tf_sysroot, +- ) ++ ) + ["@cuda_wrapper_includes@"] + cuda_defines = {} + cuda_defines["%{builtin_sysroot}"] = tf_sysroot + cuda_defines["%{cuda_toolkit_path}"] = "" diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 0aa9e2fd929e0..f73c0b8af28a3 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -15876,8 +15876,9 @@ self: super: with self; { tensorflow-build = let compat = rec { + abseil-cppTF = pkgs.abseil-cpp_202301; protobufTF = pkgs.protobuf_21.override { - abseil-cpp = pkgs.abseil-cpp_202301; + abseil-cpp = abseil-cppTF; }; # https://www.tensorflow.org/install/source#gpu cudaPackagesTF = pkgs.cudaPackages_11; @@ -15922,6 +15923,7 @@ self: super: with self; { callPackage ../development/python-modules/tensorflow { inherit (pkgs.config) cudaSupport; inherit (pkgs.darwin.apple_sdk.frameworks) Foundation Security; + llvmPackages = pkgs.llvmPackages_16; flatbuffers-core = pkgs.flatbuffers; flatbuffers-python = self.flatbuffers; cudaPackages = compat.cudaPackagesTF; @@ -15930,7 +15932,8 @@ self: super: with self; { grpc = compat.grpcTF; grpcio = compat.grpcioTF; tensorboard = compat.tensorboardTF; - abseil-cpp = pkgs.abseil-cpp_202301; + abseil-cpp = compat.abseil-cppTF; + # Tensorflow 2.13 doesn't support gcc13: # https://github.com/tensorflow/tensorflow/issues/61289