diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1-foss-2022b.eb b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1-foss-2022b.eb new file mode 100644 index 000000000000..896e238a874e --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1-foss-2022b.eb @@ -0,0 +1,154 @@ +name = 'PyTorch' +version = '1.13.1' + +homepage = 'https://pytorch.org/' +description = """Tensors and Dynamic neural networks in Python with strong GPU acceleration. +PyTorch is a deep learning framework that puts Python first.""" + +toolchain = {'name': 'foss', 'version': '2022b'} + +source_urls = [GITHUB_RELEASE] +sources = ['%(namelower)s-v%(version)s.tar.gz'] +patches = [ + 'PyTorch-1.7.0_disable-dev-shm-test.patch', + 'PyTorch-1.10.0_fix-kineto-crash.patch', + 'PyTorch-1.11.0_fix-fp16-quantization-without-fbgemm.patch', + 'PyTorch-1.11.1_skip-test_init_from_local_shards.patch', + 'PyTorch-1.12.0_fix-EmbeddingBag-without-fbgemm.patch', + 'PyTorch-1.12.1_add-hypothesis-suppression.patch', + 'PyTorch-1.12.1_fix-skip-decorators.patch', + 'PyTorch-1.12.1_fix-test_cpp_extensions_jit.patch', + 'PyTorch-1.12.1_fix-test_wishart_log_prob.patch', + 'PyTorch-1.12.1_fix-TestTorch.test_to.patch', + 'PyTorch-1.12.1_fix-use-after-free-in-tensorpipe-agent.patch', + 'PyTorch-1.12.1_fix-vsx-loadu.patch', + 'PyTorch-1.12.1_fix-vsx-vector-funcs.patch', + 'PyTorch-1.12.1_skip-test_round_robin.patch', + 'PyTorch-1.13.1_disable-test-sharding.patch', + 'PyTorch-1.13.1_fix-flaky-jit-test.patch', + 'PyTorch-1.13.1_fix-fsdp-fp16-test.patch', + 'PyTorch-1.13.1_fix-gcc-12-missing-includes.patch', + 'PyTorch-1.13.1_fix-gcc-12-warning-in-fbgemm.patch', + 'PyTorch-1.13.1_fix-numpy-deprecations.patch', + 'PyTorch-1.13.1_fix-protobuf-dependency.patch', + 'PyTorch-1.13.1_fix-pytest-args.patch', + 'PyTorch-1.13.1_fix-python-3.11-compat.patch', + 'PyTorch-1.13.1_fix-test-ops-conf.patch', + 'PyTorch-1.13.1_fix-warning-in-test-cpp-api.patch', + 'PyTorch-1.13.1_increase-tolerance-test_ops.patch', + 'PyTorch-1.13.1_install-vsx-vec-headers.patch', + 'PyTorch-1.13.1_no-cuda-stubs-rpath.patch', + 'PyTorch-1.13.1_remove-flaky-test-in-testnn.patch', + 'PyTorch-1.13.1_skip-failing-grad-test.patch', + 'PyTorch-1.13.1_skip-failing-singular-grad-test.patch', + 'PyTorch-1.13.1_skip-tests-without-fbgemm.patch', +] +checksums = [ + {'pytorch-v1.13.1.tar.gz': 'dbc229ee9750b02b514937d017744443a269ea0241ed3f32b9af0703589d25d4'}, + {'PyTorch-1.7.0_disable-dev-shm-test.patch': '622cb1eaeadc06e13128a862d9946bcc1f1edd3d02b259c56a9aecc4d5406b8a'}, + {'PyTorch-1.10.0_fix-kineto-crash.patch': 'dc467333b28162149af8f675929d8c6bf219f23230bfc0d39af02ba4f6f882eb'}, + {'PyTorch-1.11.0_fix-fp16-quantization-without-fbgemm.patch': + 'cc526130b6446bbbf5f0f7372d3aeee3e7d4c4d6e471524dff028b430b152934'}, + {'PyTorch-1.11.1_skip-test_init_from_local_shards.patch': + '4aeb1b0bc863d4801b0095cbce69f8794066748f0df27c6aaaf729c5ecba04b7'}, + {'PyTorch-1.12.0_fix-EmbeddingBag-without-fbgemm.patch': + '090598592283e3fc46ee08a68b6a6afe07be41b26514afba51834408bf1c98ed'}, + {'PyTorch-1.12.1_add-hypothesis-suppression.patch': + 'e71ffb94ebe69f580fa70e0de84017058325fdff944866d6bd03463626edc32c'}, + {'PyTorch-1.12.1_fix-skip-decorators.patch': 'e3ca6e42b2fa592ea095939fb59ab875668a058479407db3f3684cc5c6f4146c'}, + {'PyTorch-1.12.1_fix-test_cpp_extensions_jit.patch': + '1efc9850c431d702e9117d4766277d3f88c5c8b3870997c9974971bce7f2ab83'}, + {'PyTorch-1.12.1_fix-test_wishart_log_prob.patch': + 'cf475ae6e6234b96c8d1bf917597c5176c94b3ccd940b72f2e1cd0c979580f45'}, + {'PyTorch-1.12.1_fix-TestTorch.test_to.patch': '75f27987c3f25c501e719bd2b1c70a029ae0ee28514a97fe447516aee02b1535'}, + {'PyTorch-1.12.1_fix-use-after-free-in-tensorpipe-agent.patch': + '0bd7e88b92c4c6f0fecf01746009858ba19f2df68b10b88c41485328a531875d'}, + {'PyTorch-1.12.1_fix-vsx-loadu.patch': '8bfe3c94ada1dd1f7974a1261a8b576fb7ae944050fa1c7830fca033831123b2'}, + {'PyTorch-1.12.1_fix-vsx-vector-funcs.patch': 'caccbf60f62eac313896c1eaec78b08f5d0fdfcb907079087490bb13d1561aa2'}, + {'PyTorch-1.12.1_skip-test_round_robin.patch': '63d4849b78605aa088fdff695637d9473ea60dee603a3ff7f788690d70c55349'}, + {'PyTorch-1.13.1_disable-test-sharding.patch': 'df2074adeba47998ce2993d99ca64eb6f1c79ab7057f553b436efdec264d3572'}, + {'PyTorch-1.13.1_fix-flaky-jit-test.patch': '71efdeb29b5e5b4982c9f5cb2182733654a34d52f85bb5487bc4d7d99b86101b'}, + {'PyTorch-1.13.1_fix-fsdp-fp16-test.patch': '8ae68e60d6e1f92f50322b7f0381c7e65251fba32d7606e3a238a36a2f55b5cf'}, + {'PyTorch-1.13.1_fix-gcc-12-missing-includes.patch': + '18df8c61ecaa9fb659346c1e172828bca6b069f0145bb8f6a36b0a23b7bef0a6'}, + {'PyTorch-1.13.1_fix-gcc-12-warning-in-fbgemm.patch': + '5c7be91a6096083a0b1315efe0001537499c600f1f569953c6a2c7f4cc1d0910'}, + {'PyTorch-1.13.1_fix-numpy-deprecations.patch': + 'f461b570efe0434ddd806bf2fa7020eb213e3ed89d0eb4403e076f4276ba2a46'}, + {'PyTorch-1.13.1_fix-protobuf-dependency.patch': + '8bd755a0cab7233a243bc65ca57c9630dfccdc9bf8c9792f0de4e07a644fcb00'}, + {'PyTorch-1.13.1_fix-pytest-args.patch': 'd3e3c841cf8d73683750f29326f2be56ee0bb5df7ff522baf7d7c3f301a91ec2'}, + {'PyTorch-1.13.1_fix-python-3.11-compat.patch': + 'fa4eb0e27e00a90bb217b77c0023089c4659c03f37d781ab4a681bdcb4f0432f'}, + {'PyTorch-1.13.1_fix-test-ops-conf.patch': 'df652eec7753864ebebbfeca546929a53e3fb8f24259d5c9b964266a8551198c'}, + {'PyTorch-1.13.1_fix-warning-in-test-cpp-api.patch': + 'bdde0f2105215c95a54de64ec4b1a4520528510663174fef6d5b900eb1db3937'}, + {'PyTorch-1.13.1_increase-tolerance-test_ops.patch': + 'c909fdfc2b12df457e1eb5514265ffec3eab653994949416f3f048668421e223'}, + {'PyTorch-1.13.1_install-vsx-vec-headers.patch': + '7b678f54bb947afd4767f5877ac424b4b94ce5db609ea20f5a869ccf4027035f'}, + {'PyTorch-1.13.1_no-cuda-stubs-rpath.patch': '4c636059850fc9d1ecb27ce275f8aad5d5b6fdc19e35aff0c25b86cb3201352a'}, + {'PyTorch-1.13.1_remove-flaky-test-in-testnn.patch': + 'be83ff61fe2dedab6d49c232936d5622df81ab49154264490021c6c828e53315'}, + {'PyTorch-1.13.1_skip-failing-grad-test.patch': '6681200f9509893cb9231b5c93ac9bc5e6d9d9ae4febefca52e7cbc843ba8f51'}, + {'PyTorch-1.13.1_skip-failing-singular-grad-test.patch': + '72688a57b2bb617665ad1a1d5e362c5111ae912c10936bb38a089c0204729f48'}, + {'PyTorch-1.13.1_skip-tests-without-fbgemm.patch': + '481e595f673baf8ae58b41697a6792b83048b0264aa79b422f48cd8c22948bb7'}, +] + +osdependencies = [OS_PKG_IBVERBS_DEV] + +builddependencies = [ + ('CMake', '3.24.3'), + ('hypothesis', '6.68.2'), + # For tests + ('pytest-rerunfailures', '12.0'), + ('pytest-shard', '0.1.2'), +] + +dependencies = [ + ('Ninja', '1.11.1'), # Required for JIT compilation of C++ extensions + ('Python', '3.10.8'), + ('protobuf', '23.0'), + ('protobuf-python', '4.23.0'), + ('pybind11', '2.10.3'), + ('SciPy-bundle', '2023.02'), + ('PyYAML', '6.0'), + ('MPFR', '4.2.0'), + ('GMP', '6.2.1'), + ('numactl', '2.0.16'), + ('FFmpeg', '5.1.2'), + ('Pillow', '9.4.0'), + ('expecttest', '0.1.3'), +] + +custom_opts = ['CMAKE_CXX_STANDARD=17'] + +excluded_tests = { + '': [ + # This test seems to take too long on NVIDIA Ampere at least. + 'distributed/test_distributed_spawn', + # Broken on CUDA 11.6/11.7: https://github.com/pytorch/pytorch/issues/75375 + 'distributions/test_constraints', + # no xdoctest + 'doctests', + # failing on broadwell + # See https://github.com/easybuilders/easybuild-easyconfigs/issues/17712 + 'test_native_mha', + # intermittent failures on various systems + # See https://github.com/easybuilders/easybuild-easyconfigs/issues/17712 + 'distributed/rpc/test_tensorpipe_agent', + ] +} + +runtest = 'cd test && PYTHONUNBUFFERED=1 %(python)s run_test.py --continue-through-error --verbose %(excluded_tests)s' + +# Especially test_quantization has a few corner cases that are triggered by the random input values, +# those cannot be easily avoided, see https://github.com/pytorch/pytorch/issues/107030 +# So allow a low number of tests to fail as the tests "usually" succeed +max_failed_tests = 2 + +tests = ['PyTorch-check-cpp-extension.py'] + +moduleclass = 'ai' diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_disable-test-sharding.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_disable-test-sharding.patch new file mode 100644 index 000000000000..551d7a4830af --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_disable-test-sharding.patch @@ -0,0 +1,41 @@ +Our error checking doesn't work well with the parallel/sharded pytorch test. +As the overall gain is low, disable it and always run the full test suite in a single process. +Additionally remove the fail-on-first-error flag (-x) to collect all errors. + +Author: Alexander Grund (TU Dresden) + +diff --git a/test/run_test.py b/test/run_test.py +index f7c80f3f0a6..9e930d774ed 100755 +--- a/test/run_test.py ++++ b/test/run_test.py +@@ -718,18 +718,18 @@ def print_log_file(test: str, file_path: str, failed: bool) -> None: + + + def run_test_ops(test_module, test_directory, options): +- if 'slow-gradcheck' in os.getenv("BUILD_ENVIRONMENT", ""): ++ if True: + # there are a lot of tests that take up a lot of space in slowgrad check, so don't bother parallelizing + # it's also on periodic so we don't care about TTS as much + return run_test(test_module, test_directory, copy.deepcopy(options), +- extra_unittest_args=["--use-pytest", '-vv', '-x', '--reruns=2', '-rfEX'], ++ extra_unittest_args=["--use-pytest", '-vv', '--reruns=2', '-rfEX'], + ) + return_codes = [] + os.environ["NUM_PARALLEL_PROCS"] = str(NUM_PROCS) + pool = get_context("spawn").Pool(NUM_PROCS) + for i in range(NUM_PROCS): + return_code = pool.apply_async(run_test, args=(test_module, test_directory, copy.deepcopy(options)), +- kwds={"extra_unittest_args": ["--use-pytest", '-vv', '-x', '--reruns=2', '-rfEX', ++ kwds={"extra_unittest_args": ["--use-pytest", '-vv', '--reruns=2', '-rfEX', + f'--shard-id={i}', f'--num-shards={NUM_PROCS}', + "-k=not _linalg_cholesky_"], + }) +@@ -742,7 +742,7 @@ def run_test_ops(test_module, test_directory, options): + if return_code.get() != 0: + return return_code.get() + return_code = run_test(test_module, test_directory, copy.deepcopy(options), +- extra_unittest_args=["--use-pytest", '-vv', '-x', '--reruns=2', '-rfEX', ++ extra_unittest_args=["--use-pytest", '-vv', '--reruns=2', '-rfEX', + "-k=_linalg_cholesky_"], + ) + return return_code diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-gcc-12-missing-includes.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-gcc-12-missing-includes.patch new file mode 100644 index 000000000000..a74b69d95f47 --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-gcc-12-missing-includes.patch @@ -0,0 +1,29 @@ +Add missing headers that are no longer transitively included in GCC 12 + +Author: Alexander Grund (TU Dresden) + +Submodule third_party/gloo contains modified content +diff --git a/third_party/gloo/gloo/transport/ibverbs/pair.h b/third_party/gloo/gloo/transport/ibverbs/pair.h +index 1ccc050..54dbc7e 100644 +--- a/third_party/gloo/gloo/transport/ibverbs/pair.h ++++ b/third_party/gloo/gloo/transport/ibverbs/pair.h +@@ -8,6 +8,7 @@ + + #pragma once + ++#include + #include + #include + #include +diff --git a/third_party/gloo/gloo/transport/tcp/device.cc b/third_party/gloo/gloo/transport/tcp/device.cc +index 05cf0a4..4408d60 100644 +--- a/third_party/gloo/gloo/transport/tcp/device.cc ++++ b/third_party/gloo/gloo/transport/tcp/device.cc +@@ -8,6 +8,7 @@ + + #include "gloo/transport/tcp/device.h" + ++#include + #include + #include + #include diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-gcc-12-warning-in-fbgemm.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-gcc-12-warning-in-fbgemm.patch new file mode 100644 index 000000000000..ec1168d39177 --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-gcc-12-warning-in-fbgemm.patch @@ -0,0 +1,24 @@ +GCC 12 has a regression (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593) +leading to warnings/errors during build: https://github.com/pytorch/FBGEMM/issues/1666 +Suppress the affected warning in FBGEMM. + +Author: Alexander Grund (TU Dresden) + +Submodule third_party/fbgemm contains modified content +diff --git a/third_party/fbgemm/CMakeLists.txt b/third_party/fbgemm/CMakeLists.txt +index 58dcb9ae..c888f0f8 100644 +--- a/third_party/fbgemm/CMakeLists.txt ++++ b/third_party/fbgemm/CMakeLists.txt +@@ -147,10 +147,10 @@ else(MSVC) + string(APPEND CMAKE_CXX_FLAGS " -Werror") + string(APPEND CMAKE_CXX_FLAGS " -Wno-deprecated-declarations") + target_compile_options(fbgemm_avx2 PRIVATE +- "-m64" "-mavx2" "-mf16c" "-mfma") ++ "-m64" "-mavx2" "-mf16c" "-mfma" "-Wno-uninitialized") + target_compile_options(fbgemm_avx512 PRIVATE + "-m64" "-mavx2" "-mfma" "-mavx512f" "-mavx512bw" "-mavx512dq" +- "-mavx512vl") ++ "-mavx512vl" "-Wno-uninitialized") + set_source_files_properties( + src/FbgemmFP16UKernelsAvx2.cc + src/FbgemmFP16UKernelsAvx512.cc diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-numpy-deprecations.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-numpy-deprecations.patch new file mode 100644 index 000000000000..e95867963108 --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-numpy-deprecations.patch @@ -0,0 +1,102 @@ +Numpy 1.24 removed some types deprecated earlier leading to failures in e.g. test_torch: +> ERROR: test_parsing_intlist (__main__.TestTorch) +> ... +> AttributeError: module 'numpy' has no attribute 'float'. + +Backported from https://github.com/pytorch/pytorch/pull/93997 +Author: Alexander Grund (TU Dresden) + +diff --git a/test/quantization/core/test_quantized_op.py b/test/quantization/core/test_quantized_op.py +index 79297e073f0..827a781df10 100644 +--- a/test/quantization/core/test_quantized_op.py ++++ b/test/quantization/core/test_quantized_op.py +@@ -3010,7 +3010,7 @@ class TestDynamicQuantizedOps(TestCase): + # W_scale = 1.0 + # W_zp = 0 + W_scales = np.ones(output_channels) +- W_zps = np.zeros(output_channels).astype(np.int) ++ W_zps = np.zeros(output_channels).astype(int) + W_value_min = -128 + W_value_max = 127 + W_q0 = np.round( +@@ -3581,9 +3581,9 @@ class TestQuantizedLinear(TestCase): + # xnnpack forces W_zp to 0 when using symmetric quantization + # ONEDNN only supports symmetric quantization of weight + if dtype == torch.qint8 or qengine_is_onednn(): +- W_zps = np.zeros(output_channels).astype(np.int) ++ W_zps = np.zeros(output_channels).astype(int) + else: +- W_zps = np.round(np.random.rand(output_channels) * 100 - 50).astype(np.int) ++ W_zps = np.round(np.random.rand(output_channels) * 100 - 50).astype(int) + # when using symmetric quantization + # special restriction for xnnpack fully connected op weight + # [-127, 127] instead of [-128, 127] +diff --git a/test/test_reductions.py b/test/test_reductions.py +index 0e36906f25f..20fcb6ef9ae 100644 +--- a/test/test_reductions.py ++++ b/test/test_reductions.py +@@ -1323,7 +1323,7 @@ class TestReductions(TestCase): + vals = [[True, True], [True, False], [False, False], []] + for val in vals: + result = torch.prod(torch.tensor(val, device=device), dtype=torch.bool).item() +- expect = np.prod(np.array(val), dtype=np.bool) ++ expect = np.prod(np.array(val), dtype=bool) + self.assertEqual(result, expect) + + result = torch.prod(torch.tensor(val, device=device)).item() +diff --git a/test/test_tensor_creation_ops.py b/test/test_tensor_creation_ops.py +index aab26452496..a71761ce670 100644 +--- a/test/test_tensor_creation_ops.py ++++ b/test/test_tensor_creation_ops.py +@@ -1440,14 +1440,14 @@ class TestTensorCreation(TestCase): + def test_ctor_with_numpy_array(self, device): + correct_dtypes = [ + np.double, +- np.float, ++ float, + np.float16, + np.int64, + np.int32, + np.int16, + np.int8, + np.uint8, +- np.bool, ++ bool, + ] + + incorrect_byteorder = '>' if sys.byteorder == 'little' else '<' +diff --git a/test/test_tensorboard.py b/test/test_tensorboard.py +index e836b0f1ba8..0857873a5fa 100644 +--- a/test/test_tensorboard.py ++++ b/test/test_tensorboard.py +@@ -796,7 +796,7 @@ class TestTensorBoardNumpy(BaseTestCase): + model = ModelHelper(name="mnist") + # how come those inputs don't break the forward pass =.=a + workspace.FeedBlob("data", np.random.randn(1, 3, 64, 64).astype(np.float32)) +- workspace.FeedBlob("label", np.random.randn(1, 1000).astype(np.int)) ++ workspace.FeedBlob("label", np.random.randn(1, 1000).astype(int)) + + with core.NameScope("conv1"): + conv1 = brew.conv(model, "data", 'conv1', dim_in=1, dim_out=20, kernel=5) +@@ -831,7 +831,7 @@ class TestTensorBoardNumpy(BaseTestCase): + def test_caffe2_simple_cnnmodel(self): + model = cnn.CNNModelHelper("NCHW", name="overfeat") + workspace.FeedBlob("data", np.random.randn(1, 3, 64, 64).astype(np.float32)) +- workspace.FeedBlob("label", np.random.randn(1, 1000).astype(np.int)) ++ workspace.FeedBlob("label", np.random.randn(1, 1000).astype(int)) + with core.NameScope("conv1"): + conv1 = model.Conv("data", "conv1", 3, 96, 11, stride=4) + relu1 = model.Relu(conv1, conv1) +diff --git a/test/test_torch.py b/test/test_torch.py +index 8de5b822d00..3121e256b21 100644 +--- a/test/test_torch.py ++++ b/test/test_torch.py +@@ -6182,7 +6182,7 @@ class TestTorch(TestCase): + # fail parse with float variables + self.assertRaises(TypeError, lambda: torch.ones((torch.tensor(3.), torch.tensor(4)))) + # fail parse with numpy floats +- self.assertRaises(TypeError, lambda: torch.ones((np.float(3.), torch.tensor(4)))) ++ self.assertRaises(TypeError, lambda: torch.ones((3., torch.tensor(4)))) + self.assertRaises(TypeError, lambda: torch.ones((np.array(3.), torch.tensor(4)))) + + # fail parse with > 1 element variables diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-protobuf-dependency.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-protobuf-dependency.patch new file mode 100644 index 000000000000..c92a9d49d02d --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-protobuf-dependency.patch @@ -0,0 +1,35 @@ +Add proper dependency in libprotobuf target to get the correct set of flags. +Otherwise the build will fail with e.g.: + undefined reference to `google::protobuf::internal::ThreadSafeArena::thread_cache_' +This is caused by missing the `PROTOBUF_USE_DLLS` define required for libprotobuf as a shared library. +See https://github.com/pytorch/pytorch/issues/106297 + +Author: Alexander Grund (TU Dresden) + +diff --git a/caffe2/proto/CMakeLists.txt b/caffe2/proto/CMakeLists.txt +index ba6b696dde4..5033e228119 100644 +--- a/caffe2/proto/CMakeLists.txt ++++ b/caffe2/proto/CMakeLists.txt +@@ -7,6 +7,7 @@ endif() + caffe2_protobuf_generate_cpp_py(Caffe2_PROTO_SRCS Caffe2_PROTO_HEADERS Caffe2_PROTO_PY ${Caffe2_PROTOBUF_FILES}) + + add_library(Caffe2_PROTO OBJECT ${Caffe2_PROTO_HEADERS} ${Caffe2_PROTO_SRCS}) ++target_link_libraries(Caffe2_PROTO PRIVATE protobuf::libprotobuf) + + if(MSVC) + if(BUILD_SHARED_LIBS) +diff --git a/cmake/ProtoBuf.cmake b/cmake/ProtoBuf.cmake +index 8d7633c4ab0..fb0e9a0c5f7 100644 +--- a/cmake/ProtoBuf.cmake ++++ b/cmake/ProtoBuf.cmake +@@ -122,10 +122,6 @@ if((NOT TARGET protobuf::libprotobuf) AND (NOT TARGET protobuf::libprotobuf-lite + # "Please set the proper paths so that I can find protobuf correctly.") + endif() + +-get_target_property(__tmp protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES) +-message(STATUS "Caffe2 protobuf include directory: " ${__tmp}) +-include_directories(BEFORE SYSTEM ${__tmp}) +- + # If Protobuf_VERSION is known (true in most cases, false if we are building + # local protobuf), then we will add a protobuf version check in + # Caffe2Config.cmake.in. diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-python-3.11-compat.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-python-3.11-compat.patch new file mode 100644 index 000000000000..327ff5fba84b --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-python-3.11-compat.patch @@ -0,0 +1,160 @@ +Some code isn't compatible with Python 3.11+ +Backport https://github.com/pytorch/pytorch/pull/92895 to fix this. + +Author: Alexander Grund (TU Dresden) + +diff --git a/test/test_fx.py b/test/test_fx.py +index c8da9d3d2cf..6aefb4b9b2c 100644 +--- a/test/test_fx.py ++++ b/test/test_fx.py +@@ -3295,6 +3295,7 @@ class TestFX(JitTestCase): + finally: + del sys.modules["__future__"] + ++ @unittest.skipIf(sys.version_info > (3, 11), "Does not work in 3.11") + def test_annotations_empty_tuple(self): + class Foo(torch.nn.Module): + def forward(self, x: Tuple[()], y: Tuple[str, Tuple[()]]): +@@ -4030,7 +4031,7 @@ class TestFunctionalTracing(JitTestCase): + "max_pool2d": PROXY_ITERATED, + "max_pool3d": PROXY_ITERATED, + +- "group_norm": LEN_ERROR ++ "group_norm": PROXY_ITERATED + } + + @classmethod +@@ -4069,7 +4070,7 @@ class TestFunctionalTracing(JitTestCase): + + def functional_test(self): + if func_name in self.UNTRACEABLE_FUNCTIONALS_PY38 and \ +- sys.version_info >= (3, 8) and sys.version_info < (3, 11): ++ sys.version_info >= (3, 8) and sys.version_info < (3, 12): + exc, err = self.UNTRACEABLE_FUNCTIONALS_PY38[func_name] + with self.assertRaisesRegex(exc, err): + symbolic_trace(fn) +diff --git a/torch/fx/_symbolic_trace.py b/torch/fx/_symbolic_trace.py +index ff9df1161a7..521bb02c846 100644 +--- a/torch/fx/_symbolic_trace.py ++++ b/torch/fx/_symbolic_trace.py +@@ -119,7 +119,29 @@ def _patch_function(fn: FunctionType, nargs: int) -> FunctionType: + co = fn.__code__ + co_flags = co.co_flags & ~HAS_VARSTUFF + co_args: tuple +- if hasattr(co, "co_posonlyargcount"): ++ if hasattr(co, "co_qualname"): ++ # Python-3.11+ code signature ++ co_args = ( ++ nargs, ++ 0, ++ 0, ++ co.co_nlocals, ++ co.co_stacksize, ++ co_flags, ++ co.co_code, ++ co.co_consts, ++ co.co_names, ++ co.co_varnames, ++ co.co_filename, ++ co.co_name, ++ co.co_qualname, # type: ignore[attr-defined] ++ co.co_firstlineno, ++ co.co_lnotab, ++ co.co_exceptiontable, # type: ignore[attr-defined] ++ co.co_freevars, ++ co.co_cellvars, ++ ) ++ elif hasattr(co, "co_posonlyargcount"): + co_args = ( + nargs, + 0, +diff --git a/torch/fx/operator_schemas.py b/torch/fx/operator_schemas.py +index 7ec3bc9a673..7c8971fb3ec 100644 +--- a/torch/fx/operator_schemas.py ++++ b/torch/fx/operator_schemas.py +@@ -61,18 +61,29 @@ def _torchscript_type_to_python_type(ts_type : 'torch._C.JitType') -> Any: + return eval(ts_type.annotation_str, _type_eval_globals) + + def _torchscript_schema_to_signature(ts_schema : torch._C.FunctionSchema) -> inspect.Signature: +- parameters : List[inspect.Parameter] = [] ++ from inspect import Parameter ++ parameters : List[Parameter] = [] + for arg in ts_schema.arguments: + arg_type = _torchscript_type_to_python_type(arg.type) +- default = arg.default_value if arg.has_default_value() else inspect.Parameter.empty ++ default = arg.default_value if arg.has_default_value() else Parameter.empty + # TODO: Figure out if this is safe. It seems like when generating the type signatures for + # PythonArgParser, we emit signatures with `input` instead of `self` as the first tensor + # argument name. Downstream, if someone converts that positional argument to a keyword + # argument, the name mismatch will break things, so here we're going to normalize the + # name to "input" + name = arg.name if arg.name != 'self' else 'input' +- kind = inspect.Parameter.KEYWORD_ONLY if arg.kwarg_only else inspect.Parameter.POSITIONAL_OR_KEYWORD +- parameters.append(inspect.Parameter(name=name, kind=kind, default=default, annotation=arg_type)) ++ kind = Parameter.KEYWORD_ONLY if arg.kwarg_only else Parameter.POSITIONAL_OR_KEYWORD ++ # "from" is a keyword therefore it must be a POSITIONAL_ONLY argument ++ if name == "from": ++ assert kind == Parameter.POSITIONAL_OR_KEYWORD ++ # ParameterKind type is internal implementation detail to inspec package ++ # which makes it hard to do type annoation ++ kind = Parameter.POSITIONAL_ONLY # type: ignore[assignment] ++ # This renders all previous arguments to positional only ++ for idx, p in enumerate(parameters): ++ assert p.kind == Parameter.POSITIONAL_OR_KEYWORD ++ parameters[idx] = Parameter(name=p.name, kind=Parameter.POSITIONAL_ONLY, default=p.default, annotation=p.annotation) ++ parameters.append(Parameter(name=name, kind=kind, default=default, annotation=arg_type)) + return_types = [_torchscript_type_to_python_type(ret.type) for ret in ts_schema.returns] + if len(return_types) == 0: + return_type = None +@@ -392,7 +403,12 @@ def _args_kwargs_to_normalized_args_kwargs(sig : inspect.Signature, args : Tuple + supported_parameter_types = { + inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY} + if any(p.kind not in supported_parameter_types for p in sig.parameters.values()): +- return None ++ # Add an exception for one signature, which is common for random/uniform, i.e.: ++ # Tensor(a!) self, float from=0, float to=1, *, Generator? generator=None ++ # `from` is Python keyword and as such functions with that signature should have ++ # positional-only args, but at the same time they could be dispatched as kwargs ++ if list(sig.parameters.keys()) != ['input', 'from', 'to', 'generator']: ++ return None + + bound_args = sig.bind(*args, **kwargs) + bound_args.apply_defaults() +diff --git a/torch/fx/proxy.py b/torch/fx/proxy.py +index 5d65f03e29b..120452dfd71 100644 +--- a/torch/fx/proxy.py ++++ b/torch/fx/proxy.py +@@ -1,4 +1,5 @@ + import dis ++import sys + import torch + import inspect + import operator +@@ -267,7 +268,13 @@ class Proxy: + assert frame is not None + calling_frame = frame.f_back + assert calling_frame is not None +- inst = list(dis.get_instructions(calling_frame.f_code))[calling_frame.f_lasti // 2] ++ inst_list = list(dis.get_instructions(calling_frame.f_code)) ++ if sys.version_info >= (3, 11): ++ from bisect import bisect_left ++ inst_idx = bisect_left(inst_list, calling_frame.f_lasti, key=lambda x: x.offset) ++ else: ++ inst_idx = calling_frame.f_lasti // 2 ++ inst = inst_list[inst_idx] + if inst.opname == 'UNPACK_SEQUENCE': + return (self[i] for i in range(inst.argval)) # type: ignore[index] + +@@ -282,7 +289,11 @@ class Proxy: + calling_frame = frame.f_back + assert calling_frame is not None + insts = list(dis.get_instructions(calling_frame.f_code)) +- cur = calling_frame.f_lasti // 2 ++ if sys.version_info >= (3, 11): ++ from bisect import bisect_left ++ cur = bisect_left(insts, calling_frame.f_lasti, key=lambda x: x.offset) ++ else: ++ cur = calling_frame.f_lasti // 2 + inst = insts[cur] + + if inst.opname == 'POP_JUMP_IF_TRUE': diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-warning-in-test-cpp-api.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-warning-in-test-cpp-api.patch new file mode 100644 index 000000000000..186cfe89dfab --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_fix-warning-in-test-cpp-api.patch @@ -0,0 +1,29 @@ +From 367387bab836c73c23719f97d929572c8b4e0fad Mon Sep 17 00:00:00 2001 +From: Nick Rossenbach +Date: Tue, 18 Apr 2023 23:31:51 +0200 +Subject: [PATCH] Add -Wno-error=nonnull for test/cpp/api/ + +On some platforms the build might fail due to the nonnull error +being triggered by different compiler behavior. + +Fix for #99278. +--- + test/cpp/api/CMakeLists.txt | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/test/cpp/api/CMakeLists.txt b/test/cpp/api/CMakeLists.txt +index 6b801a07318279..181139b01360c3 100644 +--- a/test/cpp/api/CMakeLists.txt ++++ b/test/cpp/api/CMakeLists.txt +@@ -80,6 +80,11 @@ if(NOT MSVC) + target_compile_options_if_supported(test_api "-Wno-maybe-uninitialized") + # gcc gives nonsensical warnings about variadic.h + target_compile_options_if_supported(test_api "-Wno-unused-but-set-parameter") ++ # the nonnull check might trigger for some build configurations, ++ # probably happening due to different code optimization ++ # (see e.g. https://rkoucha.fr/tech_corner/nonnull_gcc_attribute.html) ++ # this happened for a riscv build: https://github.com/pytorch/pytorch/issues/99278 ++ target_compile_options_if_supported(test_api "-Wno-error=nonnull") + endif() + + if(INSTALL_TEST) diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_skip-failing-singular-grad-test.patch b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_skip-failing-singular-grad-test.patch new file mode 100644 index 000000000000..0933e5a3abcd --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.13.1_skip-failing-singular-grad-test.patch @@ -0,0 +1,28 @@ +TestGradientsCPU.test_forward_mode_AD_linalg_det_singular_cpu_complex128 sometimes fails with + + raise GradcheckError(_get_notallclose_msg(a, n, j, i, complex_indices, test_imag, is_forward_ad) + jacobians_str) + torch.autograd.gradcheck.GradcheckError: While considering the imaginary part of complex inputs only, Jacobian computed with forward mode mismatch for output 0 with respect to input 0, + numerical:tensor([-0.1722-0.0510j, 0.0925-0.0695j, 0.1424+0.0206j, 0.0174-0.0618j], + dtype=torch.complex128) + analytical:tensor([-0.0000+0.0000j, 0.0925-0.0695j, 0.1424+0.0206j, 0.0174-0.0618j], + dtype=torch.complex128, grad_fn=) + +Author: Alexander Grund (TU Dresden) + +diff --git a/torch/testing/_internal/opinfo/definitions/linalg.py b/torch/testing/_internal/opinfo/definitions/linalg.py +index 0eb1ca07644..f3e5f9ec6ed 100644 +--- a/torch/testing/_internal/opinfo/definitions/linalg.py ++++ b/torch/testing/_internal/opinfo/definitions/linalg.py +@@ -1341,6 +1341,12 @@ op_db: List[OpInfo] = [ + "test_fn_fwgrad_bwgrad", + dtypes=(torch.complex128,), + ), ++ DecorateInfo( ++ unittest.skip(""), ++ "TestGradients", ++ "test_forward_mode_AD", ++ dtypes=(torch.complex128,), ++ ), + DecorateInfo( + unittest.skip("Skipped, see https://github.com//issues/84192"), + "TestGradients", diff --git a/easybuild/easyconfigs/p/pytest-rerunfailures/pytest-rerunfailures-12.0-GCCcore-12.2.0.eb b/easybuild/easyconfigs/p/pytest-rerunfailures/pytest-rerunfailures-12.0-GCCcore-12.2.0.eb new file mode 100644 index 000000000000..eefd52471b14 --- /dev/null +++ b/easybuild/easyconfigs/p/pytest-rerunfailures/pytest-rerunfailures-12.0-GCCcore-12.2.0.eb @@ -0,0 +1,21 @@ +easyblock = 'PythonPackage' + +name = 'pytest-rerunfailures' +version = '12.0' + +homepage = 'https://github.com/pytest-dev/pytest-rerunfailures' +description = """pytest plugin to re-run tests to eliminate flaky failures.""" + +toolchain = {'name': 'GCCcore', 'version': '12.2.0'} + +builddependencies = [('binutils', '2.39')] +dependencies = [('Python', '3.10.8')] + +use_pip = True +sanity_pip_check = True +download_dep_fail = False + +sources = [SOURCE_TAR_GZ] +checksums = ['784f462fa87fe9bdf781d0027d856b47a4bfe6c12af108f6bd887057a917b48e'] + +moduleclass = 'tools' diff --git a/easybuild/easyconfigs/p/pytest-shard/pytest-shard-0.1.2-GCCcore-12.2.0.eb b/easybuild/easyconfigs/p/pytest-shard/pytest-shard-0.1.2-GCCcore-12.2.0.eb new file mode 100644 index 000000000000..bd08dfe54d6d --- /dev/null +++ b/easybuild/easyconfigs/p/pytest-shard/pytest-shard-0.1.2-GCCcore-12.2.0.eb @@ -0,0 +1,28 @@ +easyblock = 'PythonPackage' + +name = 'pytest-shard' +version = '0.1.2' + +homepage = 'https://github.com/AdamGleave/pytest-shard' +description = """pytest plugin to support parallelism across multiple machines. + +Shards tests based on a hash of their test name enabling easy parallelism across machines, +suitable for a wide variety of continuous integration services. +Tests are split at the finest level of granularity, individual test cases, +enabling parallelism even if all of your tests are in a single file +(or even single parameterized test method). +""" + +toolchain = {'name': 'GCCcore', 'version': '12.2.0'} + +builddependencies = [('binutils', '2.39')] +dependencies = [('Python', '3.10.8')] + +use_pip = True +sanity_pip_check = True +download_dep_fail = False + +sources = [SOURCE_TAR_GZ] +checksums = ['b86a967fbfd1c8e50295095ccda031b7e890862ee06531d5142844f4c1d1cd67'] + +moduleclass = 'tools'