From aef3eeeb5ce80afff447b2cf6fc788f128f0e1fd Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Thu, 13 Oct 2022 16:53:03 +0200 Subject: [PATCH 1/2] adding easyconfigs: PyTorch-1.12.0-foss-2022a.eb --- .../p/PyTorch/PyTorch-1.12.0-foss-2022a.eb | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb new file mode 100644 index 000000000000..06f29a6402ef --- /dev/null +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb @@ -0,0 +1,97 @@ +name = 'PyTorch' +version = '1.12.0' + +homepage = 'https://pytorch.org/' +description = """Tensors and Dynamic neural networks in Python with strong GPU acceleration. +PyTorch is a deep learning framework that puts Python first.""" + +toolchain = {'name': 'foss', 'version': '2022a'} + +sources = [{ + 'filename': '%(name)s-%(version)s.tar.gz', + 'git_config': { + 'url': 'https://github.com/pytorch', + 'repo_name': 'pytorch', + 'tag': 'v%(version)s', + 'recursive': True, + }, +}] +patches = [ + '%(name)s-1.7.0_avoid-nan-in-test-torch.patch', + '%(name)s-1.7.0_disable-dev-shm-test.patch', + '%(name)s-1.9.0_limit-world-size-for-zero-redundancy-opt-test.patch', + '%(name)s-1.10.0_fix-test-dataloader-fixed-affinity.patch', + '%(name)s-1.10.0_skip_cmake_rpath.patch', + '%(name)s-1.11.0_increase-distributed-test-timeout.patch', + '%(name)s-1.11.0_increase_c10d_gloo_timeout.patch', + '%(name)s-1.11.0_disable_failing_jit_cuda_fuser_tests.patch', +] +checksums = [ + None, # PyTorch-1.12.0.tar.gz + 'b899aa94d9e60f11ee75a706563312ccefa9cf432756c470caa8e623991c8f18', # PyTorch-1.7.0_avoid-nan-in-test-torch.patch + '622cb1eaeadc06e13128a862d9946bcc1f1edd3d02b259c56a9aecc4d5406b8a', # PyTorch-1.7.0_disable-dev-shm-test.patch + # PyTorch-1.9.0_limit-world-size-for-zero-redundancy-opt-test.patch + 'ff573660913ce055e24cfd194ce747ba5685091c631cfd443eae2a99d56b57ea', + # PyTorch-1.10.0_fix-test-dataloader-fixed-affinity.patch + '313dca681f45ce3bc7c4557fdcdcbe0b77216d2c708fa30a2ec0e22c44876707', + 'ac05943bb205623f91ef140aa00869efc5fe844184bd666bebf5405808610448', # PyTorch-1.10.0_skip_cmake_rpath.patch + # PyTorch-1.11.0_increase-distributed-test-timeout.patch + '087ad20163a1291773ae3457569b80523080eb3731e210946459b2333a919f3f', + # PyTorch-1.11.0_increase_c10d_gloo_timeout.patch + '20cd4a8663f74ab326fdb032b926bf5c7e94d9750c515ab9050927ba00cf1953', + # PyTorch-1.11.0_disable_failing_jit_cuda_fuser_tests.patch + 'e7bfe120a8b3fe2b40dac6839852a5fbab3cb3429fbe44a0fc3a1800adaaee51', +] + +builddependencies = [ + ('CMake', '3.23.1'), + ('hypothesis', '6.46.7'), +] +dependencies = [ + ('Ninja', '1.10.2'), # Required for JIT compilation of C++ extensions + ('Python', '3.10.4'), + ('protobuf', '3.19.4'), + ('protobuf-python', '3.19.4'), + ('pybind11', '2.9.2'), + ('SciPy-bundle', '2022.05'), + ('PyYAML', '6.0'), + ('MPFR', '4.1.0'), + ('GMP', '6.2.1'), + ('numactl', '2.0.14'), + ('FFmpeg', '4.4.2'), + ('Pillow', '9.1.1'), + ('expecttest', '0.1.3'), +] + +osdependencies = [('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel')] + +excluded_tests = { + '': [ + # Bad tests: https://github.com/pytorch/pytorch/issues/60260 + 'distributed/elastic/utils/distributed_test', + 'distributed/elastic/multiprocessing/api_test', + # These tests fail on A10s at the very least, they time out forever no matter how long the timeout is. + # Possibly related to NCCL 2.8.3: https://docs.nvidia.com/deeplearning/nccl/release-notes/rel_2-8-3.html + # 'distributed/test_distributed_fork', + 'distributed/test_distributed_spawn', + # Fails on A10s: https://github.com/pytorch/pytorch/issues/63079 + 'test_optim', + # Test from this suite timeout often. The process group backend is deprecated anyway + # 'distributed/rpc/test_process_group_agent', + # This test fails constently when run as part of the test suite, but succeeds when run interactively + 'test_model_dump', + # These tests appear flaky, possibly related to number of GPUs that are used + 'distributed/fsdp/test_fsdp_memory', + 'distributed/fsdp/test_fsdp_overlap', + ] +} + +runtest = "cd test && PYTHONUNBUFFERED=1 %(python)s run_test.py --continue-through-error --verbose %(excluded_tests)s" + +# several tests are known to be flaky, and fail in some contexts (like having multiple GPUs available), +# so we allow up to 400 (out of ~90k) tests to fail before treating the installation to be faulty +max_failed_tests = 400 + +tests = ['%(name)s-check-cpp-extension.py'] + +moduleclass = 'ai' From 7ab5413e8737afd32254d31a3200c980d3d23e16 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Thu, 13 Oct 2022 20:53:48 +0200 Subject: [PATCH 2/2] use source tarball --- .../p/PyTorch/PyTorch-1.12.0-foss-2022a.eb | 39 ++++++++----------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb index 06f29a6402ef..acaa8d0b12a9 100644 --- a/easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb +++ b/easybuild/easyconfigs/p/PyTorch/PyTorch-1.12.0-foss-2022a.eb @@ -7,15 +7,8 @@ PyTorch is a deep learning framework that puts Python first.""" toolchain = {'name': 'foss', 'version': '2022a'} -sources = [{ - 'filename': '%(name)s-%(version)s.tar.gz', - 'git_config': { - 'url': 'https://github.com/pytorch', - 'repo_name': 'pytorch', - 'tag': 'v%(version)s', - 'recursive': True, - }, -}] +source_urls = [GITHUB_RELEASE] +sources = ['%(namelower)s-v%(version)s.tar.gz'] patches = [ '%(name)s-1.7.0_avoid-nan-in-test-torch.patch', '%(name)s-1.7.0_disable-dev-shm-test.patch', @@ -27,20 +20,20 @@ patches = [ '%(name)s-1.11.0_disable_failing_jit_cuda_fuser_tests.patch', ] checksums = [ - None, # PyTorch-1.12.0.tar.gz - 'b899aa94d9e60f11ee75a706563312ccefa9cf432756c470caa8e623991c8f18', # PyTorch-1.7.0_avoid-nan-in-test-torch.patch - '622cb1eaeadc06e13128a862d9946bcc1f1edd3d02b259c56a9aecc4d5406b8a', # PyTorch-1.7.0_disable-dev-shm-test.patch - # PyTorch-1.9.0_limit-world-size-for-zero-redundancy-opt-test.patch - 'ff573660913ce055e24cfd194ce747ba5685091c631cfd443eae2a99d56b57ea', - # PyTorch-1.10.0_fix-test-dataloader-fixed-affinity.patch - '313dca681f45ce3bc7c4557fdcdcbe0b77216d2c708fa30a2ec0e22c44876707', - 'ac05943bb205623f91ef140aa00869efc5fe844184bd666bebf5405808610448', # PyTorch-1.10.0_skip_cmake_rpath.patch - # PyTorch-1.11.0_increase-distributed-test-timeout.patch - '087ad20163a1291773ae3457569b80523080eb3731e210946459b2333a919f3f', - # PyTorch-1.11.0_increase_c10d_gloo_timeout.patch - '20cd4a8663f74ab326fdb032b926bf5c7e94d9750c515ab9050927ba00cf1953', - # PyTorch-1.11.0_disable_failing_jit_cuda_fuser_tests.patch - 'e7bfe120a8b3fe2b40dac6839852a5fbab3cb3429fbe44a0fc3a1800adaaee51', + {'pytorch-v1.12.0.tar.gz': '46eff236370b759c427b03ff535c3597099043e8e467b8f81f9cd4b258a7a321'}, + {'PyTorch-1.7.0_avoid-nan-in-test-torch.patch': 'b899aa94d9e60f11ee75a706563312ccefa9cf432756c470caa8e623991c8f18'}, + {'PyTorch-1.7.0_disable-dev-shm-test.patch': '622cb1eaeadc06e13128a862d9946bcc1f1edd3d02b259c56a9aecc4d5406b8a'}, + {'PyTorch-1.9.0_limit-world-size-for-zero-redundancy-opt-test.patch': + 'ff573660913ce055e24cfd194ce747ba5685091c631cfd443eae2a99d56b57ea'}, + {'PyTorch-1.10.0_fix-test-dataloader-fixed-affinity.patch': + '313dca681f45ce3bc7c4557fdcdcbe0b77216d2c708fa30a2ec0e22c44876707'}, + {'PyTorch-1.10.0_skip_cmake_rpath.patch': 'ac05943bb205623f91ef140aa00869efc5fe844184bd666bebf5405808610448'}, + {'PyTorch-1.11.0_increase-distributed-test-timeout.patch': + '087ad20163a1291773ae3457569b80523080eb3731e210946459b2333a919f3f'}, + {'PyTorch-1.11.0_increase_c10d_gloo_timeout.patch': + '20cd4a8663f74ab326fdb032b926bf5c7e94d9750c515ab9050927ba00cf1953'}, + {'PyTorch-1.11.0_disable_failing_jit_cuda_fuser_tests.patch': + 'e7bfe120a8b3fe2b40dac6839852a5fbab3cb3429fbe44a0fc3a1800adaaee51'}, ] builddependencies = [