-
Notifications
You must be signed in to change notification settings - Fork 37
/
lecture_08-content.js
267 lines (267 loc) · 90 KB
/
lecture_08-content.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 16}], "This lecture: distributed training across multiple GPUs", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 18}], "Situation: compute (arithmetic logic units) is spread out, communication is slow", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 20}], "Hierarchy", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 21}], "- L1 cache / shared memory (small, fast)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 22}], "- L2 cache", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 23}], "- DRAM", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 24}], "- Single node, multi-GPU", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 25}], "- Multi-node, multi-GPU (big, slow)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 27}], "Last week: reduce DRAM accesses via fusion/tiling", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 28}], "This week: reduce communication across GPUs/nodes via replication/sharding", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 30}], "Game: organize computation so that communication is minimized", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 32}], "## Part 1: building blocks of distributed communication/computation", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 60}], "## Single GPU", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 61}], "https://miro.medium.com/v2/resize:fit:2000/format:webp/1*6xoBKi5kL2dZpivFe1-zgw.jpeg", {"width": "100%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 62}], "Memory bandwidth for DRAM for H100 NVL is 7.8 TB/sec", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 62}], "https://resources.nvidia.com/en-us-tensor-core/nvidia-tensor-core-gpu-datasheet", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 64}], "## Multi-node, multi-GPU", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 66}], "Traditionally:", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 67}], "https://media.springernature.com/lw685/springer-static/image/art%3A10.1186%2Fs42774-021-00098-3/MediaObjects/42774_2021_98_Fig1_HTML.png?as=webp", {"width": "40.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 68}], "- GPUs on same node communicate via a PCI(e) bus (v7.0, 16 lanes => 242 GB/sec)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 68}], "https://en.wikipedia.org/wiki/PCI_Express", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 69}], "- GPUs on different nodes communicate via Ethernet (~200 MB/sec)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 71}], "Both are too slow...", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 72}], "Key hardware advance: have GPUs connect *directly*, bypassing CPU", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 74}], "## InfiniBand", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 76}], "Standard developed in 1999; Mellanox created InfiniBand hardware, acquired by NVIDIA in 2019", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 77}], "Idea: Remote Direct Memory Access (RDMA) to connect nodes directly", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 78}], "https://lambdalabs.com/hubfs/Imported_Blog_Media/nvlink-diagram-update.png", {"width": "60.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 80}], "## NVLink/NVSwitch", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 82}], "NVIDIA developed proprietary protocol since 2014", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 83}], "4.5x more bandwidth than InfiniBand", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 84}], "https://developer.nvidia.com/blog/upgrading-multi-gpu-interconnectivity-with-the-third-generation-nvidia-nvswitch/", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 86}], "Within a node: NVLink connects GPUs directly, bypass CPU", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 87}], "https://developer-blogs.nvidia.com/wp-content/uploads/2022/08/NVLink-generations-1.png", {"width": "60.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 89}], "Across nodes: NVSwitch connects GPUs directly, bypass Ethernet", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 90}], "https://developer-blogs.nvidia.com/wp-content/uploads/2022/08/NVLink-all-to-all-connectivity-1.png", {"width": "60.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 92}], "H100 (Hopper): 18 NVLink 4.0 links => 900GB/sec", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 94}], "Bonus: NVSwitch has SHARP acceleration, which halves the communication for all-reduce", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 95}], "https://developer-blogs.nvidia.com/wp-content/uploads/2022/08/NVLink-SHARP-acceleration-1.png", {"width": "60.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 96}], "Will likely be many other hardware innovations in future", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 98}], "Let's check what our hardware setup is.", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 98}], "https://guide.ncloud-docs.com/docs/en/server-baremetal-a100-check-vpc", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "\tGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tNIC0\tNIC1\tNIC2\tNIC3\tNIC4\tNIC5\tNIC6\tNIC7\tNIC8\tNIC9\tNIC10\tNIC11\tNIC12\tNIC13\tNIC14\tNIC15\tNIC16\tNIC17\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU0\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tPXB\tPXB\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU1\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNODE\tNODE\tNODE\tPXB\tPXB\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU2\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNODE\tNODE\tNODE\tNODE\tNODE\tPXB\tPXB\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU3\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tPXB\tPXB\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU4\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tPXB\tPXB\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\t56-111,168-223\t1\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU5\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tPXB\tPXB\tNODE\tNODE\tNODE\tNODE\t56-111,168-223\t1\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU6\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tPXB\tPXB\tNODE\tNODE\t56-111,168-223\t1\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "GPU7\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tPXB\tPXB\t56-111,168-223\t1\t\tN/A", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC0\tPXB\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t X \tPIX\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC1\tPXB\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tPIX\t X \tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC2\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\t X \tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC3\tNODE\tPXB\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\t X \tPIX\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC4\tNODE\tPXB\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tPIX\t X \tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC5\tNODE\tNODE\tPXB\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\t X \tPIX\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC6\tNODE\tNODE\tPXB\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tPIX\t X \tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC7\tNODE\tNODE\tNODE\tPXB\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\t X \tPIX\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC8\tNODE\tNODE\tNODE\tPXB\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tPIX\t X \tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC9\tSYS\tSYS\tSYS\tSYS\tPXB\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t X \tPIX\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC10\tSYS\tSYS\tSYS\tSYS\tPXB\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tPIX\t X \tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC11\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\t X \tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC12\tSYS\tSYS\tSYS\tSYS\tNODE\tPXB\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\t X \tPIX\tNODE\tNODE\tNODE\tNODE\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC13\tSYS\tSYS\tSYS\tSYS\tNODE\tPXB\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tPIX\t X \tNODE\tNODE\tNODE\tNODE\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC14\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tPXB\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\t X \tPIX\tNODE\tNODE\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC15\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tPXB\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tPIX\t X \tNODE\tNODE\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC16\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tPXB\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\t X \tPIX\t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC17\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tPXB\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tPIX\t X \t\t\t\t", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "Legend:", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " X = Self", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " PIX = Connection traversing at most a single PCIe bridge", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NV# = Connection traversing a bonded set of # NVLinks", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "NIC Legend:", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC0: mlx5_0", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC1: mlx5_1", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC2: mlx5_2", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC3: mlx5_3", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC4: mlx5_4", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC5: mlx5_5", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC6: mlx5_6", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC7: mlx5_7", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC8: mlx5_8", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC9: mlx5_9", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC10: mlx5_10", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC11: mlx5_11", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC12: mlx5_12", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC13: mlx5_13", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC14: mlx5_14", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC15: mlx5_15", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC16: mlx5_16", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], " NIC17: mlx5_17", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 100}, {"name": "note_system", "filename": "lecture_08.py", "lineno": 713}], "", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 33}, {"name": "hardware", "filename": "lecture_08.py", "lineno": 101}], "Note GPUs are connected via NV18, also connected to NICs (for PCIe)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 105}], "Collective operations are the conceptual primitives used for distributed programming", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 105}], "https://en.wikipedia.org/wiki/Collective_operation", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 107}], "- Collective means that specify communication pattern across many (e.g., 256) nodes", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 108}], "- These are classic in the parallel programming literature from the 1980s", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 109}], "- For SIMD (Single Instruction, Multiple Data) parallelism", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 110}], "- Better/faster abstraction than managing point-to-point communication yourself", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 112}], "Terminology:", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 113}], "- Rank: a device (e.g., GPU)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 114}], "- World size: number of devices", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 116}], "## Broadcast", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 116}], "https://pytorch.org/tutorials/_images/broadcast.png", {"width": "30.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 118}], "## Scatter", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 118}], "https://pytorch.org/tutorials/_images/scatter.png", {"width": "30.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 120}], "## Gather", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 120}], "https://pytorch.org/tutorials/_images/gather.png", {"width": "30.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 122}], "## Reduce", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 122}], "https://pytorch.org/tutorials/_images/reduce.png", {"width": "30.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 124}], "## All-gather", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 124}], "https://pytorch.org/tutorials/_images/all_gather.png", {"width": "30.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 126}], "## Reduce-scatter", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 126}], "https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/_images/reducescatter.png", {"width": "30.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 128}], "## All-reduce = reduce-scatter + all-gather", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 128}], "https://pytorch.org/tutorials/_images/all_reduce.png", {"width": "30.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 130}], "Framework to think about it:", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 131}], "- Reduce: performs some associative/commutative operation (sum, min, max)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 132}], "- Broadcast/scatter is inverse of gather", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 34}, {"name": "collective_operations", "filename": "lecture_08.py", "lineno": 133}], "- All: destination is all", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 137}], "## PyTorch distributed library (`torch.distributed`)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 139}], "Reference", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 139}], "https://pytorch.org/docs/stable/distributed.html", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 141}], "- Provides clean interface for collective operations", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 141}], "<function all_reduce at 0x14f04de8f910>", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 142}], "- Backends: gloo (CPU), nccl (GPU)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 143}], "- Also supports higher-level abstractions", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 143}], "<class 'torch.distributed.fsdp.fully_sharded_data_parallel.FullyShardedDataParallel'>", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 145}], "## NVIDIA Collective Communication Library (NCCL)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 147}], "NCCL translates collective operations into low-level packets.", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 148}], "https://www.nvidia.com/en-us/on-demand/session/gtcspring21-s31880/", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 150}], "1. Detect toplogy of hardware (e.g., number of nodes, switches, NVLink/PCIe)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 151}], "2. Optimize the path between ranks; ring (good bandwidth), tree (good latency)", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 152}], "https://developer-blogs.nvidia.com/wp-content/uploads/2019/02/DBtree.png", {"width": "40.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 153}], "3. Launches CUDA kernels to send/receive data", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 155}], "## Examples", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 35}, {"name": "torch_distributed", "filename": "lecture_08.py", "lineno": 156}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 4)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 167}], "### All-reduce", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 172}], "Rank 3 [before all-reduce]: tensor([3., 4., 5., 6.], device='cuda:3')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 172}], "Rank 0 [before all-reduce]: tensor([0., 1., 2., 3.], device='cuda:0')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 172}], "Rank 1 [before all-reduce]: tensor([1., 2., 3., 4.], device='cuda:1')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 172}], "Rank 2 [before all-reduce]: tensor([2., 3., 4., 5.], device='cuda:2')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 174}], "Rank 0 [after all-reduce]: tensor([ 6., 10., 14., 18.], device='cuda:0')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 174}], "Rank 3 [after all-reduce]: tensor([ 6., 10., 14., 18.], device='cuda:3')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 174}], "Rank 1 [after all-reduce]: tensor([ 6., 10., 14., 18.], device='cuda:1')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 174}], "Rank 2 [after all-reduce]: tensor([ 6., 10., 14., 18.], device='cuda:2')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 178}], "### Reduce-scatter", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 184}], "Rank 1 [before reduce-scatter]: input = tensor([1., 2., 3., 4.], device='cuda:1'), output = tensor([0.], device='cuda:1')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 184}], "Rank 2 [before reduce-scatter]: input = tensor([2., 3., 4., 5.], device='cuda:2'), output = tensor([0.], device='cuda:2')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 184}], "Rank 0 [before reduce-scatter]: input = tensor([0., 1., 2., 3.], device='cuda:0'), output = tensor([0.], device='cuda:0')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 184}], "Rank 3 [before reduce-scatter]: input = tensor([3., 4., 5., 6.], device='cuda:3'), output = tensor([0.], device='cuda:3')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 186}], "Rank 0 [after reduce-scatter]: input = tensor([0., 1., 2., 3.], device='cuda:0'), output = tensor([6.], device='cuda:0')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 186}], "Rank 2 [after reduce-scatter]: input = tensor([2., 3., 4., 5.], device='cuda:2'), output = tensor([14.], device='cuda:2')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 186}], "Rank 1 [after reduce-scatter]: input = tensor([1., 2., 3., 4.], device='cuda:1'), output = tensor([10.], device='cuda:1')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 186}], "Rank 3 [after reduce-scatter]: input = tensor([3., 4., 5., 6.], device='cuda:3'), output = tensor([18.], device='cuda:3')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 190}], "### All-gather", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 196}], "Rank 2 [before all-gather]: input = tensor([14.], device='cuda:2'), output = tensor([2., 3., 4., 5.], device='cuda:2')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 196}], "Rank 3 [before all-gather]: input = tensor([18.], device='cuda:3'), output = tensor([3., 4., 5., 6.], device='cuda:3')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 196}], "Rank 0 [before all-gather]: input = tensor([6.], device='cuda:0'), output = tensor([ 6., 10., 14., 18.], device='cuda:0')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 196}], "Rank 1 [before all-gather]: input = tensor([10.], device='cuda:1'), output = tensor([1., 2., 3., 4.], device='cuda:1')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 198}], "Rank 3 [after all-gather]: input = tensor([18.], device='cuda:3'), output = tensor([ 6., 10., 14., 18.], device='cuda:3')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 198}], "Rank 2 [after all-gather]: input = tensor([14.], device='cuda:2'), output = tensor([ 6., 10., 14., 18.], device='cuda:2')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 198}], "Rank 1 [after all-gather]: input = tensor([10.], device='cuda:1'), output = tensor([ 6., 10., 14., 18.], device='cuda:1')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 198}], "Rank 0 [after all-gather]: input = tensor([6.], device='cuda:0'), output = tensor([ 6., 10., 14., 18.], device='cuda:0')", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "collective_operations_main", "filename": "lecture_08.py", "lineno": 201}], "Recall that all-reduce = reduce-scatter + all-gather!", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 207}], "## Benchmarking", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 207}], "https://github.com/stas00/ml-engineering/blob/master/network/benchmarks/all_reduce_bench.py", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 209}], "Let's see how fast commmunication happens (will restrict to just one node).", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 211}], "### All-reduce", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 213}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 2)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 243}], "Rank 0: all_reduce(world_size=2, num_elements=1073741824) took 12 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 243}], "Rank 1: all_reduce(world_size=2, num_elements=1073741824) took 13 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 250}], "Rank 0: all_reduce estimated bandwidth = 320 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 250}], "Rank 1: all_reduce estimated bandwidth = 320 GB/sec", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 214}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 4)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 243}], "Rank 1: all_reduce(world_size=4, num_elements=1073741824) took 18 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 243}], "Rank 0: all_reduce(world_size=4, num_elements=1073741824) took 18 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 243}], "Rank 3: all_reduce(world_size=4, num_elements=1073741824) took 18 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 243}], "Rank 2: all_reduce(world_size=4, num_elements=1073741824) took 18 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 250}], "Rank 1: all_reduce estimated bandwidth = 335 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 250}], "Rank 2: all_reduce estimated bandwidth = 335 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 250}], "Rank 3: all_reduce estimated bandwidth = 335 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "all_reduce", "filename": "lecture_08.py", "lineno": 250}], "Rank 0: all_reduce estimated bandwidth = 335 GB/sec", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 216}], "### Reduce-scatter", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 218}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 2)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 275}], "Rank 0: reduce_scatter(world_size=2, num_elements=1073741824) took 15 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 275}], "Rank 1: reduce_scatter(world_size=2, num_elements=1073741824) took 15 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 282}], "Rank 0: reduce_scatter estimated bandwidth = 132 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 282}], "Rank 1: reduce_scatter estimated bandwidth = 132 GB/sec", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 219}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 4)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 275}], "Rank 3: reduce_scatter(world_size=4, num_elements=1073741824) took 36 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 275}], "Rank 0: reduce_scatter(world_size=4, num_elements=1073741824) took 36 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 275}], "Rank 2: reduce_scatter(world_size=4, num_elements=1073741824) took 36 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 275}], "Rank 1: reduce_scatter(world_size=4, num_elements=1073741824) took 36 ms", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 282}], "Rank 3: reduce_scatter estimated bandwidth = 83 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 282}], "Rank 0: reduce_scatter estimated bandwidth = 83 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 282}], "Rank 2: reduce_scatter estimated bandwidth = 83 GB/sec", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "reduce_scatter", "filename": "lecture_08.py", "lineno": 282}], "Rank 1: reduce_scatter estimated bandwidth = 83 GB/sec", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 221}], "Reference on reasoning about operations:", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 37}, {"name": "benchmarking", "filename": "lecture_08.py", "lineno": 221}], "https://github.com/NVIDIA/nccl-tests/blob/master/doc/PERFORMANCE.md#allreduce", {"color": "gray"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 288}], "Execution model: when launch a kernel (e.g., matmul), grid -> blocks -> threads", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 289}], "Multiple kernels execute asynchronously (A || B)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 290}], "Synchronization points: `cuda.torch.synchornize` or copy to CPU", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 291}], "Need a way to do more fine-grained synchronization and stay on GPU...", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 293}], "CUDA stream: a sequence of operations that execute in order", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 294}], "Different streams can execute concurrently", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 301}], "Simple example: need to do two operations", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 302}], "1. Matrix multiplication", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 303}], "2. Copy matrix from CPU to GPU", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 328}], "run1: [276.9, 276.9, 277.0] (mean 276.9 ms)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 38}, {"name": "cuda_streams", "filename": "lecture_08.py", "lineno": 329}], "run2: [164.3, 164.3, 164.4] (mean 164.3 ms)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 40}], "## Part 2: distributed training", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 41}], "https://uvadlc-notebooks.readthedocs.io/en/latest/_images/parallelism_strategies_overview.svg", {"width": "100.0%"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 42}, {"name": "ddp", "filename": "lecture_08.py", "lineno": 333}], "## Distributed data parallel", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 42}, {"name": "ddp", "filename": "lecture_08.py", "lineno": 340}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 1)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 0, loss = 0.013002010062336922, params = ['1024x1024[-0.0279...]', '1024x1024[-0.0299...]', '1024x1024[-0.0279...]', '1024x1024[-0.0299...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 1, loss = 0.0031716423109173775, params = ['1024x1024[-0.0269...]', '1024x1024[-0.0308...]', '1024x1024[-0.0269...]', '1024x1024[-0.0307...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 2, loss = 0.002139497082680464, params = ['1024x1024[-0.0259...]', '1024x1024[-0.0318...]', '1024x1024[-0.0259...]', '1024x1024[-0.0315...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 3, loss = 0.002267548581585288, params = ['1024x1024[-0.0249...]', '1024x1024[-0.0327...]', '1024x1024[-0.0249...]', '1024x1024[-0.0324...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 4, loss = 0.002412830712273717, params = ['1024x1024[-0.0239...]', '1024x1024[-0.0336...]', '1024x1024[-0.0239...]', '1024x1024[-0.0333...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 42}, {"name": "ddp", "filename": "lecture_08.py", "lineno": 341}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 2)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 0, loss = 0.013183665461838245, params = ['1024x1024[-0.0279...]', '1024x1024[-0.0299...]', '1024x1024[-0.0279...]', '1024x1024[-0.0299...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 0, loss = 0.012820353731513023, params = ['1024x1024[-0.0279...]', '1024x1024[-0.0299...]', '1024x1024[-0.0279...]', '1024x1024[-0.0299...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 1, loss = 0.003184368833899498, params = ['1024x1024[-0.0269...]', '1024x1024[-0.0308...]', '1024x1024[-0.0269...]', '1024x1024[-0.0307...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 1, loss = 0.003158915787935257, params = ['1024x1024[-0.0269...]', '1024x1024[-0.0308...]', '1024x1024[-0.0269...]', '1024x1024[-0.0307...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 2, loss = 0.002138055395334959, params = ['1024x1024[-0.0259...]', '1024x1024[-0.0318...]', '1024x1024[-0.0259...]', '1024x1024[-0.0315...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 2, loss = 0.0021409383043646812, params = ['1024x1024[-0.0259...]', '1024x1024[-0.0318...]', '1024x1024[-0.0259...]', '1024x1024[-0.0315...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 3, loss = 0.0022640812676399946, params = ['1024x1024[-0.0249...]', '1024x1024[-0.0327...]', '1024x1024[-0.0249...]', '1024x1024[-0.0324...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 3, loss = 0.002271015662699938, params = ['1024x1024[-0.0249...]', '1024x1024[-0.0327...]', '1024x1024[-0.0249...]', '1024x1024[-0.0324...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 4, loss = 0.002411414170637727, params = ['1024x1024[-0.0239...]', '1024x1024[-0.0336...]', '1024x1024[-0.0239...]', '1024x1024[-0.0333...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 4, loss = 0.0024142470210790634, params = ['1024x1024[-0.0239...]', '1024x1024[-0.0336...]', '1024x1024[-0.0239...]', '1024x1024[-0.0333...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 42}, {"name": "ddp", "filename": "lecture_08.py", "lineno": 342}, {"name": "spawn", "filename": "lecture_08.py", "lineno": 704}], "spawn(world_size = 4)", {})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 2: step = 0, loss = 0.01232093945145607, params = ['1024x1024[-0.0279...]', '1024x1024[-0.0299...]', '1024x1024[-0.0279...]', '1024x1024[-0.0299...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 3: step = 0, loss = 0.013319768011569977, params = ['1024x1024[-0.0279...]', '1024x1024[-0.0299...]', '1024x1024[-0.0279...]', '1024x1024[-0.0299...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 0, loss = 0.013228634372353554, params = ['1024x1024[-0.0279...]', '1024x1024[-0.0299...]', '1024x1024[-0.0279...]', '1024x1024[-0.0299...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 0, loss = 0.013138696551322937, params = ['1024x1024[-0.0279...]', '1024x1024[-0.0299...]', '1024x1024[-0.0279...]', '1024x1024[-0.0299...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 3: step = 1, loss = 0.0031576615292578936, params = ['1024x1024[-0.0269...]', '1024x1024[-0.0308...]', '1024x1024[-0.0269...]', '1024x1024[-0.0307...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 2: step = 1, loss = 0.0031601698137819767, params = ['1024x1024[-0.0269...]', '1024x1024[-0.0308...]', '1024x1024[-0.0269...]', '1024x1024[-0.0307...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 1, loss = 0.0032222899608314037, params = ['1024x1024[-0.0269...]', '1024x1024[-0.0308...]', '1024x1024[-0.0269...]', '1024x1024[-0.0307...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 1, loss = 0.003146447241306305, params = ['1024x1024[-0.0269...]', '1024x1024[-0.0308...]', '1024x1024[-0.0269...]', '1024x1024[-0.0307...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 2, loss = 0.0021610602270811796, params = ['1024x1024[-0.0259...]', '1024x1024[-0.0318...]', '1024x1024[-0.0259...]', '1024x1024[-0.0315...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 2, loss = 0.002115050796419382, params = ['1024x1024[-0.0259...]', '1024x1024[-0.0318...]', '1024x1024[-0.0259...]', '1024x1024[-0.0315...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 3: step = 2, loss = 0.0021079229190945625, params = ['1024x1024[-0.0259...]', '1024x1024[-0.0318...]', '1024x1024[-0.0259...]', '1024x1024[-0.0315...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 2: step = 2, loss = 0.0021739539224654436, params = ['1024x1024[-0.0259...]', '1024x1024[-0.0318...]', '1024x1024[-0.0259...]', '1024x1024[-0.0315...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 3, loss = 0.002295861719176173, params = ['1024x1024[-0.0249...]', '1024x1024[-0.0327...]', '1024x1024[-0.0249...]', '1024x1024[-0.0324...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 3, loss = 0.0022323005832731724, params = ['1024x1024[-0.0249...]', '1024x1024[-0.0327...]', '1024x1024[-0.0249...]', '1024x1024[-0.0324...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 2: step = 3, loss = 0.002271739998832345, params = ['1024x1024[-0.0249...]', '1024x1024[-0.0327...]', '1024x1024[-0.0249...]', '1024x1024[-0.0324...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 3: step = 3, loss = 0.0022702913265675306, params = ['1024x1024[-0.0249...]', '1024x1024[-0.0327...]', '1024x1024[-0.0249...]', '1024x1024[-0.0324...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 2: step = 4, loss = 0.0024013773072510958, params = ['1024x1024[-0.0239...]', '1024x1024[-0.0336...]', '1024x1024[-0.0239...]', '1024x1024[-0.0333...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 1: step = 4, loss = 0.002457376103848219, params = ['1024x1024[-0.0239...]', '1024x1024[-0.0336...]', '1024x1024[-0.0239...]', '1024x1024[-0.0333...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 0: step = 4, loss = 0.0023654524702578783, params = ['1024x1024[-0.0239...]', '1024x1024[-0.0336...]', '1024x1024[-0.0239...]', '1024x1024[-0.0333...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "spawn_main", "filename": "spawn.py", "lineno": 116}, {"name": "_main", "filename": "spawn.py", "lineno": 129}, {"name": "_bootstrap", "filename": "process.py", "lineno": 314}, {"name": "run", "filename": "process.py", "lineno": 108}, {"name": "_wrap", "filename": "spawn.py", "lineno": 68}, {"name": "ddp_main", "filename": "lecture_08.py", "lineno": 381}], "Rank 3: step = 4, loss = 0.0024271169677376747, params = ['1024x1024[-0.0239...]', '1024x1024[-0.0336...]', '1024x1024[-0.0239...]', '1024x1024[-0.0333...]']", {"font-family": "monospace", "white-space": "pre"})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 42}, {"name": "ddp", "filename": "lecture_08.py", "lineno": 344}], "Notes:", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 42}, {"name": "ddp", "filename": "lecture_08.py", "lineno": 345}], "- Losses are different across nodes (computed on local data)", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 42}, {"name": "ddp", "filename": "lecture_08.py", "lineno": 346}], "- Gradients are same, and therefore parameters are the same", {})
addText([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 43}, {"name": "ddp_zero3", "filename": "lecture_08.py", "lineno": 393}], "## Distributed data parallel (DDP) with ZeRO stage 3", {})
addImage([{"name": "lecture_08", "filename": "lecture_08.py", "lineno": 43}, {"name": "ddp_zero3", "filename": "lecture_08.py", "lineno": 395}], "https://production-media.paperswithcode.com/methods/Screen_Shot_2021-07-26_at_3.17.43_PM_3oyU7Qb.png", {"width": "50.0%"})