Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

IPU Integration 5/5 #7867

Merged
merged 69 commits into from
Jun 11, 2021
Merged
Changes from 1 commit
Commits
Show all changes
69 commits
Select commit Hold shift + click to select a range
f75f445
Initial changes
Feb 22, 2021
a4a60c2
Merge branch 'master' into wip/acc
Mar 24, 2021
dc9744b
Add broken example for now
Mar 24, 2021
931bb74
Fix reference
Apr 7, 2021
9b18baf
Merge branch 'master' into wip/acc
May 11, 2021
c617f02
Fix format
May 11, 2021
522a81f
Code runs
May 11, 2021
0c00360
Fixes
May 26, 2021
30c1370
Merge branch 'master' into wip/acc
May 26, 2021
adbdb2a
Clear up files
May 26, 2021
3e733af
Add tests, helpers, fixes
May 27, 2021
a51f23e
Small cleanups
May 27, 2021
be7de87
Refactors based on review
Jun 1, 2021
83c8a79
Swap to special tests
Jun 1, 2021
a6018e5
Add special tests
Jun 1, 2021
0e71bbe
Add source
Jun 1, 2021
6e38bd1
Cleanups
Jun 1, 2021
526383f
Add logic to attach/detach model from devices
Jun 2, 2021
e18039c
Fixes for tests
Jun 2, 2021
2e43fee
Fixes for tests
Jun 2, 2021
53d31a0
Move earlier
Jun 2, 2021
6241432
Cleanups
Jun 2, 2021
d249a13
Add check for nvcc
Jun 2, 2021
d08cf39
Add tests, cleanups
Jun 2, 2021
7469744
Fix errors
Jun 3, 2021
f474c5b
fix
Jun 3, 2021
e178d5f
Try condition
Jun 3, 2021
c704920
Add missing annotation
Jun 3, 2021
c54a216
Clearer
Jun 3, 2021
2ea1766
Clearer message
Jun 3, 2021
751f0ea
Fix variable
Jun 3, 2021
87e4c8a
Merge branch 'master' into wip/acc
Jun 7, 2021
61d2014
Cleanups
Jun 7, 2021
d76f491
Merge branch 'master' into wip/acc
Jun 7, 2021
62860ff
Add comment
Jun 7, 2021
b5a5032
CHANGELOG.md
Jun 7, 2021
72ed367
Add simple selection test
Jun 7, 2021
88fba4a
Merge branch 'master' into wip/acc
Jun 7, 2021
3fb031d
Remove special=True to see what happens
Jun 7, 2021
515d491
Fix test
Jun 7, 2021
ed16808
Update tests/accelerators/test_ipu.py
Jun 7, 2021
7f50295
Convert ipu_cores -> ipus
Jun 7, 2021
c53cf88
Add typing, fail earlier
Jun 7, 2021
a6dbd8a
simplify precision
Jun 7, 2021
953454b
Add test, add helper
Jun 8, 2021
24829bf
fix accum
Jun 8, 2021
d7d38c5
Update pytorch_lightning/plugins/training_type/ipu.py
Jun 8, 2021
c333e27
Use stages
Jun 8, 2021
9d3741a
Make sure warning message returned
Jun 8, 2021
fd1899a
thorw error
Jun 8, 2021
0727954
Add more tests, use fs
Jun 8, 2021
ce182f7
add comment
Jun 8, 2021
7e81bcd
Clean
Jun 8, 2021
d1788d1
Address feedback, add IPU tests
Jun 9, 2021
08e5338
Fixes
Jun 9, 2021
45dc6a6
Fix signature
Jun 9, 2021
de040c6
Add types
Jun 9, 2021
42d7ab0
Remove autoround
Jun 9, 2021
8ab62c4
Merge branch 'master' into wip/acc
Jun 9, 2021
36f3672
Add docstring
Jun 9, 2021
5f89714
Merge branch 'master' into wip/acc
Jun 10, 2021
d0f98f3
Merge branch 'master' into wip/acc
Jun 10, 2021
f9d61c5
ipu_cores -> ipus
Jun 10, 2021
cf48ff8
Add test, remove unnecessary precision set
Jun 11, 2021
02a75b5
Add optimizer test
Jun 11, 2021
d18fc55
Add precision back with test
Jun 11, 2021
043884a
Address code review
Jun 11, 2021
b249391
Change to probs
Jun 11, 2021
b0dd206
Move some of the asserts earlier
Jun 11, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Address code review
SeanNaren committed Jun 11, 2021
commit 043884af85baad959545048f0bb29c4d1abaa147
23 changes: 17 additions & 6 deletions pytorch_lightning/plugins/training_type/ipu.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
# Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
SeanNaren marked this conversation as resolved.
Show resolved Hide resolved
import json
import os
@@ -94,9 +107,7 @@ def __init__(
options = {"autoReport.all": self.autoreport}
if self.autoreport_dir:
self._fs = get_filesystem(str(self.autoreport_dir))

if not self._fs.exists(self.autoreport_dir):
self._fs.makedirs(self.autoreport_dir)
self._fs.makedirs(self.autoreport_dir, exist_ok=True)
options["autoReport.directory"] = self.autoreport_dir
os.environ["POPLAR_ENGINE_OPTIONS"] = json.dumps(options)

@@ -161,7 +172,7 @@ def _validate_opts(self, opts: 'poptorch.Options', training: bool) -> None:
rank_zero_warn(
f"Manual poptorch.Options set replicationFactor to {opts.replication_factor} "
f"which differs to the ipus={self.replication_factor} flag passed to the Trainer. "
f"Setting to {self.replication_factor} in the poptorch.Options.", UserWarning
f"Setting to {self.replication_factor} in the poptorch.Options."
)
opts.set(replication_factor=self.replication_factor)
if training:
@@ -171,13 +182,13 @@ def _validate_opts(self, opts: 'poptorch.Options', training: bool) -> None:
f"Training poptorch.Options set gradientAccumulation to {opts.Training.gradient_accumulation}. "
f"This is different to accumulate_grad_batches which was set to {accumulate_grad_batches}. "
f"To change gradientAccumulation, please set accumulate_grad_batches in the Trainer. "
f"Setting poptorch.Options gradientAccumulation to {accumulate_grad_batches}", UserWarning
f"Setting poptorch.Options gradientAccumulation to {accumulate_grad_batches}"
)
opts.Training.set(gradient_accumulation=accumulate_grad_batches)
elif opts.Training.gradient_accumulation != 1:
rank_zero_warn(
"Inference poptorch.Options should set gradientAccumulation to 1. "
"Setting gradientAccumulation to 1 for inference options.", UserWarning
"Setting gradientAccumulation to 1 for inference options."
)
opts.Training.set(gradient_accumulation=1)

Original file line number Diff line number Diff line change
@@ -637,7 +637,7 @@ def set_distributed_mode(self, distributed_backend: Optional[str] = None):
rank_zero_info(f'TPU available: {_TPU_AVAILABLE}, using: {num_tpu_cores} TPU cores')

num_ipus = self.ipus if self.ipus is not None else 0
rank_zero_info(f'IPU available: {_IPU_AVAILABLE}, using: {num_ipus} IPU cores')
rank_zero_info(f'IPU available: {_IPU_AVAILABLE}, using: {num_ipus} IPUs')

if torch.cuda.is_available() and self._device_type != DeviceType.GPU:
rank_zero_warn(
71 changes: 40 additions & 31 deletions tests/accelerators/test_ipu.py
Original file line number Diff line number Diff line change
@@ -98,25 +98,25 @@ def test_epoch_end(self, outputs) -> None:
@pytest.mark.skipif(_IPU_AVAILABLE, reason="test requires non-IPU machine")
def test_fail_if_no_ipus(tmpdir):
with pytest.raises(MisconfigurationException, match="IPU Accelerator requires IPU devices to run"):
Trainer(ipus=1)
Trainer(default_root_dir=tmpdir, ipus=1)

with pytest.raises(MisconfigurationException, match="IPU Accelerator requires IPU devices to run"):
Trainer(ipus=1, accelerator='ipu')
Trainer(default_root_dir=tmpdir, ipus=1, accelerator='ipu')


@RunIf(ipu=True)
def test_accelerator_selected(tmpdir):
SeanNaren marked this conversation as resolved.
Show resolved Hide resolved
trainer = Trainer(ipus=1)
trainer = Trainer(default_root_dir=tmpdir, ipus=1)
assert isinstance(trainer.accelerator, IPUAccelerator)
trainer = Trainer(ipus=1, accelerator='ipu')
trainer = Trainer(default_root_dir=tmpdir, ipus=1, accelerator='ipu')
assert isinstance(trainer.accelerator, IPUAccelerator)


SeanNaren marked this conversation as resolved.
Show resolved Hide resolved
@RunIf(ipu=True)
@pytest.mark.parametrize('ipus', [1, 4])
def test_all_stages(tmpdir, ipus):
model = IPUModel()
trainer = Trainer(fast_dev_run=True, ipus=ipus)
trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True, ipus=ipus)
trainer.fit(model)
trainer.validate(model)
trainer.test(model)
@@ -128,7 +128,7 @@ def test_all_stages(tmpdir, ipus):
def test_inference_only(tmpdir, ipus):
model = IPUModel()

trainer = Trainer(fast_dev_run=True, ipus=ipus)
trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True, ipus=ipus)
trainer.validate(model)
trainer.test(model)
trainer.predict(model, model.val_dataloader())
@@ -188,7 +188,7 @@ def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[st
raise SystemExit

model = IPUModel()
trainer = Trainer(fast_dev_run=True, ipus=1, precision=16, callbacks=TestCallback())
trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True, ipus=1, precision=16, callbacks=TestCallback())
with pytest.raises(SystemExit):
trainer.fit(model)

@@ -210,6 +210,7 @@ def on_train_start(self, trainer: Trainer, pl_module: LightningModule) -> None:

model = IPUModel()
trainer = Trainer(
default_root_dir=tmpdir,
fast_dev_run=True,
ipus=1,
precision=16,
@@ -234,7 +235,13 @@ def on_train_start(self, trainer: Trainer, pl_module: LightningModule) -> None:
raise SystemExit

model = IPUModel()
trainer = Trainer(fast_dev_run=True, ipus=1, plugins=IPUPlugin(device_iterations=2), callbacks=TestCallback())
trainer = Trainer(
default_root_dir=tmpdir,
fast_dev_run=True,
ipus=1,
plugins=IPUPlugin(device_iterations=2),
callbacks=TestCallback()
)
with pytest.raises(SystemExit):
trainer.fit(model)

@@ -254,7 +261,9 @@ def on_train_start(self, trainer: Trainer, pl_module: LightningModule) -> None:
raise SystemExit

model = IPUModel()
trainer = Trainer(fast_dev_run=True, ipus=1, accumulate_grad_batches=2, callbacks=TestCallback())
trainer = Trainer(
default_root_dir=tmpdir, fast_dev_run=True, ipus=1, accumulate_grad_batches=2, callbacks=TestCallback()
)
with pytest.raises(SystemExit):
trainer.fit(model)

@@ -298,7 +307,7 @@ def on_predict_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, da
assert torch.all(outputs == 4).item()

model = StageModel()
trainer = Trainer(fast_dev_run=True, ipus=1, callbacks=TestCallback())
trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True, ipus=1, callbacks=TestCallback())
trainer.fit(model)
trainer.test(model)
trainer.validate(model)
@@ -308,7 +317,7 @@ def on_predict_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, da
@RunIf(ipu=True)
def test_accumulate_grad_batches_dict_fails(tmpdir):
model = IPUModel()
trainer = Trainer(ipus=1, accumulate_grad_batches={0: 1})
trainer = Trainer(default_root_dir=tmpdir, ipus=1, accumulate_grad_batches={0: 1})
with pytest.raises(
MisconfigurationException, match="IPUs currently only support accumulate_grad_batches being an integer value."
):
@@ -318,7 +327,7 @@ def test_accumulate_grad_batches_dict_fails(tmpdir):
@RunIf(ipu=True)
def test_clip_gradients_fails(tmpdir):
model = IPUModel()
trainer = Trainer(ipus=1, gradient_clip_val=10)
trainer = Trainer(default_root_dir=tmpdir, ipus=1, gradient_clip_val=10)
with pytest.raises(MisconfigurationException, match="IPUs currently do not support clipping gradients."):
trainer.fit(model)

@@ -328,7 +337,12 @@ def test_autoreport(tmpdir):
"""Ensure autoreport dumps to a file."""
model = IPUModel()
autoreport_path = os.path.join(tmpdir, 'report/')
trainer = Trainer(ipus=1, fast_dev_run=True, plugins=IPUPlugin(autoreport=True, autoreport_dir=autoreport_path))
trainer = Trainer(
default_root_dir=tmpdir,
ipus=1,
fast_dev_run=True,
plugins=IPUPlugin(autoreport=True, autoreport_dir=autoreport_path)
)
trainer.fit(model)
assert os.path.exists(autoreport_path)
assert os.path.isfile(autoreport_path + 'profile.pop')
@@ -342,7 +356,10 @@ def test_manual_poptorch_opts(tmpdir):
training_opts = poptorch.Options()

trainer = Trainer(
ipus=1, fast_dev_run=True, plugins=IPUPlugin(inference_opts=inference_opts, training_opts=training_opts)
default_root_dir=tmpdir,
ipus=1,
fast_dev_run=True,
plugins=IPUPlugin(inference_opts=inference_opts, training_opts=training_opts)
)
trainer.fit(model)

@@ -368,6 +385,7 @@ def test_manual_poptorch_opts_ipu_count(tmpdir):
training_opts.replicationFactor(manual_ipus)

trainer = Trainer(
default_root_dir=tmpdir,
ipus=expected_ipus,
fast_dev_run=True,
plugins=IPUPlugin(inference_opts=inference_opts, training_opts=training_opts)
@@ -399,7 +417,10 @@ def test_manual_poptorch_opts_inference_grad_accum(tmpdir):
training_opts.Training.gradientAccumulation(1)

trainer = Trainer(
ipus=1, fast_dev_run=True, plugins=IPUPlugin(inference_opts=inference_opts, training_opts=training_opts)
default_root_dir=tmpdir,
ipus=1,
fast_dev_run=True,
plugins=IPUPlugin(inference_opts=inference_opts, training_opts=training_opts)
)
with pytest.warns(
UserWarning,
@@ -426,6 +447,7 @@ def test_manual_poptorch_opts_train_grad_accum(tmpdir):
training_opts.Training.gradientAccumulation(2)

trainer = Trainer(
default_root_dir=tmpdir,
ipus=1,
fast_dev_run=True,
accumulate_grad_batches=1,
@@ -451,7 +473,7 @@ def test_default_opts(tmpdir):

model = IPUModel()

trainer = Trainer(ipus=1, fast_dev_run=True)
trainer = Trainer(default_root_dir=tmpdir, ipus=1, fast_dev_run=True)
trainer.fit(model)
assert isinstance(trainer.accelerator.training_type_plugin, IPUPlugin)
inference_opts = trainer.accelerator.training_type_plugin.inference_opts
@@ -464,20 +486,7 @@ def test_default_opts(tmpdir):


@RunIf(ipu=True)
def test_clip_val_fail(tmpdir):
"""
Ensure if clipping value is greater than 0 or not None, we throw an exception.
"""

model = IPUModel()

trainer = Trainer(ipus=1, gradient_clip_val=10)
with pytest.raises(MisconfigurationException, match="IPUs currently do not support clipping gradients."):
trainer.fit(model)


@RunIf(ipu=True)
def test_multi_optimizers_fail(tmpdir):
def test_multi_optimizers_fails(tmpdir):
"""
Ensure if there are multiple optimizers, we throw an exception
"""
@@ -489,7 +498,7 @@ def configure_optimizers(self):

model = TestModel()

trainer = Trainer(ipus=1)
trainer = Trainer(default_root_dir=tmpdir, ipus=1)
with pytest.raises(MisconfigurationException, match="IPUs currently only support one optimizer."):
trainer.fit(model)