From 0bbb7887c253f31f81be0d627d2f957762cd253d Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Sat, 12 Aug 2023 15:06:24 +0200 Subject: [PATCH 01/12] E501. The test file failed with message: 'test_cuba.py:19:80: E501 line too long (80 > 79 characters)' Fixed by reformatting. --- tests/lava/lib/dl/slayer/block/test_cuba.py | 63 ++++++++++++--------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index 6d2e1d69..b1bb79f2 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -15,8 +15,9 @@ from lava.proc.conv import utils from lava.proc import io -verbose = True if (('-v' in sys.argv) or ('--verbose' in sys.argv)) else False -# Enabling torch sometimes causes multiprocessing error, especially in unittests +verbose = True if (("-v" in sys.argv) or ("--verbose" in sys.argv)) else False +# Enabling torch sometimes causes multiprocessing error, +# especially in unittests utils.TORCH_IS_AVAILABLE = False # seed = np.random.randint(1000) @@ -25,22 +26,22 @@ torch.manual_seed(seed) torch.cuda.manual_seed(seed) if verbose: - print(f'{seed=}') + print(f"{seed=}") if torch.cuda.is_available(): - device = torch.device('cuda') + device = torch.device("cuda") else: if verbose: - print('CUDA is not available in the system. ' - 'Testing for CPU version only.') - device = torch.device('cpu') + print( + "CUDA is not available in the system. " + "Testing for CPU version only." + ) + device = torch.device("cpu") -tempdir = os.path.dirname(__file__) + '/temp' +tempdir = os.path.dirname(__file__) + "/temp" os.makedirs(tempdir, exist_ok=True) -neuron_param = {'threshold': 0.5, - 'current_decay': 0.5, - 'voltage_decay': 0.5} +neuron_param = {"threshold": 0.5, "current_decay": 0.5, "voltage_decay": 0.5} class TestCUBA(unittest.TestCase): @@ -58,17 +59,18 @@ def test_dense_block(self): y = net(x) # export slayer network - net.export_hdf5(h5py.File(tempdir + '/cuba_dense.net', - 'w').create_group('layer/0')) + net.export_hdf5( + h5py.File(tempdir + "/cuba_dense.net", "w").create_group("layer/0") + ) # create equivalent lava network using netx and evaluate output - lava_net = netx.hdf5.Network(net_config=tempdir + '/cuba_dense.net') + lava_net = netx.hdf5.Network(net_config=tempdir + "/cuba_dense.net") source = io.source.RingBuffer(data=x[0]) sink = io.sink.RingBuffer(shape=lava_net.out.shape, buffer=time_steps) source.s_out.connect(lava_net.inp) lava_net.out.connect(sink.a_in) run_condition = RunSteps(num_steps=time_steps) - run_config = Loihi1SimCfg(select_tag='fixed_pt') + run_config = Loihi1SimCfg(select_tag="fixed_pt") lava_net.run(condition=run_condition, run_cfg=run_config) output = sink.data.get() lava_net.stop() @@ -76,9 +78,9 @@ def test_dense_block(self): if verbose: print() print(lava_net) - print('slayer output:') + print("slayer output:") print(y[0]) - print('lava output:') + print("lava output:") print(output) self.assertTrue(np.abs(y[0].data.numpy() - output).sum() == 0) @@ -93,25 +95,30 @@ def test_conv_block(self): time_steps = 10 # create slayer network and evaluate output - net = slayer.block.cuba.Conv(neuron_param, - in_features, out_features, kernel_size) - x = (torch.rand([1, in_features, - height, width, time_steps]) > 0.5).float() + net = slayer.block.cuba.Conv( + neuron_param, in_features, out_features, kernel_size + ) + x = ( + torch.rand([1, in_features, height, width, time_steps]) > 0.5 + ).float() y = net(x).permute((0, 3, 2, 1, 4)) # export slayer network - net.export_hdf5(h5py.File(tempdir + '/cuba_conv.net', - 'w').create_group('layer/0')) + net.export_hdf5( + h5py.File(tempdir + "/cuba_conv.net", "w").create_group("layer/0") + ) # create equivalent lava network using netx and evaluate output - lava_net = netx.hdf5.Network(net_config=tempdir + '/cuba_conv.net', - input_shape=(width, height, in_features)) + lava_net = netx.hdf5.Network( + net_config=tempdir + "/cuba_conv.net", + input_shape=(width, height, in_features), + ) source = io.source.RingBuffer(data=x[0].permute((2, 1, 0, 3))) sink = io.sink.RingBuffer(shape=lava_net.out.shape, buffer=time_steps) source.s_out.connect(lava_net.inp) lava_net.out.connect(sink.a_in) run_condition = RunSteps(num_steps=time_steps) - run_config = Loihi1SimCfg(select_tag='fixed_pt') + run_config = Loihi1SimCfg(select_tag="fixed_pt") lava_net.run(condition=run_condition, run_cfg=run_config) output = sink.data.get() lava_net.stop() @@ -119,9 +126,9 @@ def test_conv_block(self): if verbose: print() print(lava_net) - print('slayer output:') + print("slayer output:") print(y[0][0, 0, 0]) - print('lava output:') + print("lava output:") print(output[0, 0, 0]) self.assertTrue(np.abs(y[0].data.numpy() - output).sum() == 0) From b01623b04465e325368704c61d82318dc9354e8a Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Sat, 12 Aug 2023 15:17:52 +0200 Subject: [PATCH 02/12] Added test for Affine hdf5 export. The test will fail, if a network consisting of an Affine layer cannot be exported. This is the case, e.g., if the layer has no shape ('none'). In that case, h5py will interpret it as an ('o') object, which is not supported by the hdf5 format. --- tests/lava/lib/dl/slayer/block/test_cuba.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index b1bb79f2..f305e6f2 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -16,7 +16,7 @@ from lava.proc import io verbose = True if (("-v" in sys.argv) or ("--verbose" in sys.argv)) else False -# Enabling torch sometimes causes multiprocessing error, +# Enabling torch sometimes causes multiprocessing error, # especially in unittests utils.TORCH_IS_AVAILABLE = False @@ -47,6 +47,21 @@ class TestCUBA(unittest.TestCase): """Test CUBA blocks""" + def test_affine_block_hdf5_export(self): + """Test affine block hdf5 export.""" + in_features = 10 + out_features = 5 + + # create slayer network and evaluate output + net = slayer.block.cuba.Dense(neuron_param, in_features, out_features) + + # export slayer network + net.export_hdf5( + h5py.File(tempdir + "/cuba_affine.net", "w").create_group( + "layer/0" + ) + ) + def test_dense_block(self): """Test dense block with lava process implementation.""" in_features = 10 From 469dead5faf304d145611ded218de2dd61b02078 Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Sat, 12 Aug 2023 15:31:19 +0200 Subject: [PATCH 03/12] Dense -> Affine. --- tests/lava/lib/dl/slayer/block/test_cuba.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index f305e6f2..3401af23 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -53,7 +53,7 @@ def test_affine_block_hdf5_export(self): out_features = 5 # create slayer network and evaluate output - net = slayer.block.cuba.Dense(neuron_param, in_features, out_features) + net = slayer.block.cuba.Affine(neuron_param, in_features, out_features) # export slayer network net.export_hdf5( From 5d5885e17e1d827294abef6ce35f488bc676de47 Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Sat, 12 Aug 2023 15:49:21 +0200 Subject: [PATCH 04/12] Added shape. The shape of the Affine layer is set with respect to the output of the respective Dense layer. Previously this was 'None', resulting in an error while exporting to hdf5 due to being interpreted as object type by h5py. Now everything works due to being set to Float. --- src/lava/lib/dl/slayer/block/cuba.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lava/lib/dl/slayer/block/cuba.py b/src/lava/lib/dl/slayer/block/cuba.py index 14b71be0..60a286a0 100644 --- a/src/lava/lib/dl/slayer/block/cuba.py +++ b/src/lava/lib/dl/slayer/block/cuba.py @@ -69,6 +69,8 @@ def __init__(self, *args, **kwargs): if 'pre_hook_fx' not in kwargs.keys(): self.synapse.pre_hook_fx = self.neuron.quantize_8bit self.neuron._threshold = None + # set the shape according to synapse output + self.neuron.shape = self.synapse.out_channels # this disables spike and reset in dynamics del self.synapse_params From 61fd2b6f4c0f5c07c5b7add864b96daaa3f75746 Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Thu, 17 Aug 2023 12:50:18 +0200 Subject: [PATCH 05/12] Changed the shape to 'torch.Size()'. --- src/lava/lib/dl/slayer/block/cuba.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lava/lib/dl/slayer/block/cuba.py b/src/lava/lib/dl/slayer/block/cuba.py index 60a286a0..263f4d5b 100644 --- a/src/lava/lib/dl/slayer/block/cuba.py +++ b/src/lava/lib/dl/slayer/block/cuba.py @@ -3,6 +3,7 @@ """CUBA-LIF layer blocks""" +import numpy as np import torch from . import base @@ -70,7 +71,7 @@ def __init__(self, *args, **kwargs): self.synapse.pre_hook_fx = self.neuron.quantize_8bit self.neuron._threshold = None # set the shape according to synapse output - self.neuron.shape = self.synapse.out_channels + self.neuron.shape = torch.Size([self.synapse.out_channels]) # this disables spike and reset in dynamics del self.synapse_params From bae1d312a40554c48dcd472a34d36243675819cc Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Thu, 17 Aug 2023 12:59:47 +0200 Subject: [PATCH 06/12] Added two tests for different modes of the "dynamics" flag. Currently, both tests are failing. This is due to 'l.394' of '/lib/dl/slayer/block/base.py' not being executed. This has to be fixed (?). --- tests/lava/lib/dl/slayer/block/test_cuba.py | 52 +++++++++++++++++---- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index 3401af23..effe02dc 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -47,20 +47,56 @@ class TestCUBA(unittest.TestCase): """Test CUBA blocks""" - def test_affine_block_hdf5_export(self): - """Test affine block hdf5 export.""" + def test_affine_block_hdf5_export_dynamics_false(self): + """Test affine block hdf5 export in dynamics=false mode.""" in_features = 10 out_features = 5 - # create slayer network and evaluate output - net = slayer.block.cuba.Affine(neuron_param, in_features, out_features) + net = slayer.block.cuba.Affine( + neuron_params=neuron_param, + in_neurons=in_features, + out_neurons=out_features, + dynamics=False, + count_log=False, + ) # export slayer network - net.export_hdf5( - h5py.File(tempdir + "/cuba_affine.net", "w").create_group( - "layer/0" - ) + h = h5py.File(tempdir + "/cuba_affine_dynamics_true.net", "w") + net.export_hdf5(h.create_group("layer/0")) + + # reload net from h5 and check if 'neuron' exists. + lava_net = netx.hdf5.Network( + net_config=tempdir + "/cuba_affine_dynamics_true.net" ) + layer = lava_net.layers[0] + + assert "neuron" not in layer.__dict__ + + def test_affine_block_hdf5_export_dynamics_true(self): + """Test affine block hdf5 export in dynamics=true mode.""" + in_features = 10 + out_features = 5 + + net = slayer.block.cuba.Affine( + neuron_params=neuron_param, + in_neurons=in_features, + out_neurons=out_features, + dynamics=True, + count_log=False, + ) + + # export slayer network + h = h5py.File(tempdir + "/cuba_affine_dynamics_false.net", "w") + net.export_hdf5(h.create_group("layer/0")) + + # reload net from h5 and check if 'vThMant' is '(1 << 17)'. + lava_net = netx.hdf5.Network( + net_config=tempdir + "/cuba_affine_dynamics_false.net" + ) + layer = lava_net.layers[0] + neuron = layer.__dict__["neuron"].__dict__ + + assert neuron["vThMant"] == (1 << 18) - 1 def test_dense_block(self): """Test dense block with lava process implementation.""" From a8bdd55bd80bd9c7029c4f456e3696b41fae8e9c Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Thu, 17 Aug 2023 16:24:27 +0200 Subject: [PATCH 07/12] Some corrections to the temp_file names. --- tests/lava/lib/dl/slayer/block/test_cuba.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index effe02dc..87922374 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -61,16 +61,16 @@ def test_affine_block_hdf5_export_dynamics_false(self): ) # export slayer network - h = h5py.File(tempdir + "/cuba_affine_dynamics_true.net", "w") + h = h5py.File(tempdir + "/cuba_affine_dynamics_false.net", "w") net.export_hdf5(h.create_group("layer/0")) # reload net from h5 and check if 'neuron' exists. lava_net = netx.hdf5.Network( - net_config=tempdir + "/cuba_affine_dynamics_true.net" + net_config=tempdir + "/cuba_affine_dynamics_false.net" ) layer = lava_net.layers[0] - assert "neuron" not in layer.__dict__ + assert 'neuron' not in layer.__dict__.keys() def test_affine_block_hdf5_export_dynamics_true(self): """Test affine block hdf5 export in dynamics=true mode.""" @@ -86,12 +86,12 @@ def test_affine_block_hdf5_export_dynamics_true(self): ) # export slayer network - h = h5py.File(tempdir + "/cuba_affine_dynamics_false.net", "w") + h = h5py.File(tempdir + "/cuba_affine_dynamics_true.net", "w") net.export_hdf5(h.create_group("layer/0")) # reload net from h5 and check if 'vThMant' is '(1 << 17)'. lava_net = netx.hdf5.Network( - net_config=tempdir + "/cuba_affine_dynamics_false.net" + net_config=tempdir + "/cuba_affine_dynamics_true.net" ) layer = lava_net.layers[0] neuron = layer.__dict__["neuron"].__dict__ From 4a5a36a39bda245f5f3f3c4584fafcd19090c98a Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Fri, 18 Aug 2023 09:22:19 +0200 Subject: [PATCH 08/12] Added 'vThMant' export in 'base.py'. --- src/lava/lib/dl/slayer/block/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lava/lib/dl/slayer/block/base.py b/src/lava/lib/dl/slayer/block/base.py index f491c91e..88cc7760 100644 --- a/src/lava/lib/dl/slayer/block/base.py +++ b/src/lava/lib/dl/slayer/block/base.py @@ -391,8 +391,11 @@ def delay(d): self.delay.clamp() # clamp the delay value handle.create_dataset('delay', data=delay(self.delay)) - # for key, value in self.neuron.device_params.items(): - # handle.create_dataset(f'neuron/{key}', data=value) + if self.dynamics is True: + for key, value in self.neuron.device_params.items(): + if key == 'vThMant': + value = (1 << 17) - 1 # set the maximum possible threshold + handle.create_dataset(f'neuron/{key}', data=value) class AbstractTimeDecimation(torch.nn.Module): From ddc264800a16c63b767b1e897cf09950470f48e8 Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Fri, 18 Aug 2023 09:58:38 +0200 Subject: [PATCH 09/12] Added modiefied tests. Still failing. --- tests/lava/lib/dl/slayer/block/test_cuba.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index 87922374..711e363c 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -68,9 +68,8 @@ def test_affine_block_hdf5_export_dynamics_false(self): lava_net = netx.hdf5.Network( net_config=tempdir + "/cuba_affine_dynamics_false.net" ) - layer = lava_net.layers[0] - assert 'neuron' not in layer.__dict__.keys() + self.assertTrue('neuron' in lava_net.net_config['layer'][0].keys()) def test_affine_block_hdf5_export_dynamics_true(self): """Test affine block hdf5 export in dynamics=true mode.""" @@ -96,7 +95,7 @@ def test_affine_block_hdf5_export_dynamics_true(self): layer = lava_net.layers[0] neuron = layer.__dict__["neuron"].__dict__ - assert neuron["vThMant"] == (1 << 18) - 1 + self.assertTrue(neuron["vThMant"] == (1 << 18) - 1) def test_dense_block(self): """Test dense block with lava process implementation.""" From 0009b7edb45787c5d4ba28fe66bb67187202fe09 Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Fri, 18 Aug 2023 10:00:45 +0200 Subject: [PATCH 10/12] 'neuron' in ... -> 'neuron' not in. 1/2 pass. --- tests/lava/lib/dl/slayer/block/test_cuba.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index 711e363c..e3bcf90b 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -69,7 +69,7 @@ def test_affine_block_hdf5_export_dynamics_false(self): net_config=tempdir + "/cuba_affine_dynamics_false.net" ) - self.assertTrue('neuron' in lava_net.net_config['layer'][0].keys()) + self.assertTrue('neuron' not in lava_net.net_config['layer'][0].keys()) def test_affine_block_hdf5_export_dynamics_true(self): """Test affine block hdf5 export in dynamics=true mode.""" From 482aba019c8d66bfc2edb7355a34815ac042874a Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Fri, 18 Aug 2023 11:59:14 +0200 Subject: [PATCH 11/12] Set threshold=-1 and corrected 1 << 17 to 1 << 18. --- src/lava/lib/dl/slayer/block/base.py | 2 +- src/lava/lib/dl/slayer/block/cuba.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lava/lib/dl/slayer/block/base.py b/src/lava/lib/dl/slayer/block/base.py index 88cc7760..546f1600 100644 --- a/src/lava/lib/dl/slayer/block/base.py +++ b/src/lava/lib/dl/slayer/block/base.py @@ -394,7 +394,7 @@ def delay(d): if self.dynamics is True: for key, value in self.neuron.device_params.items(): if key == 'vThMant': - value = (1 << 17) - 1 # set the maximum possible threshold + value = (1 << 18) - 1 # set the maximum possible threshold handle.create_dataset(f'neuron/{key}', data=value) diff --git a/src/lava/lib/dl/slayer/block/cuba.py b/src/lava/lib/dl/slayer/block/cuba.py index 263f4d5b..2c46dcda 100644 --- a/src/lava/lib/dl/slayer/block/cuba.py +++ b/src/lava/lib/dl/slayer/block/cuba.py @@ -69,7 +69,11 @@ def __init__(self, *args, **kwargs): self.synapse = synapse.Dense(**self.synapse_params) if 'pre_hook_fx' not in kwargs.keys(): self.synapse.pre_hook_fx = self.neuron.quantize_8bit - self.neuron._threshold = None + # if 'dynamics=True', set threshold to not 'none' value + if self.dynamics: + self.neuron._threshold = -1 + else: + self.neuron._threshold = None # set the shape according to synapse output self.neuron.shape = torch.Size([self.synapse.out_channels]) # this disables spike and reset in dynamics From 661f29fc6b6c9a14b021e24e8706e74cb9e3d9cb Mon Sep 17 00:00:00 2001 From: Alexander Henkes Date: Mon, 28 Aug 2023 16:49:07 +0200 Subject: [PATCH 12/12] Check 'vThMant' in h5 file. --- tests/lava/lib/dl/slayer/block/test_cuba.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/lava/lib/dl/slayer/block/test_cuba.py b/tests/lava/lib/dl/slayer/block/test_cuba.py index e3bcf90b..a55858d8 100644 --- a/tests/lava/lib/dl/slayer/block/test_cuba.py +++ b/tests/lava/lib/dl/slayer/block/test_cuba.py @@ -69,7 +69,7 @@ def test_affine_block_hdf5_export_dynamics_false(self): net_config=tempdir + "/cuba_affine_dynamics_false.net" ) - self.assertTrue('neuron' not in lava_net.net_config['layer'][0].keys()) + self.assertTrue("neuron" not in lava_net.net_config["layer"][0].keys()) def test_affine_block_hdf5_export_dynamics_true(self): """Test affine block hdf5 export in dynamics=true mode.""" @@ -89,13 +89,17 @@ def test_affine_block_hdf5_export_dynamics_true(self): net.export_hdf5(h.create_group("layer/0")) # reload net from h5 and check if 'vThMant' is '(1 << 17)'. - lava_net = netx.hdf5.Network( - net_config=tempdir + "/cuba_affine_dynamics_true.net" - ) - layer = lava_net.layers[0] - neuron = layer.__dict__["neuron"].__dict__ + # lava_net = netx.hdf5.Network( + # net_config=tempdir + "/cuba_affine_dynamics_true.net" + # ) + # layer = lava_net.layers[0] + # neuron = layer.__dict__["neuron"].__dict__ + + # load network file and check neuron + with h5py.File(tempdir + "/cuba_affine_dynamics_true.net", "r") as hf: + vThMant = np.array(hf["layer"]["0"]["neuron"]["vThMant"]) - self.assertTrue(neuron["vThMant"] == (1 << 18) - 1) + self.assertTrue(vThMant == (1 << 18) - 1) def test_dense_block(self): """Test dense block with lava process implementation."""